Delete NoteDb migration code
Change-Id: I31b79a49f46cd60671db8de5b2378d43c6eed9cd
diff --git a/java/com/google/gerrit/pgm/Daemon.java b/java/com/google/gerrit/pgm/Daemon.java
index 016ad1a..e7e8ea3 100644
--- a/java/com/google/gerrit/pgm/Daemon.java
+++ b/java/com/google/gerrit/pgm/Daemon.java
@@ -15,7 +15,6 @@
package com.google.gerrit.pgm;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Objects.requireNonNull;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
@@ -85,8 +84,6 @@
import com.google.gerrit.server.mail.receive.MailReceiver;
import com.google.gerrit.server.mail.send.SmtpEmailSender;
import com.google.gerrit.server.mime.MimeUtil2Module;
-import com.google.gerrit.server.notedb.rebuild.NoteDbMigrator;
-import com.google.gerrit.server.notedb.rebuild.OnlineNoteDbMigrator;
import com.google.gerrit.server.patch.DiffExecutorModule;
import com.google.gerrit.server.permissions.DefaultPermissionBackendModule;
import com.google.gerrit.server.plugins.PluginGuiceEnvironment;
@@ -126,7 +123,6 @@
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jgit.lib.Config;
import org.kohsuke.args4j.Option;
-import org.kohsuke.args4j.spi.ExplicitBooleanOptionHandler;
/** Run SSH daemon portions of Gerrit. */
public class Daemon extends SiteProgram {
@@ -175,15 +171,6 @@
@Option(name = "--stop-only", usage = "Stop the daemon", hidden = true)
private boolean stopOnly;
- @Option(
- name = "--migrate-to-note-db",
- usage = "Automatically migrate changes to NoteDb",
- handler = ExplicitBooleanOptionHandler.class)
- private boolean migrateToNoteDb;
-
- @Option(name = "--trial", usage = "(With --migrate-to-note-db) " + MigrateToNoteDb.TRIAL_USAGE)
- private boolean trial;
-
private final LifecycleManager manager = new LifecycleManager();
private Injector dbInjector;
private Injector cfgInjector;
@@ -492,9 +479,6 @@
modules.add(new AccountDeactivator.Module());
modules.add(new ChangeCleanupRunner.Module());
}
- if (migrateToNoteDb()) {
- modules.add(new OnlineNoteDbMigrator.Module(trial));
- }
if (testSysModule != null) {
modules.add(testSysModule);
}
@@ -504,18 +488,11 @@
ModuleOverloader.override(modules, LibModuleLoader.loadModules(cfgInjector)));
}
- private boolean migrateToNoteDb() {
- return migrateToNoteDb || NoteDbMigrator.getAutoMigrate(requireNonNull(config));
- }
-
private Module createIndexModule() {
if (luceneModule != null) {
return luceneModule;
}
- boolean onlineUpgrade =
- VersionManager.getOnlineUpgrade(config)
- // Schema upgrade is handled by OnlineNoteDbMigrator in this case.
- && !migrateToNoteDb();
+ boolean onlineUpgrade = VersionManager.getOnlineUpgrade(config);
switch (indexType) {
case LUCENE:
return onlineUpgrade
diff --git a/java/com/google/gerrit/pgm/MigrateToNoteDb.java b/java/com/google/gerrit/pgm/MigrateToNoteDb.java
deleted file mode 100644
index 21799be..0000000
--- a/java/com/google/gerrit/pgm/MigrateToNoteDb.java
+++ /dev/null
@@ -1,195 +0,0 @@
-// Copyright (C) 2014 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.pgm;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.stream.Collectors.joining;
-import static java.util.stream.Collectors.toList;
-
-import com.google.common.collect.ImmutableList;
-import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.gerrit.lifecycle.LifecycleManager;
-import com.google.gerrit.pgm.util.BatchProgramModule;
-import com.google.gerrit.pgm.util.RuntimeShutdown;
-import com.google.gerrit.pgm.util.SiteProgram;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.server.change.ChangeResource;
-import com.google.gerrit.server.git.GarbageCollection;
-import com.google.gerrit.server.index.DummyIndexModule;
-import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
-import com.google.gerrit.server.notedb.rebuild.GcAllUsers;
-import com.google.gerrit.server.notedb.rebuild.NoteDbMigrator;
-import com.google.gerrit.server.plugins.PluginGuiceEnvironment;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Provider;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.List;
-import org.kohsuke.args4j.Option;
-import org.kohsuke.args4j.spi.ExplicitBooleanOptionHandler;
-
-// TODO(dborowitz): Delete this program.
-public class MigrateToNoteDb extends SiteProgram {
- static final String TRIAL_USAGE =
- "Trial mode: migrate changes and turn on reading from NoteDb, but leave ReviewDb as the"
- + " source of truth";
-
- @Option(name = "--threads", usage = "Number of threads to use for rebuilding NoteDb")
- private int threads = Runtime.getRuntime().availableProcessors();
-
- @Option(
- name = "--project",
- usage =
- "Only rebuild these projects, do no other migration; incompatible with --change;"
- + " recommended for debugging only")
- private List<String> projects = new ArrayList<>();
-
- @Option(
- name = "--change",
- usage =
- "Only rebuild these changes, do no other migration; incompatible with --project;"
- + " recommended for debugging only")
- private List<Integer> changes = new ArrayList<>();
-
- @Option(
- name = "--force",
- usage =
- "Force rebuilding changes where ReviewDb is still the source of truth, even if they"
- + " were previously migrated")
- private boolean force;
-
- @Option(name = "--trial", usage = TRIAL_USAGE)
- private boolean trial;
-
- @Option(
- name = "--sequence-gap",
- usage =
- "gap in change sequence numbers between last ReviewDb number and first NoteDb number;"
- + " negative indicates using the value of noteDb.changes.initialSequenceGap (default"
- + " 1000)")
- private int sequenceGap;
-
- @Option(
- name = "--reindex",
- usage =
- "Reindex all changes after migration; defaults to false in trial mode, true otherwise",
- handler = ExplicitBooleanOptionHandler.class)
- private Boolean reindex;
-
- private Injector dbInjector;
- private Injector sysInjector;
- private LifecycleManager dbManager;
- private LifecycleManager sysManager;
-
- @Inject private GcAllUsers gcAllUsers;
- @Inject private Provider<NoteDbMigrator.Builder> migratorBuilderProvider;
-
- @Override
- public int run() throws Exception {
- RuntimeShutdown.add(this::stop);
- try {
- mustHaveValidSite();
- dbInjector = createDbInjector();
-
- dbManager = new LifecycleManager();
- dbManager.add(dbInjector);
- dbManager.start();
-
- sysInjector = createSysInjector();
- sysInjector.injectMembers(this);
- sysManager = new LifecycleManager();
- sysManager.add(sysInjector);
- sysInjector
- .getInstance(PluginGuiceEnvironment.class)
- .setDbCfgInjector(dbInjector, dbInjector);
- sysManager.start();
-
- try (NoteDbMigrator migrator =
- migratorBuilderProvider
- .get()
- .setThreads(threads)
- .setProgressOut(System.err)
- .setProjects(projects.stream().map(Project.NameKey::new).collect(toList()))
- .setChanges(changes.stream().map(Change.Id::new).collect(toList()))
- .setTrialMode(trial)
- .setForceRebuild(force)
- .setSequenceGap(sequenceGap)
- .build()) {
- if (!projects.isEmpty() || !changes.isEmpty()) {
- migrator.rebuild();
- } else {
- migrator.migrate();
- }
- }
- try (PrintWriter w = new PrintWriter(new OutputStreamWriter(System.out, UTF_8), true)) {
- gcAllUsers.run(w);
- }
- } finally {
- stop();
- }
-
- boolean reindex = firstNonNull(this.reindex, !trial);
- if (!reindex) {
- return 0;
- }
- // Reindex all indices, to save the user from having to run yet another program by hand while
- // their server is offline.
- List<String> reindexArgs =
- ImmutableList.of(
- "--site-path",
- getSitePath().toString(),
- "--threads",
- Integer.toString(threads),
- "--index",
- ChangeSchemaDefinitions.NAME);
- System.out.println("Migration complete, reindexing changes with:");
- System.out.println(" reindex " + reindexArgs.stream().collect(joining(" ")));
- Reindex reindexPgm = new Reindex();
- return reindexPgm.main(reindexArgs.stream().toArray(String[]::new));
- }
-
- private Injector createSysInjector() {
- return dbInjector.createChildInjector(
- new FactoryModule() {
- @Override
- public void configure() {
- install(dbInjector.getInstance(BatchProgramModule.class));
- install(new DummyIndexModule());
- factory(ChangeResource.Factory.class);
- factory(GarbageCollection.Factory.class);
- }
- });
- }
-
- private void stop() {
- try {
- LifecycleManager m = sysManager;
- sysManager = null;
- if (m != null) {
- m.stop();
- }
- } finally {
- LifecycleManager m = dbManager;
- dbManager = null;
- if (m != null) {
- m.stop();
- }
- }
- }
-}
diff --git a/java/com/google/gerrit/server/config/GerritServerIdProvider.java b/java/com/google/gerrit/server/config/GerritServerIdProvider.java
index c609cc4..4898f55 100644
--- a/java/com/google/gerrit/server/config/GerritServerIdProvider.java
+++ b/java/com/google/gerrit/server/config/GerritServerIdProvider.java
@@ -48,9 +48,7 @@
// We're not generally supposed to do work in provider constructors, but this is a bit of a
// special case because we really need to have the ID available by the time the dbInjector
- // is created. This even applies during MigrateToNoteDb, which otherwise would have been a
- // reasonable place to do the ID generation. Fortunately, it's not much work, and it happens
- // once.
+ // is created. Fortunately, it's not much work, and it happens once.
id = generate();
Config newCfg = readGerritConfig(sitePaths);
newCfg.setString(SECTION, null, KEY, id);
diff --git a/java/com/google/gerrit/server/notedb/NotesMigration.java b/java/com/google/gerrit/server/notedb/NotesMigration.java
index c7acaf1..28754a6 100644
--- a/java/com/google/gerrit/server/notedb/NotesMigration.java
+++ b/java/com/google/gerrit/server/notedb/NotesMigration.java
@@ -46,8 +46,7 @@
* <p>This class controls the state of the migration according to options in {@code gerrit.config}.
* In general, any changes to these options should only be made by adventurous administrators, who
* know what they're doing, on non-production data, for the purposes of testing the NoteDb
- * implementation. Changing options quite likely requires re-running {@code MigrateToNoteDb}. For
- * these reasons, the options remain undocumented.
+ * implementation.
*
* <p><strong>Note:</strong> Callers should not assume the values returned by {@code
* NotesMigration}'s methods will not change in a running server.
diff --git a/java/com/google/gerrit/server/notedb/PrimaryStorageMigrator.java b/java/com/google/gerrit/server/notedb/PrimaryStorageMigrator.java
deleted file mode 100644
index 7b427b4..0000000
--- a/java/com/google/gerrit/server/notedb/PrimaryStorageMigrator.java
+++ /dev/null
@@ -1,510 +0,0 @@
-// Copyright (C) 2017 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.server.notedb;
-
-import static com.google.common.base.Preconditions.checkState;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import static java.util.concurrent.TimeUnit.SECONDS;
-
-import com.github.rholder.retry.RetryException;
-import com.github.rholder.retry.Retryer;
-import com.github.rholder.retry.RetryerBuilder;
-import com.github.rholder.retry.StopStrategies;
-import com.github.rholder.retry.WaitStrategies;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.common.Nullable;
-import com.google.gerrit.extensions.restapi.RestApiException;
-import com.google.gerrit.reviewdb.client.Account;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
-import com.google.gerrit.reviewdb.server.ReviewDbUtil;
-import com.google.gerrit.server.InternalUser;
-import com.google.gerrit.server.config.AllUsersName;
-import com.google.gerrit.server.config.GerritServerConfig;
-import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.git.RepoRefCache;
-import com.google.gerrit.server.index.change.ChangeField;
-import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
-import com.google.gerrit.server.notedb.NoteDbChangeState.RefState;
-import com.google.gerrit.server.notedb.rebuild.ChangeRebuilder;
-import com.google.gerrit.server.query.change.ChangeData;
-import com.google.gerrit.server.query.change.InternalChangeQuery;
-import com.google.gerrit.server.update.BatchUpdate;
-import com.google.gerrit.server.update.BatchUpdateOp;
-import com.google.gerrit.server.update.ChangeContext;
-import com.google.gerrit.server.update.RetryHelper;
-import com.google.gerrit.server.update.UpdateException;
-import com.google.gerrit.server.util.time.TimeUtil;
-import com.google.gwtorm.server.AtomicUpdate;
-import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.OrmRuntimeException;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-import java.io.IOException;
-import java.sql.Timestamp;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.eclipse.jgit.errors.RepositoryNotFoundException;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.Repository;
-
-/** Helper to migrate the {@link PrimaryStorage} of individual changes. */
-@Singleton
-public class PrimaryStorageMigrator {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- /**
- * Exception thrown during migration if the change has no {@code noteDbState} field at the
- * beginning of the migration.
- */
- public static class NoNoteDbStateException extends RuntimeException {
- private static final long serialVersionUID = 1L;
-
- private NoNoteDbStateException(Change.Id id) {
- super("change " + id + " has no note_db_state; rebuild it first");
- }
- }
-
- private final AllUsersName allUsers;
- private final ChangeNotes.Factory changeNotesFactory;
- private final ChangeRebuilder rebuilder;
- private final ChangeUpdate.Factory updateFactory;
- private final GitRepositoryManager repoManager;
- private final InternalUser.Factory internalUserFactory;
- private final Provider<InternalChangeQuery> queryProvider;
- private final Provider<ReviewDb> db;
- private final RetryHelper retryHelper;
-
- private final long skewMs;
- private final long timeoutMs;
- private final Retryer<NoteDbChangeState> testEnsureRebuiltRetryer;
-
- @Inject
- PrimaryStorageMigrator(
- @GerritServerConfig Config cfg,
- Provider<ReviewDb> db,
- GitRepositoryManager repoManager,
- AllUsersName allUsers,
- ChangeRebuilder rebuilder,
- ChangeNotes.Factory changeNotesFactory,
- Provider<InternalChangeQuery> queryProvider,
- ChangeUpdate.Factory updateFactory,
- InternalUser.Factory internalUserFactory,
- RetryHelper retryHelper) {
- this(
- cfg,
- db,
- repoManager,
- allUsers,
- rebuilder,
- null,
- changeNotesFactory,
- queryProvider,
- updateFactory,
- internalUserFactory,
- retryHelper);
- }
-
- @VisibleForTesting
- public PrimaryStorageMigrator(
- Config cfg,
- Provider<ReviewDb> db,
- GitRepositoryManager repoManager,
- AllUsersName allUsers,
- ChangeRebuilder rebuilder,
- @Nullable Retryer<NoteDbChangeState> testEnsureRebuiltRetryer,
- ChangeNotes.Factory changeNotesFactory,
- Provider<InternalChangeQuery> queryProvider,
- ChangeUpdate.Factory updateFactory,
- InternalUser.Factory internalUserFactory,
- RetryHelper retryHelper) {
- this.db = db;
- this.repoManager = repoManager;
- this.allUsers = allUsers;
- this.rebuilder = rebuilder;
- this.testEnsureRebuiltRetryer = testEnsureRebuiltRetryer;
- this.changeNotesFactory = changeNotesFactory;
- this.queryProvider = queryProvider;
- this.updateFactory = updateFactory;
- this.internalUserFactory = internalUserFactory;
- this.retryHelper = retryHelper;
- skewMs = NoteDbChangeState.getReadOnlySkew(cfg);
-
- String s = "notedb";
- timeoutMs =
- cfg.getTimeUnit(
- s,
- null,
- "primaryStorageMigrationTimeout",
- MILLISECONDS.convert(60, SECONDS),
- MILLISECONDS);
- }
-
- /**
- * Migrate a change's primary storage from ReviewDb to NoteDb.
- *
- * <p>This method will return only if the primary storage of the change is NoteDb afterwards. (It
- * may return early if the primary storage was already NoteDb.)
- *
- * <p>If this method throws an exception, then the primary storage of the change is probably not
- * NoteDb. (It is possible that the primary storage of the change is NoteDb in this case, but
- * there was an error reading the state.) Moreover, after an exception, the change may be
- * read-only until a lease expires. If the caller chooses to retry, they should wait until the
- * read-only lease expires; this method will fail relatively quickly if called on a read-only
- * change.
- *
- * <p>Note that if the change is read-only after this method throws an exception, that does not
- * necessarily guarantee that the read-only lease was acquired during that particular method
- * invocation; this call may have in fact failed because another thread acquired the lease first.
- *
- * @param id change ID.
- * @throws OrmException if a ReviewDb-level error occurs.
- * @throws IOException if a repo-level error occurs.
- */
- public void migrateToNoteDbPrimary(Change.Id id) throws OrmException, IOException {
- // Since there are multiple non-atomic steps in this method, we need to
- // consider what happens when there is another writer concurrent with the
- // thread executing this method.
- //
- // Let:
- // * OR = other writer writes noteDbState & new data to ReviewDb (in one
- // transaction)
- // * ON = other writer writes to NoteDb
- // * MRO = migrator sets state to read-only
- // * MR = ensureRebuilt writes rebuilt noteDbState to ReviewDb (but does not
- // otherwise update ReviewDb in this transaction)
- // * MN = ensureRebuilt writes rebuilt state to NoteDb
- //
- // Consider all the interleavings of these operations.
- //
- // * OR,ON,MRO,...
- // Other writer completes before migrator begins; this is not a concurrent
- // write.
- // * MRO,...,OR,...
- // OR will fail, since it atomically checks that the noteDbState is not
- // read-only before proceeding. This results in an exception, but not a
- // concurrent write.
- //
- // Thus all the "interesting" interleavings start with OR,MRO, and differ on
- // where ON falls relative to MR/MN.
- //
- // * OR,MRO,ON,MR,MN
- // The other NoteDb write succeeds despite the noteDbState being
- // read-only. Because the read-only state from MRO includes the update
- // from OR, the change is up-to-date at this point. Thus MR,MN is a no-op.
- // The end result is an up-to-date, read-only change.
- //
- // * OR,MRO,MR,ON,MN
- // The change is out-of-date when ensureRebuilt begins, because OR
- // succeeded but the corresponding ON has not happened yet. ON will
- // succeed, because there have been no intervening NoteDb writes. MN will
- // fail, because ON updated the state in NoteDb to something other than
- // what MR claimed. This leaves the change in an out-of-date, read-only
- // state.
- //
- // If this method threw an exception in this case, the change would
- // eventually switch back to read-write when the read-only lease expires,
- // so this situation is recoverable. However, it would be inconvenient for
- // a change to be read-only for so long.
- //
- // Thus, as an optimization, we have a retry loop that attempts
- // ensureRebuilt while still holding the same read-only lease. This
- // effectively results in the interleaving OR,MR,ON,MR,MN; in contrast
- // with the previous case, here, MR/MN actually rebuilds the change. In
- // the case of a write failure, MR/MN might fail and get retried again. If
- // it exceeds the maximum number of retries, an exception is thrown.
- //
- // * OR,MRO,MR,MN,ON
- // The change is out-of-date when ensureRebuilt begins. The change is
- // rebuilt, leaving a new state in NoteDb. ON will fail, because the old
- // NoteDb state has changed since the ref state was read when the update
- // began (prior to OR). This results in an exception from ON, but the end
- // result is still an up-to-date, read-only change. The end user that
- // initiated the other write observes an error, but this is no different
- // from other errors that need retrying, e.g. due to a backend write
- // failure.
-
- Stopwatch sw = Stopwatch.createStarted();
- Change readOnlyChange = setReadOnlyInReviewDb(id); // MRO
- if (readOnlyChange == null) {
- return; // Already migrated.
- }
-
- NoteDbChangeState rebuiltState;
- try {
- // MR,MN
- rebuiltState =
- ensureRebuiltRetryer(sw)
- .call(
- () ->
- ensureRebuilt(
- readOnlyChange.getProject(),
- id,
- NoteDbChangeState.parse(readOnlyChange)));
- } catch (RetryException | ExecutionException e) {
- throw new OrmException(e);
- }
-
- // At this point, the noteDbState in ReviewDb is read-only, and it is
- // guaranteed to match the state actually in NoteDb. Now it is safe to set
- // the primary storage to NoteDb.
-
- setPrimaryStorageNoteDb(id, rebuiltState);
- logger.atFine().log(
- "Migrated change %s to NoteDb primary in %sms", id, sw.elapsed(MILLISECONDS));
- }
-
- private Change setReadOnlyInReviewDb(Change.Id id) throws OrmException {
- AtomicBoolean alreadyMigrated = new AtomicBoolean(false);
- Change result =
- db().changes()
- .atomicUpdate(
- id,
- new AtomicUpdate<Change>() {
- @Override
- public Change update(Change change) {
- NoteDbChangeState state = NoteDbChangeState.parse(change);
- if (state == null) {
- // Could rebuild the change here, but that's more complexity, and this
- // normally shouldn't happen.
- //
- // Known cases where this happens are described in and handled by
- // NoteDbMigrator#canSkipPrimaryStorageMigration.
- throw new NoNoteDbStateException(id);
- }
- // If the change is already read-only, then the lease is held by another
- // (likely failed) migrator thread. Fail early, as we can't take over
- // the lease.
- NoteDbChangeState.checkNotReadOnly(change, skewMs);
- if (state.getPrimaryStorage() != PrimaryStorage.NOTE_DB) {
- Timestamp now = TimeUtil.nowTs();
- Timestamp until = new Timestamp(now.getTime() + timeoutMs);
- change.setNoteDbState(state.withReadOnlyUntil(until).toString());
- } else {
- alreadyMigrated.set(true);
- }
- return change;
- }
- });
- return alreadyMigrated.get() ? null : result;
- }
-
- private Retryer<NoteDbChangeState> ensureRebuiltRetryer(Stopwatch sw) {
- if (testEnsureRebuiltRetryer != null) {
- return testEnsureRebuiltRetryer;
- }
- // Retry the ensureRebuilt step with backoff until half the timeout has
- // expired, leaving the remaining half for the rest of the steps.
- long remainingNanos = (MILLISECONDS.toNanos(timeoutMs) / 2) - sw.elapsed(NANOSECONDS);
- remainingNanos = Math.max(remainingNanos, 0);
- return RetryerBuilder.<NoteDbChangeState>newBuilder()
- .retryIfException(e -> (e instanceof IOException) || (e instanceof OrmException))
- .withWaitStrategy(
- WaitStrategies.join(
- WaitStrategies.exponentialWait(250, MILLISECONDS),
- WaitStrategies.randomWait(50, MILLISECONDS)))
- .withStopStrategy(StopStrategies.stopAfterDelay(remainingNanos, NANOSECONDS))
- .build();
- }
-
- private NoteDbChangeState ensureRebuilt(
- Project.NameKey project, Change.Id id, NoteDbChangeState readOnlyState)
- throws IOException, OrmException, RepositoryNotFoundException {
- try (Repository changeRepo = repoManager.openRepository(project);
- Repository allUsersRepo = repoManager.openRepository(allUsers)) {
- if (!readOnlyState.isUpToDate(new RepoRefCache(changeRepo), new RepoRefCache(allUsersRepo))) {
- NoteDbUpdateManager.Result r = rebuilder.rebuildEvenIfReadOnly(db(), id);
- checkState(
- r.newState().getReadOnlyUntil().equals(readOnlyState.getReadOnlyUntil()),
- "state after rebuilding has different read-only lease: %s != %s",
- r.newState(),
- readOnlyState);
- readOnlyState = r.newState();
- }
- }
- return readOnlyState;
- }
-
- private void setPrimaryStorageNoteDb(Change.Id id, NoteDbChangeState expectedState)
- throws OrmException {
- db().changes()
- .atomicUpdate(
- id,
- new AtomicUpdate<Change>() {
- @Override
- public Change update(Change change) {
- NoteDbChangeState state = NoteDbChangeState.parse(change);
- if (!Objects.equals(state, expectedState)) {
- throw new OrmRuntimeException(badState(state, expectedState));
- }
- Timestamp until = state.getReadOnlyUntil().get();
- if (TimeUtil.nowTs().after(until)) {
- throw new OrmRuntimeException(
- "read-only lease on change " + id + " expired at " + until);
- }
- change.setNoteDbState(NoteDbChangeState.NOTE_DB_PRIMARY_STATE);
- return change;
- }
- });
- }
-
- private ReviewDb db() {
- return ReviewDbUtil.unwrapDb(db.get());
- }
-
- private String badState(NoteDbChangeState actual, NoteDbChangeState expected) {
- return "state changed unexpectedly: " + actual + " != " + expected;
- }
-
- public void migrateToReviewDbPrimary(Change.Id id, @Nullable Project.NameKey project)
- throws OrmException, IOException {
- // Migrating back to ReviewDb primary is much simpler than the original migration to NoteDb
- // primary, because when NoteDb is primary, each write only goes to one storage location rather
- // than both. We only need to consider whether a concurrent writer (OR) conflicts with the first
- // setReadOnlyInNoteDb step (MR) in this method.
- //
- // If OR wins, then either:
- // * MR will set read-only after OR is completed, which is not a concurrent write.
- // * MR will fail to set read-only with a lock failure. The caller will have to retry, but the
- // change is not in a read-only state, so behavior is not degraded in the meantime.
- //
- // If MR wins, then either:
- // * OR will fail with a read-only exception (via AbstractChangeNotes#apply).
- // * OR will fail with a lock failure.
- //
- // In all of these scenarios, the change is read-only if and only if MR succeeds.
- //
- // There will be no concurrent writes to ReviewDb for this change until
- // setPrimaryStorageReviewDb completes, because ReviewDb writes are not attempted when primary
- // storage is NoteDb. After the primary storage changes back, it is possible for subsequent
- // NoteDb writes to conflict with the releaseReadOnlyLeaseInNoteDb step, but at this point,
- // since ReviewDb is primary, we are back to ignoring them.
- Stopwatch sw = Stopwatch.createStarted();
- if (project == null) {
- project = getProject(id);
- }
- ObjectId newMetaId = setReadOnlyInNoteDb(project, id);
- rebuilder.rebuildReviewDb(db(), project, id);
- setPrimaryStorageReviewDb(id, newMetaId);
- releaseReadOnlyLeaseInNoteDb(project, id);
- logger.atFine().log(
- "Migrated change %s to ReviewDb primary in %sms", id, sw.elapsed(MILLISECONDS));
- }
-
- private ObjectId setReadOnlyInNoteDb(Project.NameKey project, Change.Id id)
- throws OrmException, IOException {
- Timestamp now = TimeUtil.nowTs();
- Timestamp until = new Timestamp(now.getTime() + timeoutMs);
- ChangeUpdate update =
- updateFactory.create(
- changeNotesFactory.createChecked(db.get(), project, id), internalUserFactory.create());
- update.setReadOnlyUntil(until);
- return update.commit();
- }
-
- private void setPrimaryStorageReviewDb(Change.Id id, ObjectId newMetaId)
- throws OrmException, IOException {
- ImmutableMap.Builder<Account.Id, ObjectId> draftIds = ImmutableMap.builder();
- try (Repository repo = repoManager.openRepository(allUsers)) {
- for (Ref draftRef :
- repo.getRefDatabase().getRefsByPrefix(RefNames.refsDraftCommentsPrefix(id))) {
- Account.Id accountId = Account.Id.fromRef(draftRef.getName());
- if (accountId != null) {
- draftIds.put(accountId, draftRef.getObjectId().copy());
- }
- }
- }
- NoteDbChangeState newState =
- new NoteDbChangeState(
- id,
- PrimaryStorage.REVIEW_DB,
- Optional.of(RefState.create(newMetaId, draftIds.build())),
- Optional.empty());
- db().changes()
- .atomicUpdate(
- id,
- new AtomicUpdate<Change>() {
- @Override
- public Change update(Change change) {
- if (PrimaryStorage.of(change) != PrimaryStorage.NOTE_DB) {
- throw new OrmRuntimeException(
- "change " + id + " is not NoteDb primary: " + change.getNoteDbState());
- }
- change.setNoteDbState(newState.toString());
- return change;
- }
- });
- }
-
- private void releaseReadOnlyLeaseInNoteDb(Project.NameKey project, Change.Id id)
- throws OrmException {
- // Use a BatchUpdate since ReviewDb is primary at this point, so it needs to reflect the update.
- // (In practice retrying won't happen, since we aren't using fused updates at this point.)
- try {
- retryHelper.execute(
- updateFactory -> {
- try (BatchUpdate bu =
- updateFactory.create(
- db.get(), project, internalUserFactory.create(), TimeUtil.nowTs())) {
- bu.addOp(
- id,
- new BatchUpdateOp() {
- @Override
- public boolean updateChange(ChangeContext ctx) {
- ctx.getUpdate(ctx.getChange().currentPatchSetId())
- .setReadOnlyUntil(new Timestamp(0));
- return true;
- }
- });
- bu.execute();
- return null;
- }
- });
- } catch (RestApiException | UpdateException e) {
- throw new OrmException(e);
- }
- }
-
- private Project.NameKey getProject(Change.Id id) throws OrmException {
- List<ChangeData> cds =
- queryProvider.get().setRequestedFields(ChangeField.PROJECT).byLegacyChangeId(id);
- Set<Project.NameKey> projects = new TreeSet<>();
- for (ChangeData cd : cds) {
- projects.add(cd.project());
- }
- if (projects.size() != 1) {
- throw new OrmException(
- "zero or multiple projects found for change "
- + id
- + ", must specify project explicitly: "
- + projects);
- }
- return projects.iterator().next();
- }
-}
diff --git a/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java b/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java
deleted file mode 100644
index f60744f..0000000
--- a/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java
+++ /dev/null
@@ -1,1035 +0,0 @@
-// Copyright (C) 2017 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.server.notedb.rebuild;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.gerrit.reviewdb.server.ReviewDbUtil.unwrapDb;
-import static com.google.gerrit.server.notedb.NotesMigration.SECTION_NOTE_DB;
-import static com.google.gerrit.server.notedb.NotesMigrationState.NOTE_DB;
-import static com.google.gerrit.server.notedb.NotesMigrationState.READ_WRITE_NO_SEQUENCE;
-import static com.google.gerrit.server.notedb.NotesMigrationState.READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY;
-import static com.google.gerrit.server.notedb.NotesMigrationState.READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY;
-import static com.google.gerrit.server.notedb.NotesMigrationState.WRITE;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Comparator.comparing;
-import static java.util.Objects.requireNonNull;
-import static java.util.stream.Collectors.joining;
-import static java.util.stream.Collectors.toList;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.MultimapBuilder;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.SetMultimap;
-import com.google.common.collect.Streams;
-import com.google.common.flogger.FluentLogger;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.gerrit.common.FormatUtil;
-import com.google.gerrit.common.Nullable;
-import com.google.gerrit.git.LockFailureException;
-import com.google.gerrit.git.RefUpdateUtil;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.server.ReviewDb;
-import com.google.gerrit.reviewdb.server.ReviewDbWrapper;
-import com.google.gerrit.server.GerritPersonIdent;
-import com.google.gerrit.server.InternalUser;
-import com.google.gerrit.server.Sequences;
-import com.google.gerrit.server.config.AllProjectsName;
-import com.google.gerrit.server.config.AllUsersName;
-import com.google.gerrit.server.config.GerritServerConfigProvider;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
-import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.git.WorkQueue;
-import com.google.gerrit.server.notedb.ChangeBundleReader;
-import com.google.gerrit.server.notedb.ChangeNotes;
-import com.google.gerrit.server.notedb.MutableNotesMigration;
-import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
-import com.google.gerrit.server.notedb.NoteDbTable;
-import com.google.gerrit.server.notedb.NoteDbUpdateManager;
-import com.google.gerrit.server.notedb.NotesMigrationState;
-import com.google.gerrit.server.notedb.PrimaryStorageMigrator;
-import com.google.gerrit.server.notedb.PrimaryStorageMigrator.NoNoteDbStateException;
-import com.google.gerrit.server.notedb.RepoSequence;
-import com.google.gerrit.server.notedb.rebuild.ChangeRebuilder.NoPatchSetsException;
-import com.google.gerrit.server.plugincontext.PluginSetContext;
-import com.google.gerrit.server.project.NoSuchChangeException;
-import com.google.gerrit.server.update.ChainedReceiveCommands;
-import com.google.gerrit.server.util.ManualRequestContext;
-import com.google.gerrit.server.util.ThreadLocalRequestContext;
-import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.errors.RepositoryNotFoundException;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.internal.storage.file.PackInserter;
-import org.eclipse.jgit.lib.BatchRefUpdate;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.ObjectInserter;
-import org.eclipse.jgit.lib.ObjectReader;
-import org.eclipse.jgit.lib.PersonIdent;
-import org.eclipse.jgit.lib.ProgressMonitor;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.lib.TextProgressMonitor;
-import org.eclipse.jgit.revwalk.RevWalk;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.transport.ReceiveCommand;
-import org.eclipse.jgit.util.FS;
-import org.eclipse.jgit.util.io.NullOutputStream;
-
-/** One stop shop for migrating a site's change storage from ReviewDb to NoteDb. */
-public class NoteDbMigrator implements AutoCloseable {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private static final String AUTO_MIGRATE = "autoMigrate";
- private static final String TRIAL = "trial";
-
- public static boolean getAutoMigrate(Config cfg) {
- return cfg.getBoolean(SECTION_NOTE_DB, NoteDbTable.CHANGES.key(), AUTO_MIGRATE, false);
- }
-
- private static void setAutoMigrate(Config cfg, boolean autoMigrate) {
- cfg.setBoolean(SECTION_NOTE_DB, NoteDbTable.CHANGES.key(), AUTO_MIGRATE, autoMigrate);
- }
-
- public static boolean getTrialMode(Config cfg) {
- return cfg.getBoolean(SECTION_NOTE_DB, NoteDbTable.CHANGES.key(), TRIAL, false);
- }
-
- public static void setTrialMode(Config cfg, boolean trial) {
- cfg.setBoolean(SECTION_NOTE_DB, NoteDbTable.CHANGES.key(), TRIAL, trial);
- }
-
- public static class Builder {
- private final Config cfg;
- private final SitePaths sitePaths;
- private final Provider<PersonIdent> serverIdent;
- private final AllUsersName allUsers;
- private final SchemaFactory<ReviewDb> schemaFactory;
- private final GitRepositoryManager repoManager;
- private final NoteDbUpdateManager.Factory updateManagerFactory;
- private final ChangeBundleReader bundleReader;
- private final AllProjectsName allProjects;
- private final InternalUser.Factory userFactory;
- private final ThreadLocalRequestContext requestContext;
- private final ChangeRebuilderImpl rebuilder;
- private final WorkQueue workQueue;
- private final MutableNotesMigration globalNotesMigration;
- private final PrimaryStorageMigrator primaryStorageMigrator;
- private final PluginSetContext<NotesMigrationStateListener> listeners;
- private final NoteDbSchemaVersionManager versionManager;
-
- private int threads;
- private ImmutableList<Project.NameKey> projects = ImmutableList.of();
- private ImmutableList<Change.Id> changes = ImmutableList.of();
- private OutputStream progressOut = NullOutputStream.INSTANCE;
- private NotesMigrationState stopAtState;
- private boolean trial;
- private boolean forceRebuild;
- private int sequenceGap = -1;
- private boolean autoMigrate;
-
- @Inject
- Builder(
- GerritServerConfigProvider configProvider,
- SitePaths sitePaths,
- @GerritPersonIdent Provider<PersonIdent> serverIdent,
- AllUsersName allUsers,
- SchemaFactory<ReviewDb> schemaFactory,
- GitRepositoryManager repoManager,
- NoteDbUpdateManager.Factory updateManagerFactory,
- ChangeBundleReader bundleReader,
- AllProjectsName allProjects,
- ThreadLocalRequestContext requestContext,
- InternalUser.Factory userFactory,
- ChangeRebuilderImpl rebuilder,
- WorkQueue workQueue,
- MutableNotesMigration globalNotesMigration,
- PrimaryStorageMigrator primaryStorageMigrator,
- PluginSetContext<NotesMigrationStateListener> listeners,
- NoteDbSchemaVersionManager versionManager) {
- // Reload gerrit.config/notedb.config on each migrator invocation, in case a previous
- // migration in the same process modified the on-disk contents. This ensures the defaults for
- // trial/autoMigrate get set correctly below.
- this.cfg = configProvider.loadConfig();
- this.sitePaths = sitePaths;
- this.serverIdent = serverIdent;
- this.allUsers = allUsers;
- this.schemaFactory = schemaFactory;
- this.repoManager = repoManager;
- this.updateManagerFactory = updateManagerFactory;
- this.bundleReader = bundleReader;
- this.allProjects = allProjects;
- this.requestContext = requestContext;
- this.userFactory = userFactory;
- this.rebuilder = rebuilder;
- this.workQueue = workQueue;
- this.globalNotesMigration = globalNotesMigration;
- this.primaryStorageMigrator = primaryStorageMigrator;
- this.listeners = listeners;
- this.versionManager = versionManager;
- this.trial = getTrialMode(cfg);
- this.autoMigrate = getAutoMigrate(cfg);
- }
-
- /**
- * Set the number of threads used by parallelizable phases of the migration, such as rebuilding
- * all changes.
- *
- * <p>Not all phases are parallelizable, and calling {@link #rebuild()} directly will do
- * substantial work in the calling thread regardless of the number of threads configured.
- *
- * <p>By default, all work is done in the calling thread.
- *
- * @param threads thread count; if less than 2, all work happens in the calling thread.
- * @return this.
- */
- public Builder setThreads(int threads) {
- this.threads = threads;
- return this;
- }
-
- /**
- * Limit the set of projects that are processed.
- *
- * <p>Incompatible with {@link #setChanges(Collection)}.
- *
- * <p>By default, all projects will be processed.
- *
- * @param projects set of projects; if null or empty, all projects will be processed.
- * @return this.
- */
- public Builder setProjects(@Nullable Collection<Project.NameKey> projects) {
- this.projects = projects != null ? ImmutableList.copyOf(projects) : ImmutableList.of();
- return this;
- }
-
- /**
- * Limit the set of changes that are processed.
- *
- * <p>Incompatible with {@link #setProjects(Collection)}.
- *
- * <p>By default, all changes will be processed.
- *
- * @param changes set of changes; if null or empty, all changes will be processed.
- * @return this.
- */
- public Builder setChanges(@Nullable Collection<Change.Id> changes) {
- this.changes = changes != null ? ImmutableList.copyOf(changes) : ImmutableList.of();
- return this;
- }
-
- /**
- * Set output stream for progress monitors.
- *
- * <p>By default, there is no progress monitor output (although there may be other logs).
- *
- * @param progressOut output stream.
- * @return this.
- */
- public Builder setProgressOut(OutputStream progressOut) {
- this.progressOut = requireNonNull(progressOut);
- return this;
- }
-
- /**
- * Stop at a specific migration state, for testing only.
- *
- * @param stopAtState state to stop at.
- * @return this.
- */
- @VisibleForTesting
- public Builder setStopAtStateForTesting(NotesMigrationState stopAtState) {
- this.stopAtState = stopAtState;
- return this;
- }
-
- /**
- * Rebuild in "trial mode": configure Gerrit to write to and read from NoteDb, but leave
- * ReviewDb as the source of truth for all changes.
- *
- * <p>By default, trial mode is off, and NoteDb is the source of truth for all changes following
- * the migration.
- *
- * @param trial whether to rebuild in trial mode.
- * @return this.
- */
- public Builder setTrialMode(boolean trial) {
- this.trial = trial;
- return this;
- }
-
- /**
- * Rebuild all changes in NoteDb from ReviewDb, even if Gerrit is currently configured to read
- * from NoteDb.
- *
- * <p>Only supported if ReviewDb is still the source of truth for all changes.
- *
- * <p>By default, force rebuilding is off.
- *
- * @param forceRebuild whether to force rebuilding.
- * @return this.
- */
- public Builder setForceRebuild(boolean forceRebuild) {
- this.forceRebuild = forceRebuild;
- return this;
- }
-
- /**
- * Gap between ReviewDb change sequence numbers and NoteDb.
- *
- * <p>If NoteDb sequences are enabled in a running server, there is a race between the migration
- * step that calls {@code nextChangeId()} to seed the ref, and other threads that call {@code
- * nextChangeId()} to create new changes. In order to prevent these operations stepping on one
- * another, we use this value to skip some predefined sequence numbers. This is strongly
- * recommended in a running server.
- *
- * <p>If the migration takes place offline, there is no race with other threads, and this option
- * may be set to 0. However, admins may still choose to use a gap, for example to make it easier
- * to distinguish changes that were created before and after the NoteDb migration.
- *
- * <p>By default, uses the value from {@code noteDb.changes.initialSequenceGap} in {@code
- * gerrit.config}, which defaults to 1000.
- *
- * @param sequenceGap sequence gap size; if negative, use the default.
- * @return this.
- */
- public Builder setSequenceGap(int sequenceGap) {
- this.sequenceGap = sequenceGap;
- return this;
- }
-
- /**
- * Enable auto-migration on subsequent daemon launches.
- *
- * <p>If true, prior to running any migration steps, sets the necessary configuration in {@code
- * gerrit.config} to make {@code gerrit.war daemon} retry the migration on next startup, if it
- * fails.
- *
- * @param autoMigrate whether to set auto-migration config.
- * @return this.
- */
- public Builder setAutoMigrate(boolean autoMigrate) {
- this.autoMigrate = autoMigrate;
- return this;
- }
-
- public NoteDbMigrator build() throws MigrationException {
- return new NoteDbMigrator(
- sitePaths,
- schemaFactory,
- serverIdent,
- allUsers,
- repoManager,
- updateManagerFactory,
- bundleReader,
- allProjects,
- requestContext,
- userFactory,
- rebuilder,
- globalNotesMigration,
- primaryStorageMigrator,
- listeners,
- versionManager,
- threads > 1
- ? MoreExecutors.listeningDecorator(
- workQueue.createQueue(threads, "RebuildChange", true))
- : MoreExecutors.newDirectExecutorService(),
- projects,
- changes,
- progressOut,
- stopAtState,
- trial,
- forceRebuild,
- sequenceGap >= 0 ? sequenceGap : Sequences.getChangeSequenceGap(cfg),
- autoMigrate);
- }
- }
-
- private final FileBasedConfig gerritConfig;
- private final FileBasedConfig noteDbConfig;
- private final SchemaFactory<ReviewDb> schemaFactory;
- private final Provider<PersonIdent> serverIdent;
- private final AllUsersName allUsers;
- private final GitRepositoryManager repoManager;
- private final NoteDbUpdateManager.Factory updateManagerFactory;
- private final ChangeBundleReader bundleReader;
- private final AllProjectsName allProjects;
- private final ThreadLocalRequestContext requestContext;
- private final InternalUser.Factory userFactory;
- private final ChangeRebuilderImpl rebuilder;
- private final MutableNotesMigration globalNotesMigration;
- private final PrimaryStorageMigrator primaryStorageMigrator;
- private final PluginSetContext<NotesMigrationStateListener> listeners;
- private final NoteDbSchemaVersionManager versionManager;
-
- private final ListeningExecutorService executor;
- private final ImmutableList<Project.NameKey> projects;
- private final ImmutableList<Change.Id> changes;
- private final OutputStream progressOut;
- private final NotesMigrationState stopAtState;
- private final boolean trial;
- private final boolean forceRebuild;
- private final int sequenceGap;
- private final boolean autoMigrate;
-
- private NoteDbMigrator(
- SitePaths sitePaths,
- SchemaFactory<ReviewDb> schemaFactory,
- Provider<PersonIdent> serverIdent,
- AllUsersName allUsers,
- GitRepositoryManager repoManager,
- NoteDbUpdateManager.Factory updateManagerFactory,
- ChangeBundleReader bundleReader,
- AllProjectsName allProjects,
- ThreadLocalRequestContext requestContext,
- InternalUser.Factory userFactory,
- ChangeRebuilderImpl rebuilder,
- MutableNotesMigration globalNotesMigration,
- PrimaryStorageMigrator primaryStorageMigrator,
- PluginSetContext<NotesMigrationStateListener> listeners,
- NoteDbSchemaVersionManager versionManager,
- ListeningExecutorService executor,
- ImmutableList<Project.NameKey> projects,
- ImmutableList<Change.Id> changes,
- OutputStream progressOut,
- NotesMigrationState stopAtState,
- boolean trial,
- boolean forceRebuild,
- int sequenceGap,
- boolean autoMigrate)
- throws MigrationException {
- if (!changes.isEmpty() && !projects.isEmpty()) {
- throw new MigrationException("Cannot set both changes and projects");
- }
- if (sequenceGap < 0) {
- throw new MigrationException("Sequence gap must be non-negative: " + sequenceGap);
- }
-
- this.schemaFactory = schemaFactory;
- this.serverIdent = serverIdent;
- this.allUsers = allUsers;
- this.rebuilder = rebuilder;
- this.repoManager = repoManager;
- this.updateManagerFactory = updateManagerFactory;
- this.bundleReader = bundleReader;
- this.allProjects = allProjects;
- this.requestContext = requestContext;
- this.userFactory = userFactory;
- this.globalNotesMigration = globalNotesMigration;
- this.primaryStorageMigrator = primaryStorageMigrator;
- this.listeners = listeners;
- this.versionManager = versionManager;
- this.executor = executor;
- this.projects = projects;
- this.changes = changes;
- this.progressOut = progressOut;
- this.stopAtState = stopAtState;
- this.trial = trial;
- this.forceRebuild = forceRebuild;
- this.sequenceGap = sequenceGap;
- this.autoMigrate = autoMigrate;
-
- // Stack notedb.config over gerrit.config, in the same way as GerritServerConfigProvider.
- this.gerritConfig = new FileBasedConfig(sitePaths.gerrit_config.toFile(), FS.detect());
- this.noteDbConfig =
- new FileBasedConfig(gerritConfig, sitePaths.notedb_config.toFile(), FS.detect());
- }
-
- @Override
- public void close() {
- executor.shutdownNow();
- }
-
- public void migrate() throws OrmException, IOException {
- if (!changes.isEmpty() || !projects.isEmpty()) {
- throw new MigrationException(
- "Cannot set changes or projects during full migration; call rebuild() instead");
- }
- Optional<NotesMigrationState> maybeState = loadState();
- if (!maybeState.isPresent()) {
- throw new MigrationException("Could not determine initial migration state");
- }
-
- NotesMigrationState state = maybeState.get();
- if (trial && state.compareTo(READ_WRITE_NO_SEQUENCE) > 0) {
- throw new MigrationException(
- "Migration has already progressed past the endpoint of the \"trial mode\" state;"
- + " NoteDb is already the primary storage for some changes");
- }
- if (forceRebuild && state.compareTo(READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY) > 0) {
- throw new MigrationException(
- "Cannot force rebuild changes; NoteDb is already the primary storage for some changes");
- }
- setControlFlags();
-
- boolean rebuilt = false;
- while (state.compareTo(NOTE_DB) < 0) {
- if (state.equals(stopAtState)) {
- return;
- }
- boolean stillNeedsRebuild = forceRebuild && !rebuilt;
- if (trial && state.compareTo(READ_WRITE_NO_SEQUENCE) >= 0) {
- if (stillNeedsRebuild && state == READ_WRITE_NO_SEQUENCE) {
- // We're at the end state of trial mode, but still need a rebuild due to forceRebuild. Let
- // the loop go one more time.
- } else {
- return;
- }
- }
- switch (state) {
- case REVIEW_DB:
- state = turnOnWrites(state);
- break;
- case WRITE:
- state = rebuildAndEnableReads(state);
- rebuilt = true;
- break;
- case READ_WRITE_NO_SEQUENCE:
- if (stillNeedsRebuild) {
- state = rebuildAndEnableReads(state);
- rebuilt = true;
- } else {
- state = enableSequences(state);
- }
- break;
- case READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY:
- if (stillNeedsRebuild) {
- state = rebuildAndEnableReads(state);
- rebuilt = true;
- } else {
- state = setNoteDbPrimary(state);
- }
- break;
- case READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY:
- // The only way we can get here is if there was a failure on a previous run of
- // setNoteDbPrimary, since that method moves to NOTE_DB if it completes
- // successfully. Assume that not all changes were converted and re-run the step.
- // migrateToNoteDbPrimary is a relatively fast no-op for already-migrated changes, so this
- // isn't actually repeating work.
- state = setNoteDbPrimary(state);
- break;
- case NOTE_DB:
- // Done!
- break;
- default:
- throw new MigrationException(
- "Migration out of the following state is not supported:\n" + state.toText());
- }
- }
- }
-
- private NotesMigrationState turnOnWrites(NotesMigrationState prev)
- throws OrmException, IOException {
- versionManager.init();
- return saveState(prev, WRITE);
- }
-
- private NotesMigrationState rebuildAndEnableReads(NotesMigrationState prev)
- throws OrmException, IOException {
- rebuild();
- return saveState(prev, READ_WRITE_NO_SEQUENCE);
- }
-
- private NotesMigrationState enableSequences(NotesMigrationState prev)
- throws OrmException, IOException {
- try (ReviewDb db = schemaFactory.open()) {
- @SuppressWarnings("deprecation")
- final int nextChangeId = db.nextChangeId();
-
- RepoSequence seq =
- new RepoSequence(
- repoManager,
- GitReferenceUpdated.DISABLED,
- allProjects,
- Sequences.NAME_CHANGES,
- // If sequenceGap is 0, this writes into the sequence ref the same ID that is returned
- // by the call to seq.next() below. If we actually used this as a change ID, that
- // would be a problem, but we just discard it, so this is safe.
- () -> nextChangeId + sequenceGap - 1,
- 1,
- nextChangeId);
- seq.next();
- }
- return saveState(prev, READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY);
- }
-
- private NotesMigrationState setNoteDbPrimary(NotesMigrationState prev)
- throws MigrationException, OrmException, IOException {
- checkState(
- projects.isEmpty() && changes.isEmpty(),
- "Should not have attempted setNoteDbPrimary with a subset of changes");
- checkState(
- prev == READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY
- || prev == READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY,
- "Unexpected start state for setNoteDbPrimary: %s",
- prev);
-
- // Before changing the primary storage of old changes, ensure new changes are created with
- // NoteDb primary.
- prev = saveState(prev, READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY);
-
- Stopwatch sw = Stopwatch.createStarted();
- logger.atInfo().log("Setting primary storage to NoteDb");
- List<Change.Id> allChanges;
- try (ReviewDb db = unwrapDb(schemaFactory.open())) {
- allChanges = Streams.stream(db.changes().all()).map(Change::getId).collect(toList());
- }
-
- try (ContextHelper contextHelper = new ContextHelper()) {
- List<ListenableFuture<Boolean>> futures =
- allChanges
- .stream()
- .map(
- id ->
- executor.submit(
- () -> {
- try (ManualRequestContext ctx = contextHelper.open()) {
- try {
- primaryStorageMigrator.migrateToNoteDbPrimary(id);
- } catch (NoNoteDbStateException e) {
- if (canSkipPrimaryStorageMigration(
- ctx.getReviewDbProvider().get(), id)) {
- logger.atWarning().withCause(e).log(
- "Change %s previously failed to rebuild;"
- + " skipping primary storage migration",
- id);
- } else {
- throw e;
- }
- }
- return true;
- } catch (Exception e) {
- logger.atSevere().withCause(e).log(
- "Error migrating primary storage for %s", id);
- return false;
- }
- }))
- .collect(toList());
-
- boolean ok = futuresToBoolean(futures, "Error migrating primary storage");
- double t = sw.elapsed(TimeUnit.MILLISECONDS) / 1000d;
- logger.atInfo().log(
- "Migrated primary storage of %d changes in %.01fs (%.01f/s)\n",
- allChanges.size(), t, allChanges.size() / t);
- if (!ok) {
- throw new MigrationException("Migrating primary storage for some changes failed, see log");
- }
- }
-
- return disableReviewDb(prev);
- }
-
- /**
- * Checks whether a change is so corrupt that it can be completely skipped by the primary storage
- * migration step.
- *
- * <p>To get to the point where this method is called from {@link #setNoteDbPrimary}, it means we
- * attempted to rebuild it, and encountered an error that was then caught in {@link
- * #rebuildProject} and skipped. As a result, there is no {@code noteDbState} field in the change
- * by the time we get to {@link #setNoteDbPrimary}, so {@code migrateToNoteDbPrimary} throws an
- * exception.
- *
- * <p>We have to do this hacky double-checking because we don't have a way for the rebuilding
- * phase to communicate to the primary storage migration phase that the change is skippable. It
- * would be possible to store this info in some field in this class, but there is no guarantee
- * that the rebuild and primary storage migration phases are run in the same JVM invocation.
- *
- * <p>In an ideal world, we could do this through the {@link
- * com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage} enum, having a separate value
- * for errors. However, that would be an invasive change touching many non-migration-related parts
- * of the NoteDb migration code, which is too risky to attempt in the stable branch where this bug
- * had to be fixed.
- *
- * <p>As of this writing, the only case where this happens is when a change has no patch sets.
- */
- private static boolean canSkipPrimaryStorageMigration(ReviewDb db, Change.Id id) {
- try {
- return Iterables.isEmpty(unwrapDb(db).patchSets().byChange(id));
- } catch (Exception e) {
- logger.atSevere().withCause(e).log(
- "Error checking if change %s can be skipped, assuming no", id);
- return false;
- }
- }
-
- private NotesMigrationState disableReviewDb(NotesMigrationState prev) throws IOException {
- return saveState(prev, NOTE_DB, c -> setAutoMigrate(c, false));
- }
-
- private Optional<NotesMigrationState> loadState() throws IOException {
- try {
- gerritConfig.load();
- noteDbConfig.load();
- return NotesMigrationState.forConfig(noteDbConfig);
- } catch (ConfigInvalidException | IllegalArgumentException e) {
- logger.atWarning().withCause(e).log(
- "error reading NoteDb migration options from %s", noteDbConfig.getFile());
- return Optional.empty();
- }
- }
-
- private NotesMigrationState saveState(
- NotesMigrationState expectedOldState, NotesMigrationState newState) throws IOException {
- return saveState(expectedOldState, newState, c -> {});
- }
-
- private NotesMigrationState saveState(
- NotesMigrationState expectedOldState,
- NotesMigrationState newState,
- Consumer<Config> additionalUpdates)
- throws IOException {
- synchronized (globalNotesMigration) {
- // This read-modify-write is racy. We're counting on the fact that no other Gerrit operation
- // modifies gerrit.config, and hoping that admins don't either.
- Optional<NotesMigrationState> actualOldState = loadState();
- if (!actualOldState.equals(Optional.of(expectedOldState))) {
- throw new MigrationException(
- "Cannot move to new state:\n"
- + newState.toText()
- + "\n\n"
- + "Expected this state in gerrit.config:\n"
- + expectedOldState.toText()
- + "\n\n"
- + (actualOldState.isPresent()
- ? "But found this state:\n" + actualOldState.get().toText()
- : "But could not parse the current state"));
- }
-
- preStateChange(expectedOldState, newState);
-
- newState.setConfigValues(noteDbConfig);
- additionalUpdates.accept(noteDbConfig);
- noteDbConfig.save();
-
- // Only set in-memory state once it's been persisted to storage.
- globalNotesMigration.setFrom(newState);
- logger.atInfo().log("Migration state: %s => %s", expectedOldState, newState);
-
- return newState;
- }
- }
-
- private void preStateChange(NotesMigrationState oldState, NotesMigrationState newState)
- throws IOException {
- listeners.runEach(l -> l.preStateChange(oldState, newState), IOException.class);
- }
-
- private void setControlFlags() throws MigrationException {
- synchronized (globalNotesMigration) {
- try {
- noteDbConfig.load();
- setAutoMigrate(noteDbConfig, autoMigrate);
- setTrialMode(noteDbConfig, trial);
- noteDbConfig.save();
- } catch (ConfigInvalidException | IOException e) {
- throw new MigrationException("Error saving auto-migration config", e);
- }
- }
- }
-
- public void rebuild() throws MigrationException, OrmException {
- if (!globalNotesMigration.commitChangeWrites()) {
- throw new MigrationException("Cannot rebuild without noteDb.changes.write=true");
- }
- Stopwatch sw = Stopwatch.createStarted();
- logger.atInfo().log("Rebuilding changes in NoteDb");
-
- ImmutableListMultimap<Project.NameKey, Change.Id> changesByProject = getChangesByProject();
- List<ListenableFuture<Boolean>> futures = new ArrayList<>();
- try (ContextHelper contextHelper = new ContextHelper()) {
- List<Project.NameKey> projectNames =
- Ordering.usingToString().sortedCopy(changesByProject.keySet());
- for (Project.NameKey project : projectNames) {
- ListenableFuture<Boolean> future =
- executor.submit(
- () -> {
- try {
- return rebuildProject(contextHelper.getReviewDb(), changesByProject, project);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("Error rebuilding project %s", project);
- return false;
- }
- });
- futures.add(future);
- }
-
- boolean ok = futuresToBoolean(futures, "Error rebuilding projects");
- double t = sw.elapsed(TimeUnit.MILLISECONDS) / 1000d;
- logger.atInfo().log(
- "Rebuilt %d changes in %.01fs (%.01f/s)\n",
- changesByProject.size(), t, changesByProject.size() / t);
- if (!ok) {
- throw new MigrationException("Rebuilding some changes failed, see log");
- }
- }
- }
-
- private ImmutableListMultimap<Project.NameKey, Change.Id> getChangesByProject()
- throws OrmException {
- // Memoize all changes so we can close the db connection and allow other threads to use the full
- // connection pool.
- SetMultimap<Project.NameKey, Change.Id> out =
- MultimapBuilder.treeKeys(comparing(Project.NameKey::get))
- .treeSetValues(comparing(Change.Id::get))
- .build();
- try (ReviewDb db = unwrapDb(schemaFactory.open())) {
- if (!projects.isEmpty()) {
- return byProject(db.changes().all(), c -> projects.contains(c.getProject()), out);
- }
- if (!changes.isEmpty()) {
- return byProject(db.changes().get(changes), c -> true, out);
- }
- return byProject(db.changes().all(), c -> true, out);
- }
- }
-
- private static ImmutableListMultimap<Project.NameKey, Change.Id> byProject(
- Iterable<Change> changes,
- Predicate<Change> pred,
- SetMultimap<Project.NameKey, Change.Id> out) {
- Streams.stream(changes).filter(pred).forEach(c -> out.put(c.getProject(), c.getId()));
- return ImmutableListMultimap.copyOf(out);
- }
-
- private static ObjectInserter newPackInserter(Repository repo) {
- if (!(repo instanceof FileRepository)) {
- return repo.newObjectInserter();
- }
- PackInserter ins = ((FileRepository) repo).getObjectDatabase().newPackInserter();
- ins.checkExisting(false);
- return ins;
- }
-
- private boolean rebuildProject(
- ReviewDb db,
- ImmutableListMultimap<Project.NameKey, Change.Id> allChanges,
- Project.NameKey project) {
- checkArgument(allChanges.containsKey(project));
- boolean ok = true;
- ProgressMonitor pm =
- new TextProgressMonitor(
- new PrintWriter(new BufferedWriter(new OutputStreamWriter(progressOut, UTF_8))));
- try (Repository changeRepo = repoManager.openRepository(project);
- // Only use a PackInserter for the change repo, not All-Users.
- //
- // It's not possible to share a single inserter for All-Users across all project tasks, and
- // we don't want to add one pack per project to All-Users. Adding many loose objects is
- // preferable to many packs.
- //
- // Anyway, the number of objects inserted into All-Users is proportional to the number
- // of pending draft comments, which should not be high (relative to the total number of
- // changes), so the number of loose objects shouldn't be too unreasonable.
- ObjectInserter changeIns = newPackInserter(changeRepo);
- ObjectReader changeReader = changeIns.newReader();
- RevWalk changeRw = new RevWalk(changeReader);
- Repository allUsersRepo = repoManager.openRepository(allUsers);
- ObjectInserter allUsersIns = allUsersRepo.newObjectInserter();
- ObjectReader allUsersReader = allUsersIns.newReader();
- RevWalk allUsersRw = new RevWalk(allUsersReader)) {
- ChainedReceiveCommands changeCmds = new ChainedReceiveCommands(changeRepo);
- ChainedReceiveCommands allUsersCmds = new ChainedReceiveCommands(allUsersRepo);
-
- Collection<Change.Id> changes = allChanges.get(project);
- pm.beginTask(FormatUtil.elide("Rebuilding " + project.get(), 50), changes.size());
- int toSave = 0;
- try {
- for (Change.Id changeId : changes) {
- // NoteDbUpdateManager assumes that all commands in its OpenRepo were added by itself, so
- // we can't share the top-level ChainedReceiveCommands. Use a new set of commands sharing
- // the same underlying repo, and copy commands back to the top-level
- // ChainedReceiveCommands later. This also assumes that each ref in the final list of
- // commands was only modified by a single NoteDbUpdateManager; since we use one manager
- // per change, and each ref corresponds to exactly one change, this assumption should be
- // safe.
- ChainedReceiveCommands tmpChangeCmds =
- new ChainedReceiveCommands(changeCmds.getRepoRefCache());
- ChainedReceiveCommands tmpAllUsersCmds =
- new ChainedReceiveCommands(allUsersCmds.getRepoRefCache());
-
- try (NoteDbUpdateManager manager =
- updateManagerFactory
- .create(project)
- .setAtomicRefUpdates(false)
- .setSaveObjects(false)
- .setChangeRepo(changeRepo, changeRw, changeIns, tmpChangeCmds)
- .setAllUsersRepo(allUsersRepo, allUsersRw, allUsersIns, tmpAllUsersCmds)) {
- rebuild(db, changeId, manager);
-
- // Executing with dryRun=true writes all objects to the underlying inserters and adds
- // commands to the ChainedReceiveCommands. Afterwards, we can discard the manager, so we
- // don't keep using any memory beyond what may be buffered in the PackInserter.
- manager.execute(true);
-
- tmpChangeCmds.getCommands().values().forEach(c -> addCommand(changeCmds, c));
- tmpAllUsersCmds.getCommands().values().forEach(c -> addCommand(allUsersCmds, c));
-
- toSave++;
- } catch (NoPatchSetsException e) {
- logger.atWarning().log(e.getMessage());
- } catch (ConflictingUpdateException ex) {
- logger.atWarning().log(
- "Rebuilding detected a conflicting ReviewDb update for change %s;"
- + " will be auto-rebuilt at runtime",
- changeId);
- } catch (Throwable t) {
- logger.atSevere().withCause(t).log("Failed to rebuild change %s", changeId);
- ok = false;
- }
- pm.update(1);
- }
- } finally {
- pm.endTask();
- }
-
- pm.beginTask(FormatUtil.elide("Saving " + project.get(), 50), ProgressMonitor.UNKNOWN);
- try {
- save(changeRepo, changeRw, changeIns, changeCmds);
- save(allUsersRepo, allUsersRw, allUsersIns, allUsersCmds);
- // This isn't really useful progress. If we passed a real ProgressMonitor to
- // BatchRefUpdate#execute we might get something more incremental, but that doesn't allow us
- // to specify the repo name in the task text.
- pm.update(toSave);
- } catch (LockFailureException e) {
- logger.atWarning().log(
- "Rebuilding detected a conflicting NoteDb update for the following refs, which will"
- + " be auto-rebuilt at runtime: %s",
- e.getFailedRefs().stream().distinct().sorted().collect(joining(", ")));
- } catch (IOException e) {
- logger.atSevere().withCause(e).log("Failed to save NoteDb state for %s", project);
- } finally {
- pm.endTask();
- }
- } catch (RepositoryNotFoundException e) {
- logger.atWarning().log("Repository %s not found", project);
- } catch (IOException e) {
- logger.atSevere().withCause(e).log("Failed to rebuild project %s", project);
- }
- return ok;
- }
-
- private void rebuild(ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager)
- throws OrmException, IOException {
- // Match ChangeRebuilderImpl#stage, but without calling manager.stage(), since that can only be
- // called after building updates for all changes.
- Change change =
- ChangeRebuilderImpl.checkNoteDbState(ChangeNotes.readOneReviewDbChange(db, changeId));
- if (change == null) {
- // Could log here instead, but this matches the behavior of ChangeRebuilderImpl#stage.
- throw new NoSuchChangeException(changeId);
- }
- rebuilder.buildUpdates(manager, bundleReader.fromReviewDb(db, changeId));
-
- rebuilder.execute(db, changeId, manager, true, false);
- }
-
- private static void addCommand(ChainedReceiveCommands cmds, ReceiveCommand cmd) {
- // ChainedReceiveCommands doesn't allow no-ops, but these occur when rebuilding a
- // previously-rebuilt change.
- if (!cmd.getOldId().equals(cmd.getNewId())) {
- cmds.add(cmd);
- }
- }
-
- private void save(Repository repo, RevWalk rw, ObjectInserter ins, ChainedReceiveCommands cmds)
- throws IOException {
- if (cmds.isEmpty()) {
- return;
- }
- ins.flush();
- BatchRefUpdate bru = repo.getRefDatabase().newBatchUpdate();
- bru.setRefLogMessage("Migrate changes to NoteDb", false);
- bru.setRefLogIdent(serverIdent.get());
- bru.setAtomic(false);
- bru.setAllowNonFastForwards(true);
- cmds.addTo(bru);
- RefUpdateUtil.executeChecked(bru, rw);
- }
-
- private static boolean futuresToBoolean(List<ListenableFuture<Boolean>> futures, String errMsg) {
- try {
- return Futures.allAsList(futures).get().stream().allMatch(b -> b);
- } catch (InterruptedException | ExecutionException e) {
- logger.atSevere().withCause(e).log(errMsg);
- return false;
- }
- }
-
- private class ContextHelper implements AutoCloseable {
- private final Thread callingThread;
- private ReviewDb db;
- private Runnable closeDb;
-
- ContextHelper() {
- callingThread = Thread.currentThread();
- }
-
- ManualRequestContext open() throws OrmException {
- return new ManualRequestContext(
- userFactory.create(),
- // Reuse the same lazily-opened ReviewDb on the original calling thread, otherwise open
- // SchemaFactory in the normal way.
- Thread.currentThread().equals(callingThread) ? this::getReviewDb : schemaFactory,
- requestContext);
- }
-
- synchronized ReviewDb getReviewDb() throws OrmException {
- if (db == null) {
- ReviewDb actual = schemaFactory.open();
- closeDb = actual::close;
- db =
- new ReviewDbWrapper(unwrapDb(actual)) {
- @Override
- public void close() {
- // Closed by ContextHelper#close.
- }
- };
- }
- return db;
- }
-
- @Override
- public synchronized void close() {
- if (db != null) {
- closeDb.run();
- db = null;
- closeDb = null;
- }
- }
- }
-}
diff --git a/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java b/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java
deleted file mode 100644
index b5a8236..0000000
--- a/java/com/google/gerrit/server/notedb/rebuild/OnlineNoteDbMigrator.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright (C) 2017 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.server.notedb.rebuild;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.lifecycle.LifecycleModule;
-import com.google.gerrit.server.config.GerritServerConfig;
-import com.google.gerrit.server.index.OnlineUpgrader;
-import com.google.gerrit.server.index.VersionManager;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-import com.google.inject.name.Named;
-import com.google.inject.name.Names;
-import java.util.concurrent.TimeUnit;
-import org.eclipse.jgit.lib.Config;
-
-@Singleton
-public class OnlineNoteDbMigrator implements LifecycleListener {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private static final String TRIAL = "OnlineNoteDbMigrator/trial";
-
- public static class Module extends LifecycleModule {
- private final boolean trial;
-
- public Module(boolean trial) {
- this.trial = trial;
- }
-
- @Override
- public void configure() {
- listener().to(OnlineNoteDbMigrator.class);
- bindConstant().annotatedWith(Names.named(TRIAL)).to(trial);
- }
- }
-
- private final GcAllUsers gcAllUsers;
- private final OnlineUpgrader indexUpgrader;
- private final Provider<NoteDbMigrator.Builder> migratorBuilderProvider;
- private final boolean upgradeIndex;
- private final boolean trial;
-
- @Inject
- OnlineNoteDbMigrator(
- @GerritServerConfig Config cfg,
- GcAllUsers gcAllUsers,
- OnlineUpgrader indexUpgrader,
- Provider<NoteDbMigrator.Builder> migratorBuilderProvider,
- @Named(TRIAL) boolean trial) {
- this.gcAllUsers = gcAllUsers;
- this.indexUpgrader = indexUpgrader;
- this.migratorBuilderProvider = migratorBuilderProvider;
- this.upgradeIndex = VersionManager.getOnlineUpgrade(cfg);
- this.trial = trial || NoteDbMigrator.getTrialMode(cfg);
- }
-
- @Override
- public void start() {
- Thread t = new Thread(this::migrate);
- t.setDaemon(true);
- t.setName(getClass().getSimpleName());
- t.start();
- }
-
- private void migrate() {
- logger.atInfo().log("Starting online NoteDb migration");
- if (upgradeIndex) {
- logger.atInfo().log(
- "Online index schema upgrades will be deferred until NoteDb migration is complete");
- }
- Stopwatch sw = Stopwatch.createStarted();
- // TODO(dborowitz): Tune threads, maybe expose a progress monitor somewhere.
- try (NoteDbMigrator migrator =
- migratorBuilderProvider.get().setAutoMigrate(true).setTrialMode(trial).build()) {
- migrator.migrate();
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("Error in online NoteDb migration");
- }
- gcAllUsers.runWithLogger();
- logger.atInfo().log("Online NoteDb migration completed in %ss", sw.elapsed(TimeUnit.SECONDS));
-
- if (upgradeIndex) {
- logger.atInfo().log("Starting deferred index schema upgrades");
- indexUpgrader.start();
- }
- }
-
- @Override
- public void stop() {
- // Do nothing; upgrade process uses daemon threads and knows how to recover from failures on
- // next attempt.
- }
-}
diff --git a/javatests/com/google/gerrit/acceptance/pgm/StandaloneNoteDbMigrationIT.java b/javatests/com/google/gerrit/acceptance/pgm/StandaloneNoteDbMigrationIT.java
deleted file mode 100644
index 1bb23fb..0000000
--- a/javatests/com/google/gerrit/acceptance/pgm/StandaloneNoteDbMigrationIT.java
+++ /dev/null
@@ -1,363 +0,0 @@
-// Copyright (C) 2014 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.acceptance.pgm;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth.assertWithMessage;
-import static com.google.common.truth.Truth8.assertThat;
-import static com.google.common.truth.TruthJUnit.assume;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.gerrit.acceptance.NoHttpd;
-import com.google.gerrit.acceptance.StandaloneSiteTest;
-import com.google.gerrit.extensions.api.GerritApi;
-import com.google.gerrit.extensions.common.ChangeInput;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
-import com.google.gerrit.server.config.AllUsersName;
-import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.git.LocalDiskRepositoryManager;
-import com.google.gerrit.server.index.GerritIndexStatus;
-import com.google.gerrit.server.index.change.ChangeIndexCollection;
-import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
-import com.google.gerrit.server.notedb.NoteDbChangeState;
-import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
-import com.google.gerrit.server.notedb.NoteDbChangeState.RefState;
-import com.google.gerrit.server.notedb.NotesMigrationState;
-import com.google.gerrit.server.schema.ReviewDbFactory;
-import com.google.gerrit.testing.NoteDbMode;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.stream.Stream;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.lib.StoredConfig;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for NoteDb migrations where the entry point is through a program, {@code
- * migrate-to-note-db} or {@code daemon}.
- *
- * <p><strong>Note:</strong> These tests are very slow due to the repeated daemon startup. Prefer
- * adding tests to {@link com.google.gerrit.acceptance.server.notedb.OnlineNoteDbMigrationIT} if
- * possible.
- */
-@NoHttpd
-public class StandaloneNoteDbMigrationIT extends StandaloneSiteTest {
- private StoredConfig gerritConfig;
- private StoredConfig noteDbConfig;
-
- private Project.NameKey project;
- private Change.Id changeId;
-
- @Before
- public void setUp() throws Exception {
- assume().that(NoteDbMode.get()).isEqualTo(NoteDbMode.OFF);
- gerritConfig = new FileBasedConfig(sitePaths.gerrit_config.toFile(), FS.detect());
- // Unlike in the running server, for tests, we don't stack notedb.config on gerrit.config.
- noteDbConfig = new FileBasedConfig(sitePaths.notedb_config.toFile(), FS.detect());
-
- // Set gc.pruneExpire=now so GC prunes all unreachable objects from All-Users, which allows us
- // to reliably test that it behaves as expected.
- Path cfgPath = sitePaths.site_path.resolve("git").resolve("All-Users.git").resolve("config");
- assertWithMessage("Expected All-Users config at %s", cfgPath)
- .that(Files.isRegularFile(cfgPath))
- .isTrue();
- FileBasedConfig cfg = new FileBasedConfig(cfgPath.toFile(), FS.detect());
- cfg.setString("gc", null, "pruneExpire", "now");
- cfg.save();
- }
-
- @Test
- public void rebuildOneChangeTrialMode() throws Exception {
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoAutoMigrateConfig(noteDbConfig);
- assertNotesMigrationState(NotesMigrationState.REVIEW_DB);
- setUpOneChange();
-
- migrate("--trial");
- assertNotesMigrationState(NotesMigrationState.READ_WRITE_NO_SEQUENCE);
-
- try (ServerContext ctx = startServer()) {
- GitRepositoryManager repoManager = ctx.getInjector().getInstance(GitRepositoryManager.class);
- ObjectId metaId;
- try (Repository repo = repoManager.openRepository(project)) {
- Ref ref = repo.exactRef(RefNames.changeMetaRef(changeId));
- assertThat(ref).isNotNull();
- metaId = ref.getObjectId();
- }
-
- try (ReviewDb db = openUnderlyingReviewDb(ctx)) {
- Change c = db.changes().get(changeId);
- assertThat(c).isNotNull();
- NoteDbChangeState state = NoteDbChangeState.parse(c);
- assertThat(state).isNotNull();
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.REVIEW_DB);
- assertThat(state.getRefState()).hasValue(RefState.create(metaId, ImmutableMap.of()));
- }
- }
- }
-
- @Test
- public void migrateOneChange() throws Exception {
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoAutoMigrateConfig(noteDbConfig);
- assertNotesMigrationState(NotesMigrationState.REVIEW_DB);
- setUpOneChange();
-
- migrate();
- assertNotesMigrationState(NotesMigrationState.NOTE_DB);
-
- File allUsersDir;
- try (ServerContext ctx = startServer()) {
- GitRepositoryManager repoManager = ctx.getInjector().getInstance(GitRepositoryManager.class);
- try (Repository repo = repoManager.openRepository(project)) {
- assertThat(repo.exactRef(RefNames.changeMetaRef(changeId))).isNotNull();
- }
- assertThat(repoManager).isInstanceOf(LocalDiskRepositoryManager.class);
- try (Repository repo =
- repoManager.openRepository(ctx.getInjector().getInstance(AllUsersName.class))) {
- allUsersDir = repo.getDirectory();
- }
-
- try (ReviewDb db = openUnderlyingReviewDb(ctx)) {
- Change c = db.changes().get(changeId);
- assertThat(c).isNotNull();
- NoteDbChangeState state = NoteDbChangeState.parse(c);
- assertThat(state).isNotNull();
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.NOTE_DB);
- assertThat(state.getRefState()).isEmpty();
-
- ChangeInput in = new ChangeInput(project.get(), "master", "NoteDb-only change");
- in.newBranch = true;
- GerritApi gApi = ctx.getInjector().getInstance(GerritApi.class);
- Change.Id id2 = new Change.Id(gApi.changes().create(in).info()._number);
- assertThat(db.changes().get(id2)).isNull();
- }
- }
- assertNoAutoMigrateConfig(gerritConfig);
- assertAutoMigrateConfig(noteDbConfig, false);
-
- try (FileRepository repo = new FileRepository(allUsersDir)) {
- try (Stream<Path> paths = Files.walk(repo.getObjectsDirectory().toPath())) {
- assertThat(paths.filter(p -> !p.toString().contains("pack") && Files.isRegularFile(p)))
- .named("loose object files in All-Users")
- .isEmpty();
- }
- assertThat(repo.getObjectDatabase().getPacks()).named("packfiles in All-Users").hasSize(1);
- }
- }
-
- @Test
- public void migrationWithReindex() throws Exception {
- assertNotesMigrationState(NotesMigrationState.REVIEW_DB);
- setUpOneChange();
-
- int version = ChangeSchemaDefinitions.INSTANCE.getLatest().getVersion();
- GerritIndexStatus status = new GerritIndexStatus(sitePaths);
- assertThat(status.getReady(ChangeSchemaDefinitions.NAME, version)).isTrue();
- status.setReady(ChangeSchemaDefinitions.NAME, version, false);
- status.save();
- assertServerStartupFails();
-
- migrate();
- assertNotesMigrationState(NotesMigrationState.NOTE_DB);
-
- status = new GerritIndexStatus(sitePaths);
- assertThat(status.getReady(ChangeSchemaDefinitions.NAME, version)).isTrue();
- }
-
- @Test
- public void onlineMigrationViaDaemon() throws Exception {
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoAutoMigrateConfig(noteDbConfig);
-
- testOnlineMigration(u -> startServer(u.module(), "--migrate-to-note-db", "true"));
-
- assertNoAutoMigrateConfig(gerritConfig);
- assertAutoMigrateConfig(noteDbConfig, false);
- }
-
- @Test
- public void onlineMigrationViaConfig() throws Exception {
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoAutoMigrateConfig(noteDbConfig);
-
- testOnlineMigration(
- u -> {
- gerritConfig.setBoolean("noteDb", "changes", "autoMigrate", true);
- gerritConfig.save();
- return startServer(u.module());
- });
-
- // Auto-migration is turned off in notedb.config, which takes precedence, but is still on in
- // gerrit.config. This means Puppet can continue overwriting gerrit.config without turning
- // auto-migration back on.
- assertAutoMigrateConfig(gerritConfig, true);
- assertAutoMigrateConfig(noteDbConfig, false);
- }
-
- @Test
- public void onlineMigrationTrialModeViaFlag() throws Exception {
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoTrialConfig(gerritConfig);
-
- assertNoAutoMigrateConfig(noteDbConfig);
- assertNoTrialConfig(noteDbConfig);
-
- testOnlineMigration(
- u -> startServer(u.module(), "--migrate-to-note-db", "--trial"),
- NotesMigrationState.READ_WRITE_NO_SEQUENCE);
-
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoTrialConfig(gerritConfig);
-
- assertAutoMigrateConfig(noteDbConfig, true);
- assertTrialConfig(noteDbConfig, true);
- }
-
- @Test
- public void onlineMigrationTrialModeViaConfig() throws Exception {
- assertNoAutoMigrateConfig(gerritConfig);
- assertNoTrialConfig(gerritConfig);
-
- assertNoAutoMigrateConfig(noteDbConfig);
- assertNoTrialConfig(noteDbConfig);
-
- testOnlineMigration(
- u -> {
- gerritConfig.setBoolean("noteDb", "changes", "autoMigrate", true);
- gerritConfig.setBoolean("noteDb", "changes", "trial", true);
- gerritConfig.save();
- return startServer(u.module());
- },
- NotesMigrationState.READ_WRITE_NO_SEQUENCE);
-
- assertAutoMigrateConfig(gerritConfig, true);
- assertTrialConfig(gerritConfig, true);
-
- assertAutoMigrateConfig(noteDbConfig, true);
- assertTrialConfig(noteDbConfig, true);
- }
-
- @FunctionalInterface
- private interface StartServerWithMigration {
- ServerContext start(IndexUpgradeController u) throws Exception;
- }
-
- private void testOnlineMigration(StartServerWithMigration start) throws Exception {
- testOnlineMigration(start, NotesMigrationState.NOTE_DB);
- }
-
- private void testOnlineMigration(
- StartServerWithMigration start, NotesMigrationState expectedEndState) throws Exception {
- assertNotesMigrationState(NotesMigrationState.REVIEW_DB);
- int prevVersion = ChangeSchemaDefinitions.INSTANCE.getPrevious().getVersion();
- int currVersion = ChangeSchemaDefinitions.INSTANCE.getLatest().getVersion();
-
- // Before storing any changes, switch back to the previous version.
- GerritIndexStatus status = new GerritIndexStatus(sitePaths);
- status.setReady(ChangeSchemaDefinitions.NAME, currVersion, false);
- status.setReady(ChangeSchemaDefinitions.NAME, prevVersion, true);
- status.save();
-
- setOnlineUpgradeConfig(false);
- setUpOneChange();
- setOnlineUpgradeConfig(true);
-
- IndexUpgradeController u = new IndexUpgradeController(1);
- try (ServerContext ctx = start.start(u)) {
- ChangeIndexCollection indexes = ctx.getInjector().getInstance(ChangeIndexCollection.class);
- assertThat(indexes.getSearchIndex().getSchema().getVersion()).isEqualTo(prevVersion);
-
- // Index schema upgrades happen after NoteDb migration, so waiting for those to complete
- // should be sufficient.
- u.runUpgrades();
-
- assertThat(indexes.getSearchIndex().getSchema().getVersion()).isEqualTo(currVersion);
- assertNotesMigrationState(expectedEndState);
- }
- }
-
- private void setUpOneChange() throws Exception {
- project = new Project.NameKey("project");
- try (ServerContext ctx = startServer()) {
- GerritApi gApi = ctx.getInjector().getInstance(GerritApi.class);
- gApi.projects().create("project");
-
- ChangeInput in = new ChangeInput(project.get(), "master", "Test change");
- in.newBranch = true;
- changeId = new Change.Id(gApi.changes().create(in).info()._number);
- }
- }
-
- private void migrate(String... additionalArgs) throws Exception {
- runGerrit(
- ImmutableList.of(
- "migrate-to-note-db", "-d", sitePaths.site_path.toString(), "--show-stack-trace"),
- ImmutableList.copyOf(additionalArgs));
- }
-
- private void assertNotesMigrationState(NotesMigrationState expected) throws Exception {
- noteDbConfig.load();
- assertThat(NotesMigrationState.forConfig(noteDbConfig)).hasValue(expected);
- }
-
- private ReviewDb openUnderlyingReviewDb(ServerContext ctx) throws Exception {
- return ctx.getInjector()
- .getInstance(Key.get(new TypeLiteral<SchemaFactory<ReviewDb>>() {}, ReviewDbFactory.class))
- .open();
- }
-
- private static void assertNoAutoMigrateConfig(StoredConfig cfg) throws Exception {
- cfg.load();
- assertThat(cfg.getString("noteDb", "changes", "autoMigrate")).isNull();
- }
-
- private static void assertAutoMigrateConfig(StoredConfig cfg, boolean expected) throws Exception {
- cfg.load();
- assertThat(cfg.getString("noteDb", "changes", "autoMigrate")).isNotNull();
- assertThat(cfg.getBoolean("noteDb", "changes", "autoMigrate", false)).isEqualTo(expected);
- }
-
- private static void assertNoTrialConfig(StoredConfig cfg) throws Exception {
- cfg.load();
- assertThat(cfg.getString("noteDb", "changes", "trial")).isNull();
- }
-
- private static void assertTrialConfig(StoredConfig cfg, boolean expected) throws Exception {
- cfg.load();
- assertThat(cfg.getString("noteDb", "changes", "trial")).isNotNull();
- assertThat(cfg.getBoolean("noteDb", "changes", "trial", false)).isEqualTo(expected);
- }
-
- private void setOnlineUpgradeConfig(boolean enable) throws Exception {
- gerritConfig.load();
- gerritConfig.setBoolean("index", null, "onlineUpgrade", enable);
- gerritConfig.save();
- }
-}
diff --git a/javatests/com/google/gerrit/acceptance/server/notedb/NoteDbPrimaryIT.java b/javatests/com/google/gerrit/acceptance/server/notedb/NoteDbPrimaryIT.java
deleted file mode 100644
index 26d5461..0000000
--- a/javatests/com/google/gerrit/acceptance/server/notedb/NoteDbPrimaryIT.java
+++ /dev/null
@@ -1,524 +0,0 @@
-// Copyright (C) 2016 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.acceptance.server.notedb;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
-import static com.google.common.truth.TruthJUnit.assume;
-import static com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage.REVIEW_DB;
-import static com.google.gerrit.server.notedb.NoteDbUtil.formatTime;
-import static java.util.concurrent.TimeUnit.DAYS;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static java.util.stream.Collectors.toList;
-
-import com.github.rholder.retry.Retryer;
-import com.github.rholder.retry.RetryerBuilder;
-import com.github.rholder.retry.StopStrategies;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.gerrit.acceptance.AbstractDaemonTest;
-import com.google.gerrit.acceptance.NoHttpd;
-import com.google.gerrit.acceptance.PushOneCommit;
-import com.google.gerrit.common.Nullable;
-import com.google.gerrit.extensions.api.changes.DraftInput;
-import com.google.gerrit.extensions.api.changes.ReviewInput;
-import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput;
-import com.google.gerrit.extensions.client.ChangeStatus;
-import com.google.gerrit.extensions.common.ApprovalInfo;
-import com.google.gerrit.extensions.common.ChangeInfo;
-import com.google.gerrit.extensions.common.CommentInfo;
-import com.google.gerrit.extensions.restapi.RestApiException;
-import com.google.gerrit.reviewdb.client.Account;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.reviewdb.client.ChangeMessage;
-import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDbUtil;
-import com.google.gerrit.server.ChangeMessagesUtil;
-import com.google.gerrit.server.CommentsUtil;
-import com.google.gerrit.server.InternalUser;
-import com.google.gerrit.server.git.RepoRefCache;
-import com.google.gerrit.server.notedb.ChangeBundle;
-import com.google.gerrit.server.notedb.ChangeBundleReader;
-import com.google.gerrit.server.notedb.ChangeNotes;
-import com.google.gerrit.server.notedb.ChangeUpdate;
-import com.google.gerrit.server.notedb.NoteDbChangeState;
-import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
-import com.google.gerrit.server.notedb.PrimaryStorageMigrator;
-import com.google.gerrit.server.notedb.TestChangeRebuilderWrapper;
-import com.google.gerrit.server.update.RetryHelper;
-import com.google.gerrit.server.util.time.TimeUtil;
-import com.google.gerrit.testing.ConfigSuite;
-import com.google.gerrit.testing.NoteDbMode;
-import com.google.gerrit.testing.TestTimeUtil;
-import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.OrmRuntimeException;
-import com.google.inject.Inject;
-import com.google.inject.util.Providers;
-import java.sql.Timestamp;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.revwalk.RevCommit;
-import org.eclipse.jgit.revwalk.RevWalk;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-@NoHttpd
-public class NoteDbPrimaryIT extends AbstractDaemonTest {
- @ConfigSuite.Default
- public static Config defaultConfig() {
- Config cfg = new Config();
- cfg.setString("notedb", null, "concurrentWriterTimeout", "0s");
- cfg.setString("notedb", null, "primaryStorageMigrationTimeout", "1d");
- cfg.setBoolean("noteDb", null, "testRebuilderWrapper", true);
- return cfg;
- }
-
- @Inject private ChangeBundleReader bundleReader;
- @Inject private CommentsUtil commentsUtil;
- @Inject private TestChangeRebuilderWrapper rebuilderWrapper;
- @Inject private ChangeNotes.Factory changeNotesFactory;
- @Inject private ChangeUpdate.Factory updateFactory;
- @Inject private InternalUser.Factory internalUserFactory;
- @Inject private RetryHelper retryHelper;
-
- private PrimaryStorageMigrator migrator;
-
- @Before
- public void setUp() throws Exception {
- assume().that(NoteDbMode.get()).isEqualTo(NoteDbMode.READ_WRITE);
- db = ReviewDbUtil.unwrapDb(db);
- TestTimeUtil.resetWithClockStep(1, SECONDS);
- migrator = newMigrator(null);
- }
-
- private PrimaryStorageMigrator newMigrator(
- @Nullable Retryer<NoteDbChangeState> ensureRebuiltRetryer) {
- return new PrimaryStorageMigrator(
- cfg,
- Providers.of(db),
- repoManager,
- allUsers,
- rebuilderWrapper,
- ensureRebuiltRetryer,
- changeNotesFactory,
- queryProvider,
- updateFactory,
- internalUserFactory,
- retryHelper);
- }
-
- @After
- public void tearDown() {
- TestTimeUtil.useSystemTime();
- }
-
- @Test
- public void updateChange() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- setNoteDbPrimary(id);
-
- gApi.changes().id(id.get()).current().review(ReviewInput.approve());
- gApi.changes().id(id.get()).current().submit();
-
- ChangeInfo info = gApi.changes().id(id.get()).get();
- assertThat(info.status).isEqualTo(ChangeStatus.MERGED);
- ApprovalInfo approval = Iterables.getOnlyElement(info.labels.get("Code-Review").all);
- assertThat(approval._accountId).isEqualTo(admin.id.get());
- assertThat(approval.value).isEqualTo(2);
- assertThat(info.messages).hasSize(3);
- assertThat(Iterables.getLast(info.messages).message)
- .isEqualTo("Change has been successfully merged by " + admin.fullName);
-
- ChangeNotes notes = notesFactory.create(db, project, id);
- assertThat(notes.getChange().getStatus()).isEqualTo(Change.Status.MERGED);
- assertThat(notes.getChange().getNoteDbState())
- .isEqualTo(NoteDbChangeState.NOTE_DB_PRIMARY_STATE);
-
- // Writes weren't reflected in ReviewDb.
- assertThat(db.changes().get(id).getStatus()).isEqualTo(Change.Status.NEW);
- assertThat(db.patchSetApprovals().byChange(id)).isEmpty();
- assertThat(db.changeMessages().byChange(id)).hasSize(1);
- }
-
- @Test
- public void deleteDraftComment() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- setNoteDbPrimary(id);
-
- DraftInput din = new DraftInput();
- din.path = PushOneCommit.FILE_NAME;
- din.line = 1;
- din.message = "A comment";
- gApi.changes().id(id.get()).current().createDraft(din);
-
- CommentInfo di =
- Iterables.getOnlyElement(
- gApi.changes().id(id.get()).current().drafts().get(PushOneCommit.FILE_NAME));
- assertThat(di.message).isEqualTo(din.message);
-
- assertThat(db.patchComments().draftByChangeFileAuthor(id, din.path, admin.id)).isEmpty();
-
- gApi.changes().id(id.get()).current().draft(di.id).delete();
- assertThat(gApi.changes().id(id.get()).current().drafts()).isEmpty();
- }
-
- @Test
- public void deleteVote() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- setNoteDbPrimary(id);
-
- gApi.changes().id(id.get()).current().review(ReviewInput.approve());
- List<ApprovalInfo> approvals = gApi.changes().id(id.get()).get().labels.get("Code-Review").all;
- assertThat(approvals).hasSize(1);
- assertThat(approvals.get(0).value).isEqualTo(2);
-
- gApi.changes().id(id.get()).reviewer(admin.id.toString()).deleteVote("Code-Review");
-
- approvals = gApi.changes().id(id.get()).get().labels.get("Code-Review").all;
- assertThat(approvals).hasSize(1);
- assertThat(approvals.get(0).value).isEqualTo(0);
- }
-
- @Test
- public void deleteVoteViaReview() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- setNoteDbPrimary(id);
-
- gApi.changes().id(id.get()).current().review(ReviewInput.approve());
- List<ApprovalInfo> approvals = gApi.changes().id(id.get()).get().labels.get("Code-Review").all;
- assertThat(approvals).hasSize(1);
- assertThat(approvals.get(0).value).isEqualTo(2);
-
- gApi.changes().id(id.get()).current().review(ReviewInput.noScore());
-
- approvals = gApi.changes().id(id.get()).get().labels.get("Code-Review").all;
- assertThat(approvals).hasSize(1);
- assertThat(approvals.get(0).value).isEqualTo(0);
- }
-
- @Test
- public void deleteReviewer() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- setNoteDbPrimary(id);
-
- gApi.changes().id(id.get()).addReviewer(user.id.toString());
- assertThat(getReviewers(id)).containsExactly(user.id);
- gApi.changes().id(id.get()).reviewer(user.id.toString()).remove();
- assertThat(getReviewers(id)).isEmpty();
- }
-
- @Test
- public void readOnlyReviewDb() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- testReadOnly(id);
- }
-
- @Test
- public void readOnlyNoteDb() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- setNoteDbPrimary(id);
- testReadOnly(id);
- }
-
- private void testReadOnly(Change.Id id) throws Exception {
- Timestamp before = TimeUtil.nowTs();
- Timestamp until = new Timestamp(before.getTime() + 1000 * 3600);
-
- // Set read-only.
- Change c = db.changes().get(id);
- assertThat(c).named("change " + id).isNotNull();
- NoteDbChangeState state = NoteDbChangeState.parse(c);
- state = state.withReadOnlyUntil(until);
- c.setNoteDbState(state.toString());
- db.changes().update(Collections.singleton(c));
-
- assertThat(gApi.changes().id(id.get()).get().subject).isEqualTo(PushOneCommit.SUBJECT);
- assertThat(gApi.changes().id(id.get()).get().topic).isNull();
- try {
- gApi.changes().id(id.get()).topic("a-topic");
- fail("expected read-only exception");
- } catch (RestApiException e) {
- Optional<Throwable> oe =
- Throwables.getCausalChain(e)
- .stream()
- .filter(x -> x instanceof OrmRuntimeException)
- .findFirst();
- assertThat(oe).named("OrmRuntimeException in causal chain of " + e).isPresent();
- assertThat(oe.get().getMessage()).contains("read-only");
- }
- assertThat(gApi.changes().id(id.get()).get().topic).isNull();
-
- TestTimeUtil.setClock(new Timestamp(until.getTime() + 1000));
- assertThat(gApi.changes().id(id.get()).get().subject).isEqualTo(PushOneCommit.SUBJECT);
- gApi.changes().id(id.get()).topic("a-topic");
- assertThat(gApi.changes().id(id.get()).get().topic).isEqualTo("a-topic");
- }
-
- @Test
- public void migrateToNoteDb() throws Exception {
- testMigrateToNoteDb(createChange().getChange().getId());
- }
-
- @Test
- public void migrateToNoteDbWithRebuildingFirst() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- Change c = db.changes().get(id);
- c.setNoteDbState("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef");
- db.changes().update(Collections.singleton(c));
- testMigrateToNoteDb(id);
- }
-
- private void testMigrateToNoteDb(Change.Id id) throws Exception {
- assertThat(PrimaryStorage.of(db.changes().get(id))).isEqualTo(PrimaryStorage.REVIEW_DB);
- migrator.migrateToNoteDbPrimary(id);
- assertNoteDbPrimary(id);
-
- gApi.changes().id(id.get()).topic("a-topic");
- assertThat(gApi.changes().id(id.get()).get().topic).isEqualTo("a-topic");
- assertThat(db.changes().get(id).getTopic()).isNull();
- }
-
- @Test
- public void migrateToNoteDbFailsRebuildingOnceAndRetries() throws Exception {
- Change.Id id = createChange().getChange().getId();
-
- Change c = db.changes().get(id);
- c.setNoteDbState("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef");
- db.changes().update(Collections.singleton(c));
- rebuilderWrapper.failNextUpdate();
-
- migrator =
- newMigrator(
- RetryerBuilder.<NoteDbChangeState>newBuilder()
- .retryIfException()
- .withStopStrategy(StopStrategies.neverStop())
- .build());
- migrator.migrateToNoteDbPrimary(id);
- assertNoteDbPrimary(id);
- }
-
- @Test
- public void migrateToNoteDbFailsRebuildingAndStops() throws Exception {
- Change.Id id = createChange().getChange().getId();
-
- Change c = db.changes().get(id);
- c.setNoteDbState("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef");
- db.changes().update(Collections.singleton(c));
- rebuilderWrapper.failNextUpdate();
-
- migrator =
- newMigrator(
- RetryerBuilder.<NoteDbChangeState>newBuilder()
- .retryIfException()
- .withStopStrategy(StopStrategies.stopAfterAttempt(1))
- .build());
- exception.expect(OrmException.class);
- exception.expectMessage("Retrying failed");
- migrator.migrateToNoteDbPrimary(id);
- }
-
- @Test
- public void migrateToNoteDbMissingOldState() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- Change c = db.changes().get(id);
- c.setNoteDbState(null);
- db.changes().update(Collections.singleton(c));
-
- exception.expect(PrimaryStorageMigrator.NoNoteDbStateException.class);
- exception.expectMessage("no note_db_state");
- migrator.migrateToNoteDbPrimary(id);
- }
-
- @Test
- public void migrateToNoteDbLeaseExpires() throws Exception {
- TestTimeUtil.resetWithClockStep(2, DAYS);
- exception.expect(OrmRuntimeException.class);
- exception.expectMessage("read-only lease");
- migrator.migrateToNoteDbPrimary(createChange().getChange().getId());
- }
-
- @Test
- public void migrateToNoteDbAlreadyReadOnly() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- Change c = db.changes().get(id);
- NoteDbChangeState state = NoteDbChangeState.parse(c);
- Timestamp until = new Timestamp(TimeUtil.nowMs() + MILLISECONDS.convert(1, DAYS));
- state = state.withReadOnlyUntil(until);
- c.setNoteDbState(state.toString());
- db.changes().update(Collections.singleton(c));
-
- exception.expect(OrmRuntimeException.class);
- exception.expectMessage("read-only until " + until);
- migrator.migrateToNoteDbPrimary(id);
- }
-
- @Test
- public void migrateToNoteDbAlreadyMigrated() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- assertThat(PrimaryStorage.of(db.changes().get(id))).isEqualTo(PrimaryStorage.REVIEW_DB);
- migrator.migrateToNoteDbPrimary(id);
- assertNoteDbPrimary(id);
-
- migrator.migrateToNoteDbPrimary(id);
- assertNoteDbPrimary(id);
- }
-
- @Test
- public void rebuildReviewDb() throws Exception {
- Change c = createChange().getChange().change();
- Change.Id id = c.getId();
-
- CommentInput cin = new CommentInput();
- cin.line = 1;
- cin.message = "Published comment";
- ReviewInput rin = ReviewInput.approve();
- rin.comments = ImmutableMap.of(PushOneCommit.FILE_NAME, ImmutableList.of(cin));
- gApi.changes().id(id.get()).current().review(ReviewInput.approve());
-
- DraftInput din = new DraftInput();
- din.path = PushOneCommit.FILE_NAME;
- din.line = 1;
- din.message = "Draft comment";
- gApi.changes().id(id.get()).current().createDraft(din);
- gApi.changes().id(id.get()).current().review(ReviewInput.approve());
- gApi.changes().id(id.get()).current().createDraft(din);
-
- assertThat(db.changeMessages().byChange(id)).isNotEmpty();
- assertThat(db.patchSets().byChange(id)).isNotEmpty();
- assertThat(db.patchSetApprovals().byChange(id)).isNotEmpty();
- assertThat(db.patchComments().byChange(id)).isNotEmpty();
-
- ChangeBundle noteDbBundle =
- ChangeBundle.fromNotes(commentsUtil, notesFactory.create(db, project, id));
-
- setNoteDbPrimary(id);
-
- db.changeMessages().delete(db.changeMessages().byChange(id));
- db.patchSets().delete(db.patchSets().byChange(id));
- db.patchSetApprovals().delete(db.patchSetApprovals().byChange(id));
- db.patchComments().delete(db.patchComments().byChange(id));
- ChangeMessage bogusMessage =
- ChangeMessagesUtil.newMessage(
- c.currentPatchSetId(),
- identifiedUserFactory.create(admin.getId()),
- TimeUtil.nowTs(),
- "some message",
- null);
- db.changeMessages().insert(Collections.singleton(bogusMessage));
-
- rebuilderWrapper.rebuildReviewDb(db, project, id);
-
- assertThat(db.changeMessages().byChange(id)).isNotEmpty();
- assertThat(db.patchSets().byChange(id)).isNotEmpty();
- assertThat(db.patchSetApprovals().byChange(id)).isNotEmpty();
- assertThat(db.patchComments().byChange(id)).isNotEmpty();
-
- ChangeBundle reviewDbBundle = bundleReader.fromReviewDb(ReviewDbUtil.unwrapDb(db), id);
- assertThat(reviewDbBundle.differencesFrom(noteDbBundle)).isEmpty();
- }
-
- @Test
- public void migrateBackToReviewDbPrimary() throws Exception {
- Change c = createChange().getChange().change();
- Change.Id id = c.getId();
-
- migrator.migrateToNoteDbPrimary(id);
- assertNoteDbPrimary(id);
-
- gApi.changes().id(id.get()).topic("new-topic");
- assertThat(gApi.changes().id(id.get()).topic()).isEqualTo("new-topic");
- assertThat(db.changes().get(id).getTopic()).isNotEqualTo("new-topic");
-
- migrator.migrateToReviewDbPrimary(id, null);
- ObjectId metaId;
- try (Repository repo = repoManager.openRepository(c.getProject());
- RevWalk rw = new RevWalk(repo)) {
- metaId = repo.exactRef(RefNames.changeMetaRef(id)).getObjectId();
- RevCommit commit = rw.parseCommit(metaId);
- rw.parseBody(commit);
- assertThat(commit.getFullMessage())
- .contains("Read-only-until: " + formatTime(serverIdent.get(), new Timestamp(0)));
- }
- NoteDbChangeState state = NoteDbChangeState.parse(db.changes().get(id));
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.REVIEW_DB);
- assertThat(state.getChangeMetaId()).isEqualTo(metaId);
- assertThat(gApi.changes().id(id.get()).topic()).isEqualTo("new-topic");
- assertThat(db.changes().get(id).getTopic()).isEqualTo("new-topic");
-
- ChangeNotes notes = notesFactory.create(db, project, id);
- assertThat(notes.getRevision()).isEqualTo(metaId); // No rebuilding, change was up to date.
- assertThat(notes.getReadOnlyUntil()).isNotNull();
-
- gApi.changes().id(id.get()).topic("reviewdb-topic");
- assertThat(db.changes().get(id).getTopic()).isEqualTo("reviewdb-topic");
- }
-
- private void setNoteDbPrimary(Change.Id id) throws Exception {
- Change c = db.changes().get(id);
- assertThat(c).named("change " + id).isNotNull();
- NoteDbChangeState state = NoteDbChangeState.parse(c);
- assertThat(state.getPrimaryStorage()).named("storage of " + id).isEqualTo(REVIEW_DB);
-
- try (Repository changeRepo = repoManager.openRepository(c.getProject());
- Repository allUsersRepo = repoManager.openRepository(allUsers)) {
- assertThat(state.isUpToDate(new RepoRefCache(changeRepo), new RepoRefCache(allUsersRepo)))
- .named("change " + id + " up to date")
- .isTrue();
- }
-
- c.setNoteDbState(NoteDbChangeState.NOTE_DB_PRIMARY_STATE);
- db.changes().update(Collections.singleton(c));
- }
-
- private void assertNoteDbPrimary(Change.Id id) throws Exception {
- assertThat(PrimaryStorage.of(db.changes().get(id))).isEqualTo(PrimaryStorage.NOTE_DB);
- }
-
- private List<Account.Id> getReviewers(Change.Id id) throws Exception {
- return gApi.changes()
- .id(id.get())
- .get()
- .reviewers
- .values()
- .stream()
- .flatMap(Collection::stream)
- .map(a -> new Account.Id(a._accountId))
- .collect(toList());
- }
-}
diff --git a/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java b/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java
deleted file mode 100644
index 6f226c8..0000000
--- a/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java
+++ /dev/null
@@ -1,629 +0,0 @@
-// Copyright (C) 2017 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.acceptance.server.notedb;
-
-import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
-import static com.google.common.truth.TruthJUnit.assume;
-import static com.google.gerrit.server.notedb.NoteDbChangeState.NOTE_DB_PRIMARY_STATE;
-import static com.google.gerrit.server.notedb.NotesMigrationState.NOTE_DB;
-import static com.google.gerrit.server.notedb.NotesMigrationState.READ_WRITE_NO_SEQUENCE;
-import static com.google.gerrit.server.notedb.NotesMigrationState.READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY;
-import static com.google.gerrit.server.notedb.NotesMigrationState.READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY;
-import static com.google.gerrit.server.notedb.NotesMigrationState.REVIEW_DB;
-import static com.google.gerrit.server.notedb.NotesMigrationState.WRITE;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Comparator.naturalOrder;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.gerrit.acceptance.AbstractDaemonTest;
-import com.google.gerrit.acceptance.GerritConfig;
-import com.google.gerrit.acceptance.NoHttpd;
-import com.google.gerrit.acceptance.PushOneCommit;
-import com.google.gerrit.acceptance.Sandboxed;
-import com.google.gerrit.acceptance.UseLocalDisk;
-import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
-import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gerrit.extensions.registration.RegistrationHandle;
-import com.google.gerrit.reviewdb.client.Change;
-import com.google.gerrit.reviewdb.client.PatchSet;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
-import com.google.gerrit.server.CommentsUtil;
-import com.google.gerrit.server.Sequences;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.notedb.ChangeBundle;
-import com.google.gerrit.server.notedb.ChangeBundleReader;
-import com.google.gerrit.server.notedb.NoteDbChangeState;
-import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
-import com.google.gerrit.server.notedb.NoteDbChangeState.RefState;
-import com.google.gerrit.server.notedb.NotesMigrationState;
-import com.google.gerrit.server.notedb.rebuild.MigrationException;
-import com.google.gerrit.server.notedb.rebuild.NoteDbMigrator;
-import com.google.gerrit.server.notedb.rebuild.NotesMigrationStateListener;
-import com.google.gerrit.server.schema.ReviewDbFactory;
-import com.google.gerrit.testing.ConfigSuite;
-import com.google.gerrit.testing.NoteDbMode;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Stream;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.junit.TestRepository;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectLoader;
-import org.eclipse.jgit.lib.ObjectReader;
-import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.RefUpdate;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-@Sandboxed
-@UseLocalDisk
-@NoHttpd
-public class OnlineNoteDbMigrationIT extends AbstractDaemonTest {
- private static final String INVALID_STATE = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
- @Inject private ProjectOperations projectOperations;
-
- @ConfigSuite.Default
- public static Config defaultConfig() {
- Config cfg = new Config();
- cfg.setInt("noteDb", "changes", "sequenceBatchSize", 10);
- cfg.setInt("noteDb", "changes", "initialSequenceGap", 500);
- return cfg;
- }
-
- // Tests in this class are generally interested in the actual ReviewDb contents, but the shifting
- // migration state may result in various kinds of wrappers showing up unexpectedly.
- @Inject @ReviewDbFactory private SchemaFactory<ReviewDb> schemaFactory;
-
- @Inject private ChangeBundleReader changeBundleReader;
- @Inject private CommentsUtil commentsUtil;
- @Inject private DynamicSet<NotesMigrationStateListener> listeners;
- @Inject private Provider<NoteDbMigrator.Builder> migratorBuilderProvider;
- @Inject private Sequences sequences;
- @Inject private SitePaths sitePaths;
-
- private FileBasedConfig noteDbConfig;
- private List<RegistrationHandle> addedListeners;
-
- @Before
- public void setUp() throws Exception {
- assume().that(NoteDbMode.get()).isEqualTo(NoteDbMode.OFF);
- // Unlike in the running server, for tests, we don't stack notedb.config on gerrit.config.
- noteDbConfig = new FileBasedConfig(sitePaths.notedb_config.toFile(), FS.detect());
- assertNotesMigrationState(REVIEW_DB, false, false);
- addedListeners = new ArrayList<>();
- }
-
- @After
- public void tearDown() throws Exception {
- if (addedListeners != null) {
- addedListeners.forEach(RegistrationHandle::remove);
- addedListeners = null;
- }
- }
-
- @Test
- public void preconditionsFail() throws Exception {
- List<Change.Id> cs = ImmutableList.of(new Change.Id(1));
- List<Project.NameKey> ps = ImmutableList.of(new Project.NameKey("p"));
- assertMigrationException(
- "Cannot rebuild without noteDb.changes.write=true", b -> b, NoteDbMigrator::rebuild);
- assertMigrationException(
- "Cannot set both changes and projects", b -> b.setChanges(cs).setProjects(ps), m -> {});
- assertMigrationException(
- "Cannot set changes or projects during full migration",
- b -> b.setChanges(cs),
- NoteDbMigrator::migrate);
- assertMigrationException(
- "Cannot set changes or projects during full migration",
- b -> b.setProjects(ps),
- NoteDbMigrator::migrate);
-
- setNotesMigrationState(READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY);
- assertMigrationException(
- "Migration has already progressed past the endpoint of the \"trial mode\" state",
- b -> b.setTrialMode(true),
- NoteDbMigrator::migrate);
-
- setNotesMigrationState(READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY);
- assertMigrationException(
- "Cannot force rebuild changes; NoteDb is already the primary storage for some changes",
- b -> b.setForceRebuild(true),
- NoteDbMigrator::migrate);
- }
-
- @Test
- @GerritConfig(name = "noteDb.changes.initialSequenceGap", value = "-7")
- public void initialSequenceGapMustBeNonNegative() throws Exception {
- setNotesMigrationState(READ_WRITE_NO_SEQUENCE);
- assertMigrationException("Sequence gap must be non-negative: -7", b -> b, m -> {});
- }
-
- @Test
- public void rebuildOneChangeTrialModeAndForceRebuild() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- migrate(b -> b.setTrialMode(true));
- assertNotesMigrationState(READ_WRITE_NO_SEQUENCE, false, true);
-
- ObjectId oldMetaId;
- try (Repository repo = repoManager.openRepository(project);
- ReviewDb db = schemaFactory.open()) {
- Ref ref = repo.exactRef(RefNames.changeMetaRef(id));
- assertThat(ref).isNotNull();
- oldMetaId = ref.getObjectId();
-
- Change c = db.changes().get(id);
- assertThat(c).isNotNull();
- NoteDbChangeState state = NoteDbChangeState.parse(c);
- assertThat(state).isNotNull();
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.REVIEW_DB);
- assertThat(state.getRefState()).hasValue(RefState.create(oldMetaId, ImmutableMap.of()));
-
- // Force change to be out of date, and change topic so it will get rebuilt as something other
- // than oldMetaId.
- c.setNoteDbState(INVALID_STATE);
- c.setTopic(name("a-new-topic"));
- db.changes().update(ImmutableList.of(c));
- }
-
- migrate(b -> b.setTrialMode(true));
- assertNotesMigrationState(READ_WRITE_NO_SEQUENCE, false, true);
-
- try (Repository repo = repoManager.openRepository(project);
- ReviewDb db = schemaFactory.open()) {
- // Change is out of date, but was not rebuilt without forceRebuild.
- assertThat(repo.exactRef(RefNames.changeMetaRef(id)).getObjectId()).isEqualTo(oldMetaId);
- Change c = db.changes().get(id);
- assertThat(c.getNoteDbState()).isEqualTo(INVALID_STATE);
- }
-
- migrate(b -> b.setTrialMode(true).setForceRebuild(true));
- assertNotesMigrationState(READ_WRITE_NO_SEQUENCE, false, true);
-
- try (Repository repo = repoManager.openRepository(project);
- ReviewDb db = schemaFactory.open()) {
- Ref ref = repo.exactRef(RefNames.changeMetaRef(id));
- assertThat(ref).isNotNull();
- ObjectId newMetaId = ref.getObjectId();
- assertThat(newMetaId).isNotEqualTo(oldMetaId);
-
- NoteDbChangeState state = NoteDbChangeState.parse(db.changes().get(id));
- assertThat(state).isNotNull();
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.REVIEW_DB);
- assertThat(state.getRefState()).hasValue(RefState.create(newMetaId, ImmutableMap.of()));
- }
- }
-
- @Test
- public void autoMigrateTrialMode() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- migrate(b -> b.setAutoMigrate(true).setTrialMode(true).setStopAtStateForTesting(WRITE));
- assertNotesMigrationState(WRITE, true, true);
-
- migrate(b -> b);
- // autoMigrate is still enabled so that we can continue the migration by only unsetting trial.
- assertNotesMigrationState(READ_WRITE_NO_SEQUENCE, true, true);
-
- ObjectId metaId;
- try (Repository repo = repoManager.openRepository(project);
- ReviewDb db = schemaFactory.open()) {
- Ref ref = repo.exactRef(RefNames.changeMetaRef(id));
- assertThat(ref).isNotNull();
- metaId = ref.getObjectId();
- NoteDbChangeState state = NoteDbChangeState.parse(db.changes().get(id));
- assertThat(state).isNotNull();
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.REVIEW_DB);
- assertThat(state.getRefState()).hasValue(RefState.create(metaId, ImmutableMap.of()));
- }
-
- // Unset trial mode and the next migration runs to completion.
- noteDbConfig.load();
- NoteDbMigrator.setTrialMode(noteDbConfig, false);
- noteDbConfig.save();
-
- migrate(b -> b);
- assertNotesMigrationState(NOTE_DB, false, false);
-
- try (Repository repo = repoManager.openRepository(project);
- ReviewDb db = schemaFactory.open()) {
- Ref ref = repo.exactRef(RefNames.changeMetaRef(id));
- assertThat(ref).isNotNull();
- assertThat(ref.getObjectId()).isEqualTo(metaId);
- NoteDbChangeState state = NoteDbChangeState.parse(db.changes().get(id));
- assertThat(state).isNotNull();
- assertThat(state.getPrimaryStorage()).isEqualTo(PrimaryStorage.NOTE_DB);
- }
- }
-
- @Test
- public void rebuildSubsetOfChanges() throws Exception {
- setNotesMigrationState(WRITE);
-
- PushOneCommit.Result r1 = createChange();
- PushOneCommit.Result r2 = createChange();
- Change.Id id1 = r1.getChange().getId();
- Change.Id id2 = r2.getChange().getId();
-
- try (ReviewDb db = schemaFactory.open()) {
- Change c1 = db.changes().get(id1);
- c1.setNoteDbState(INVALID_STATE);
- Change c2 = db.changes().get(id2);
- c2.setNoteDbState(INVALID_STATE);
- db.changes().update(ImmutableList.of(c1, c2));
- }
-
- migrate(b -> b.setChanges(ImmutableList.of(id2)), NoteDbMigrator::rebuild);
-
- try (ReviewDb db = schemaFactory.open()) {
- NoteDbChangeState s1 = NoteDbChangeState.parse(db.changes().get(id1));
- assertThat(s1.getChangeMetaId().name()).isEqualTo(INVALID_STATE);
-
- NoteDbChangeState s2 = NoteDbChangeState.parse(db.changes().get(id2));
- assertThat(s2.getChangeMetaId().name()).isNotEqualTo(INVALID_STATE);
- }
- }
-
- @Test
- public void rebuildSubsetOfProjects() throws Exception {
- setNotesMigrationState(WRITE);
-
- Project.NameKey p2 = projectOperations.newProject().create();
- TestRepository<?> tr2 = cloneProject(p2, admin);
-
- PushOneCommit.Result r1 = createChange();
- PushOneCommit.Result r2 = pushFactory.create(db, admin.getIdent(), tr2).to("refs/for/master");
- Change.Id id1 = r1.getChange().getId();
- Change.Id id2 = r2.getChange().getId();
-
- String invalidState = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
- try (ReviewDb db = schemaFactory.open()) {
- Change c1 = db.changes().get(id1);
- c1.setNoteDbState(invalidState);
- Change c2 = db.changes().get(id2);
- c2.setNoteDbState(invalidState);
- db.changes().update(ImmutableList.of(c1, c2));
- }
-
- migrate(b -> b.setProjects(ImmutableList.of(p2)), NoteDbMigrator::rebuild);
-
- try (ReviewDb db = schemaFactory.open()) {
- NoteDbChangeState s1 = NoteDbChangeState.parse(db.changes().get(id1));
- assertThat(s1.getChangeMetaId().name()).isEqualTo(invalidState);
-
- NoteDbChangeState s2 = NoteDbChangeState.parse(db.changes().get(id2));
- assertThat(s2.getChangeMetaId().name()).isNotEqualTo(invalidState);
- }
- }
-
- @Test
- public void enableSequencesNoGap() throws Exception {
- testEnableSequences(0, 3, "13");
- }
-
- @Test
- public void enableSequencesWithGap() throws Exception {
- testEnableSequences(-1, 502, "512");
- }
-
- private void testEnableSequences(int builderOption, int expectedFirstId, String expectedRefValue)
- throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
- assertThat(id.get()).isEqualTo(1);
-
- migrate(
- b ->
- b.setSequenceGap(builderOption)
- .setStopAtStateForTesting(READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY));
-
- assertThat(sequences.nextChangeId()).isEqualTo(expectedFirstId);
- assertThat(sequences.nextChangeId()).isEqualTo(expectedFirstId + 1);
-
- try (Repository repo = repoManager.openRepository(allProjects);
- ObjectReader reader = repo.newObjectReader()) {
- Ref ref = repo.exactRef("refs/sequences/changes");
- assertThat(ref).isNotNull();
- ObjectLoader loader = reader.open(ref.getObjectId());
- assertThat(loader.getType()).isEqualTo(Constants.OBJ_BLOB);
- // Acquired a block of 10 to serve the first nextChangeId call after migration.
- assertThat(new String(loader.getCachedBytes(), UTF_8)).isEqualTo(expectedRefValue);
- }
-
- try (ReviewDb db = schemaFactory.open()) {
- // Underlying, unused ReviewDb is still on its own sequence.
- @SuppressWarnings("deprecation")
- int nextFromReviewDb = db.nextChangeId();
- assertThat(nextFromReviewDb).isEqualTo(3);
- }
- }
-
- @Test
- public void fullMigrationSameThread() throws Exception {
- testFullMigration(1);
- }
-
- @Test
- public void fullMigrationMultipleThreads() throws Exception {
- testFullMigration(2);
- }
-
- private void testFullMigration(int threads) throws Exception {
- PushOneCommit.Result r1 = createChange();
- PushOneCommit.Result r2 = createChange();
- Change.Id id1 = r1.getChange().getId();
- Change.Id id2 = r2.getChange().getId();
-
- Set<String> objectFiles = getObjectFiles(project);
- assertThat(objectFiles).isNotEmpty();
-
- migrate(b -> b.setThreads(threads));
-
- assertNotesMigrationState(NOTE_DB, false, false);
- assertThat(sequences.nextChangeId()).isEqualTo(503);
- assertThat(getObjectFiles(project)).containsExactlyElementsIn(objectFiles);
-
- ObjectId oldMetaId = null;
- int rowVersion = 0;
- try (ReviewDb db = schemaFactory.open();
- Repository repo = repoManager.openRepository(project)) {
- for (Change.Id id : ImmutableList.of(id1, id2)) {
- String refName = RefNames.changeMetaRef(id);
- Ref ref = repo.exactRef(refName);
- assertThat(ref).named(refName).isNotNull();
-
- Change c = db.changes().get(id);
- assertThat(c.getTopic()).named("topic of change %s", id).isNull();
- NoteDbChangeState s = NoteDbChangeState.parse(c);
- assertThat(s.getPrimaryStorage())
- .named("primary storage of change %s", id)
- .isEqualTo(PrimaryStorage.NOTE_DB);
- assertThat(s.getRefState()).named("ref state of change %s").isEmpty();
-
- if (id.equals(id1)) {
- oldMetaId = ref.getObjectId();
- rowVersion = c.getRowVersion();
- }
- }
- }
-
- // Do not open a new context, to simulate races with other threads that opened a context earlier
- // in the migration process; this needs to work.
- gApi.changes().id(id1.get()).topic(name("a-topic"));
-
- // Of course, it should also work with a new context.
- resetCurrentApiUser();
- gApi.changes().id(id1.get()).topic(name("another-topic"));
-
- try (ReviewDb db = schemaFactory.open();
- Repository repo = repoManager.openRepository(project)) {
- assertThat(repo.exactRef(RefNames.changeMetaRef(id1)).getObjectId()).isNotEqualTo(oldMetaId);
-
- Change c = db.changes().get(id1);
- assertThat(c.getTopic()).isNull();
- assertThat(c.getRowVersion()).isEqualTo(rowVersion);
- }
- }
-
- @Test
- public void fullMigrationOneChangeWithNoPatchSets() throws Exception {
- PushOneCommit.Result r1 = createChange();
- PushOneCommit.Result r2 = createChange();
- Change.Id id1 = r1.getChange().getId();
- Change.Id id2 = r2.getChange().getId();
-
- db.changes().beginTransaction(id2);
- try {
- db.patchSets().delete(db.patchSets().byChange(id2));
- db.commit();
- } finally {
- db.rollback();
- }
-
- migrate(b -> b);
- assertNotesMigrationState(NOTE_DB, false, false);
-
- try (ReviewDb db = schemaFactory.open();
- Repository repo = repoManager.openRepository(project)) {
- assertThat(repo.exactRef(RefNames.changeMetaRef(id1))).isNotNull();
- assertThat(db.changes().get(id1).getNoteDbState()).isEqualTo(NOTE_DB_PRIMARY_STATE);
-
- // A change with no patch sets is so corrupt that it is completely skipped by the migration
- // process.
- assertThat(repo.exactRef(RefNames.changeMetaRef(id2))).isNull();
- assertThat(db.changes().get(id2).getNoteDbState()).isNull();
- }
- }
-
- @Test
- public void fullMigrationMissingPatchSetRefs() throws Exception {
- PushOneCommit.Result r = createChange();
- Change.Id id = r.getChange().getId();
-
- try (Repository repo = repoManager.openRepository(project)) {
- RefUpdate u = repo.updateRef(new PatchSet.Id(id, 1).toRefName());
- u.setForceUpdate(true);
- assertThat(u.delete()).isEqualTo(RefUpdate.Result.FORCED);
- }
-
- ChangeBundle reviewDbBundle;
- try (ReviewDb db = schemaFactory.open()) {
- reviewDbBundle = changeBundleReader.fromReviewDb(db, id);
- }
-
- migrate(b -> b);
- assertNotesMigrationState(NOTE_DB, false, false);
-
- try (ReviewDb db = schemaFactory.open();
- Repository repo = repoManager.openRepository(project)) {
- // Change migrated successfully even though it was missing patch set refs.
- assertThat(repo.exactRef(RefNames.changeMetaRef(id))).isNotNull();
- assertThat(db.changes().get(id).getNoteDbState()).isEqualTo(NOTE_DB_PRIMARY_STATE);
-
- ChangeBundle noteDbBundle =
- ChangeBundle.fromNotes(commentsUtil, notesFactory.createChecked(db, project, id));
- assertThat(noteDbBundle.differencesFrom(reviewDbBundle)).isEmpty();
- }
- }
-
- @Test
- public void autoMigrationConfig() throws Exception {
- createChange();
-
- migrate(b -> b.setStopAtStateForTesting(WRITE));
- assertNotesMigrationState(WRITE, false, false);
-
- migrate(b -> b.setAutoMigrate(true).setStopAtStateForTesting(READ_WRITE_NO_SEQUENCE));
- assertNotesMigrationState(READ_WRITE_NO_SEQUENCE, true, false);
-
- migrate(b -> b);
- assertNotesMigrationState(NOTE_DB, false, false);
- }
-
- @Test
- public void notesMigrationStateListener() throws Exception {
- NotesMigrationStateListener listener = createStrictMock(NotesMigrationStateListener.class);
- listener.preStateChange(REVIEW_DB, WRITE);
- expectLastCall();
- listener.preStateChange(WRITE, READ_WRITE_NO_SEQUENCE);
- expectLastCall();
- listener.preStateChange(READ_WRITE_NO_SEQUENCE, READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY);
- expectLastCall();
- listener.preStateChange(
- READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY, READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY);
- listener.preStateChange(READ_WRITE_WITH_SEQUENCE_NOTE_DB_PRIMARY, NOTE_DB);
- expectLastCall();
- replay(listener);
- addListener(listener);
-
- createChange();
- migrate(b -> b);
- assertNotesMigrationState(NOTE_DB, false, false);
- verify(listener);
- }
-
- @Test
- public void notesMigrationStateListenerFails() throws Exception {
- NotesMigrationStateListener listener = createStrictMock(NotesMigrationStateListener.class);
- listener.preStateChange(REVIEW_DB, WRITE);
- expectLastCall();
- listener.preStateChange(WRITE, READ_WRITE_NO_SEQUENCE);
- IOException listenerException = new IOException("Listener failed");
- expectLastCall().andThrow(listenerException);
- replay(listener);
- addListener(listener);
-
- createChange();
- try {
- migrate(b -> b);
- fail("expected IOException");
- } catch (IOException e) {
- assertThat(e).isSameAs(listenerException);
- }
- assertNotesMigrationState(WRITE, false, false);
- verify(listener);
- }
-
- private void assertNotesMigrationState(
- NotesMigrationState expected, boolean autoMigrate, boolean trialMode) throws Exception {
- assertThat(NotesMigrationState.forNotesMigration(notesMigration)).hasValue(expected);
- noteDbConfig.load();
- assertThat(NotesMigrationState.forConfig(noteDbConfig)).hasValue(expected);
- assertThat(NoteDbMigrator.getAutoMigrate(noteDbConfig))
- .named("noteDb.changes.autoMigrate")
- .isEqualTo(autoMigrate);
- assertThat(NoteDbMigrator.getTrialMode(noteDbConfig))
- .named("noteDb.changes.trial")
- .isEqualTo(trialMode);
- }
-
- private void setNotesMigrationState(NotesMigrationState state) throws Exception {
- noteDbConfig.load();
- state.setConfigValues(noteDbConfig);
- noteDbConfig.save();
- notesMigration.setFrom(state);
- }
-
- @FunctionalInterface
- interface PrepareBuilder {
- NoteDbMigrator.Builder prepare(NoteDbMigrator.Builder b) throws Exception;
- }
-
- @FunctionalInterface
- interface RunMigration {
- void run(NoteDbMigrator m) throws Exception;
- }
-
- private void migrate(PrepareBuilder b) throws Exception {
- migrate(b, NoteDbMigrator::migrate);
- }
-
- private void migrate(PrepareBuilder b, RunMigration m) throws Exception {
- try (NoteDbMigrator migrator = b.prepare(migratorBuilderProvider.get()).build()) {
- m.run(migrator);
- }
- }
-
- private void assertMigrationException(
- String expectMessageContains, PrepareBuilder b, RunMigration m) throws Exception {
- try {
- migrate(b, m);
- fail("expected MigrationException");
- } catch (MigrationException e) {
- assertThat(e).hasMessageThat().contains(expectMessageContains);
- }
- }
-
- private void addListener(NotesMigrationStateListener listener) {
- addedListeners.add(listeners.add("gerrit", listener));
- }
-
- private ImmutableSortedSet<String> getObjectFiles(Project.NameKey project) throws Exception {
- try (Repository repo = repoManager.openRepository(project);
- Stream<Path> paths =
- Files.walk(((FileRepository) repo).getObjectDatabase().getDirectory().toPath())) {
- return paths
- .filter(path -> !Files.isDirectory(path))
- .map(Path::toString)
- .filter(name -> !name.endsWith(".pack") && !name.endsWith(".idx"))
- .collect(toImmutableSortedSet(naturalOrder()));
- }
- }
-}
