Merge branch 'stable-2.15' into stable-2.16
* stable-2.15:
Schema_146: Disable bitmap index re-build during full gc
Schema_154: Disable bitmap index re-build during full gc
Schema_154: Periodically run full gc
Schema_146: Periodically run full gc
Revert "Keep alive database connection to prevent exceeding wait timeout"
Change-Id: I13e44e3f798a0b48adb59fe505d90e4a7f578b58
diff --git a/java/com/google/gerrit/server/schema/Schema_146.java b/java/com/google/gerrit/server/schema/Schema_146.java
index e0127c0..cdb65c9 100644
--- a/java/com/google/gerrit/server/schema/Schema_146.java
+++ b/java/com/google/gerrit/server/schema/Schema_146.java
@@ -23,7 +23,6 @@
import com.google.gerrit.server.GerritPersonIdent;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gwtorm.jdbc.JdbcSchema;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Provider;
@@ -62,6 +61,7 @@
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevSort;
import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.pack.PackConfig;
/**
* Make sure that for every account a user branch exists that has an initial empty commit with the
@@ -101,7 +101,9 @@
protected void migrateData(ReviewDb db, UpdateUI ui) throws OrmException, SQLException {
ui.message("Migrating accounts");
Set<Entry<Account.Id, Timestamp>> accounts = scanAccounts(db, ui).entrySet();
+ ui.message("Run full gc as preparation for the migration");
gc(ui);
+ ui.message(String.format("... (%.3f s) full gc completed", elapsed()));
Set<List<Entry<Account.Id, Timestamp>>> batches =
Sets.newHashSet(Iterables.partition(accounts, 500));
ExecutorService pool = createExecutor(ui);
@@ -110,7 +112,7 @@
.forEach(
batch -> {
@SuppressWarnings("unused")
- Future<?> unused = pool.submit(() -> processBatch(db, batch, ui));
+ Future<?> unused = pool.submit(() -> processBatch(batch, ui));
});
pool.shutdown();
pool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
@@ -119,6 +121,9 @@
}
ui.message(
String.format("... (%.3f s) Migrated all %d accounts to schema 146", elapsed(), i.get()));
+ ui.message("Run full gc");
+ gc(ui);
+ ui.message(String.format("... (%.3f s) full gc completed", elapsed()));
}
private ExecutorService createExecutor(UpdateUI ui) {
@@ -132,7 +137,7 @@
return Executors.newFixedThreadPool(threads);
}
- private void processBatch(ReviewDb db, List<Entry<Account.Id, Timestamp>> batch, UpdateUI ui) {
+ private void processBatch(List<Entry<Account.Id, Timestamp>> batch, UpdateUI ui) {
try (Repository repo = repoManager.openRepository(allUsersName);
RevWalk rw = new RevWalk(repo);
ObjectInserter oi = repo.newObjectInserter()) {
@@ -149,29 +154,18 @@
int count = i.incrementAndGet();
showProgress(ui, count);
if (count % 1000 == 0) {
- gc(repo, true, ui);
- keepAliveDatabaseConnection(db);
+ boolean runFullGc = count % 100000 == 0;
+ if (runFullGc) {
+ ui.message("Run full gc");
+ }
+ gc(repo, !runFullGc, ui);
+ if (runFullGc) {
+ ui.message(String.format("... (%.3f s) full gc completed", elapsed()));
+ }
}
}
} catch (IOException e) {
throw new UncheckedIOException(e);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static String dummySelectStatement() {
- // TODO(davido): Gwtorm doesn't expose dummySelectStatement() method for all supported SQL
- // dialects.
- return "SELECT version_nbr FROM schema_version";
- }
-
- private static void keepAliveDatabaseConnection(ReviewDb db) throws SQLException {
- try (Statement stmt = ((JdbcSchema) db).getConnection().createStatement();
- ResultSet rs = stmt.executeQuery(dummySelectStatement())) {
- // No Op.
- // The select is fired to prevent the SQL connection from becoming stale
- // and being closed on the server side during long running batch operation.
}
}
@@ -209,6 +203,11 @@
ui.message(String.format("... (%.3f s) pack refs", elapsed()));
gc.packRefs();
} else {
+ // TODO(ms): Enable bitmap index when this JGit performance issue is fixed:
+ // https://bugs.eclipse.org/bugs/show_bug.cgi?id=562740
+ PackConfig pconfig = new PackConfig(repo);
+ pconfig.setBuildBitmaps(false);
+ gc.setPackConfig(pconfig);
ui.message(String.format("... (%.3f s) gc --prune=now", elapsed()));
gc.setExpire(new Date());
gc.gc();
@@ -303,7 +302,7 @@
return oi.insert(cb);
}
- private static boolean isInitialEmptyCommit(ObjectId emptyTree, RevCommit c) {
+ private boolean isInitialEmptyCommit(ObjectId emptyTree, RevCommit c) {
return c.getParentCount() == 0
&& c.getTree().equals(emptyTree)
&& c.getShortMessage().equals(CREATE_ACCOUNT_MSG);
diff --git a/java/com/google/gerrit/server/schema/Schema_154.java b/java/com/google/gerrit/server/schema/Schema_154.java
index 11fa89e..8a39c0d 100644
--- a/java/com/google/gerrit/server/schema/Schema_154.java
+++ b/java/com/google/gerrit/server/schema/Schema_154.java
@@ -16,6 +16,7 @@
import static java.util.stream.Collectors.toMap;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.Account;
@@ -35,16 +36,22 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.internal.storage.file.GC;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.TextProgressMonitor;
+import org.eclipse.jgit.storage.pack.PackConfig;
/** Migrate accounts to NoteDb. */
public class Schema_154 extends SchemaVersion {
@@ -62,6 +69,7 @@
private final GitRepositoryManager repoManager;
private final AllUsersName allUsersName;
private final Provider<PersonIdent> serverIdent;
+ private final Stopwatch sw = Stopwatch.createStarted();
@Inject
Schema_154(
@@ -84,9 +92,13 @@
Set<Account> accounts = scanAccounts(db, pm);
pm.endTask();
pm.beginTask("Migrating accounts to NoteDb", accounts.size());
+ int i = 0;
for (Account account : accounts) {
updateAccountInNoteDb(repo, account);
pm.update(1);
+ if (++i % 100000 == 0) {
+ gc(repo, ui);
+ }
}
pm.endTask();
}
@@ -147,4 +159,36 @@
private interface AccountSetter {
void set(Account a, ResultSet rs, String field) throws SQLException;
}
+
+ private double elapsed() {
+ return sw.elapsed(TimeUnit.MILLISECONDS) / 1000d;
+ }
+
+ private void gc(Repository repo, UpdateUI ui) {
+ if (repo instanceof FileRepository) {
+ ProgressMonitor pm = null;
+ try {
+ pm = new TextProgressMonitor();
+ FileRepository r = (FileRepository) repo;
+ GC gc = new GC(r);
+ // TODO(davido): Enable bitmap index when this JGit performance issue is fixed:
+ // https://bugs.eclipse.org/bugs/show_bug.cgi?id=562740
+ PackConfig pconfig = new PackConfig(repo);
+ pconfig.setBuildBitmaps(false);
+ gc.setPackConfig(pconfig);
+ gc.setProgressMonitor(pm);
+ pm.beginTask("gc", ProgressMonitor.UNKNOWN);
+ ui.message(String.format("... (%.3f s) gc --prune=now", elapsed()));
+ gc.setExpire(new Date());
+ gc.gc();
+ ui.message(String.format("... (%.3f s) full gc completed", elapsed()));
+ } catch (IOException | ParseException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (pm != null) {
+ pm.endTask();
+ }
+ }
+ }
+ }
}
