blob: e800d1788e96320c97165ca8a0f370b2a39cbec8 [file] [log] [blame]
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.pgm;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toSet;
import com.google.common.cache.Cache;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.gerrit.common.Die;
import com.google.gerrit.extensions.config.FactoryModule;
import com.google.gerrit.extensions.registration.DynamicMap;
import com.google.gerrit.index.Index;
import com.google.gerrit.index.IndexDefinition;
import com.google.gerrit.index.IndexType;
import com.google.gerrit.index.SiteIndexer;
import com.google.gerrit.lifecycle.LifecycleManager;
import com.google.gerrit.lucene.LuceneIndexModule;
import com.google.gerrit.pgm.util.BatchProgramModule;
import com.google.gerrit.pgm.util.SiteProgram;
import com.google.gerrit.server.LibModuleLoader;
import com.google.gerrit.server.ModuleOverloader;
import com.google.gerrit.server.account.storage.notedb.AccountNoteDbReadStorageModule;
import com.google.gerrit.server.account.storage.notedb.AccountNoteDbWriteStorageModule;
import com.google.gerrit.server.cache.CacheDisplay;
import com.google.gerrit.server.cache.CacheInfo;
import com.google.gerrit.server.change.ChangeResource;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.WorkQueue.WorkQueueModule;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import com.google.gerrit.server.index.options.AutoFlush;
import com.google.gerrit.server.index.options.BuildBloomFilter;
import com.google.gerrit.server.index.options.IsFirstInsertForEntry;
import com.google.gerrit.server.notedb.NoteDbDraftCommentsModule;
import com.google.gerrit.server.notedb.NoteDbStarredChangesModule;
import com.google.gerrit.server.notedb.RepoSequence.RepoSequenceModule;
import com.google.gerrit.server.plugins.PluginGuiceEnvironment;
import com.google.gerrit.server.util.ReplicaUtil;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
import com.google.inject.multibindings.OptionalBinder;
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.kohsuke.args4j.Option;
public class Reindex extends SiteProgram {
@Option(
name = "--threads",
usage = "Number of threads to use for indexing. Default is index.batchThreads from config.")
private int threads = 0;
@Option(
name = "--changes-schema-version",
usage = "Schema version to reindex, for changes; default is most recent version")
private Integer changesVersion;
@Option(name = "--verbose", usage = "Output debug information for each change")
private boolean verbose;
@Option(name = "--list", usage = "List supported indices and exit")
private boolean list;
@Option(name = "--index", usage = "Only reindex specified indices")
private List<String> indices = new ArrayList<>();
@Option(name = "--show-cache-stats", usage = "Show cache statistics at the end.")
private boolean showCacheStats;
@Option(name = "--build-bloom-filter", usage = "Build bloom filter for H2 disk caches.")
private boolean buildBloomFilter;
private Boolean reuseExistingDocumentsOption;
private Injector dbInjector;
private Injector sysInjector;
private Injector cfgInjector;
private Config globalConfig;
@Inject private Collection<IndexDefinition<?, ?, ?>> indexDefs;
@Inject private DynamicMap<Cache<?, ?>> cacheMap;
@Option(name = "--reuse", usage = "Reindex only when existing index entry is stale")
public void setReuseExistingDocuments(boolean value) {
reuseExistingDocumentsOption = value;
}
@Override
public int run() throws Exception {
mustHaveValidSite();
dbInjector = createDbInjector();
cfgInjector = dbInjector.createChildInjector();
globalConfig = dbInjector.getInstance(Key.get(Config.class, GerritServerConfig.class));
overrideConfig();
LifecycleManager dbManager = new LifecycleManager();
dbManager.add(dbInjector);
dbManager.start();
sysInjector = createSysInjector();
sysInjector.getInstance(PluginGuiceEnvironment.class).setDbCfgInjector(dbInjector, cfgInjector);
LifecycleManager sysManager = new LifecycleManager();
sysManager.add(sysInjector);
sysManager.start();
sysInjector.injectMembers(this);
checkIndicesOption();
try {
boolean ok = list ? list() : reindex();
if (showCacheStats) {
printCacheStats();
}
return ok ? 0 : 1;
} catch (Exception e) {
throw die(e.getMessage(), e);
} finally {
sysManager.stop();
dbManager.stop();
}
}
private boolean list() {
for (IndexDefinition<?, ?, ?> def : indexDefs) {
System.out.format("%s\n", def.getName());
}
return true;
}
private boolean reindex() {
boolean ok = true;
for (IndexDefinition<?, ?, ?> def : indexDefs) {
if (indices.isEmpty() || indices.contains(def.getName())) {
ok &= reindex(def);
}
}
return ok;
}
private void checkIndicesOption() throws Die {
if (indices.isEmpty()) {
return;
}
requireNonNull(indexDefs, "Called this method before injectMembers?");
Set<String> valid = indexDefs.stream().map(IndexDefinition::getName).sorted().collect(toSet());
Set<String> invalid = Sets.difference(Sets.newHashSet(indices), valid);
if (invalid.isEmpty()) {
return;
}
throw die(
"invalid index name(s): " + new TreeSet<>(invalid) + " available indices are: " + valid);
}
private Injector createSysInjector() {
ImmutableMap<String, Integer> versions =
changesVersion != null
? ImmutableMap.of(ChangeSchemaDefinitions.INSTANCE.getName(), changesVersion)
: ImmutableMap.of();
boolean replica = ReplicaUtil.isReplica(globalConfig);
List<Module> modules = new ArrayList<>();
modules.add(new WorkQueueModule());
Module indexModule;
IndexType indexType = IndexModule.getIndexType(dbInjector);
if (indexType.isLucene()) {
indexModule =
LuceneIndexModule.singleVersionWithExplicitVersions(
versions, threads, replica, AutoFlush.DISABLED);
} else if (indexType.isFake()) {
// Use Reflection so that we can omit the fake index binary in production code. Test code does
// compile the component in.
try {
Class<?> clazz = Class.forName("com.google.gerrit.index.testing.FakeIndexModule");
Method m =
clazz.getMethod(
"singleVersionWithExplicitVersions", ImmutableMap.class, int.class, boolean.class);
indexModule = (Module) m.invoke(null, versions, threads, replica);
} catch (NoSuchMethodException
| ClassNotFoundException
| IllegalAccessException
| InvocationTargetException e) {
throw new IllegalStateException("can't create index", e);
}
} else {
throw new IllegalStateException("unsupported index.type = " + indexType);
}
modules.add(indexModule);
modules.add(
new AbstractModule() {
@Override
protected void configure() {
super.configure();
OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class)
.setBinding()
.toInstance(IsFirstInsertForEntry.YES);
OptionalBinder.newOptionalBinder(binder(), BuildBloomFilter.class)
.setBinding()
.toInstance(buildBloomFilter ? BuildBloomFilter.TRUE : BuildBloomFilter.FALSE);
}
});
modules.add(new BatchProgramModule(dbInjector));
modules.add(
new FactoryModule() {
@Override
protected void configure() {
factory(ChangeResource.Factory.class);
}
});
modules.add(new AccountNoteDbWriteStorageModule());
modules.add(new AccountNoteDbReadStorageModule());
modules.add(new RepoSequenceModule());
modules.add(new NoteDbDraftCommentsModule());
modules.add(new NoteDbStarredChangesModule());
return dbInjector.createChildInjector(
ModuleOverloader.override(
modules, LibModuleLoader.loadReindexModules(cfgInjector, versions, threads, replica)));
}
private void overrideConfig() {
// Disable auto-commit for speed; committing will happen at the end of the process.
if (IndexModule.getIndexType(dbInjector).isLucene()) {
globalConfig.setLong("index", "changes_open", "commitWithin", -1);
globalConfig.setLong("index", "changes_closed", "commitWithin", -1);
}
// Disable change cache.
globalConfig.setLong("cache", "changes", "maximumWeight", 0);
// Disable auto-reindexing if stale, since there are no concurrent writes to race with.
globalConfig.setBoolean("index", null, "autoReindexIfStale", false);
}
private <K, V, I extends Index<K, V>> boolean reindex(IndexDefinition<K, V, I> def) {
I index = def.getIndexCollection().getSearchIndex();
requireNonNull(
index, () -> String.format("no active search index configured for %s", def.getName()));
index.markReady(false);
boolean reuseExistingDocuments =
reuseExistingDocumentsOption != null
? reuseExistingDocumentsOption
: globalConfig.getBoolean("index", null, "reuseExistingDocuments", false);
if (!reuseExistingDocuments) {
index.deleteAll();
}
SiteIndexer<K, V, I> siteIndexer = def.getSiteIndexer(reuseExistingDocuments);
siteIndexer.setProgressOut(System.err);
siteIndexer.setVerboseOut(verbose ? System.out : NullOutputStream.INSTANCE);
SiteIndexer.Result result = siteIndexer.indexAll(index);
int n = result.doneCount() + result.failedCount();
double t = result.elapsed(TimeUnit.MILLISECONDS) / 1000d;
System.out.format(
"Reindexed %d documents in %s index in %.01fs (%.01f/s)\n", n, def.getName(), t, n / t);
if (result.success()) {
index.markReady(true);
}
System.out.format(
"Index %s in version %d is %sready\n",
def.getName(), index.getSchema().getVersion(), result.success() ? "" : "NOT ");
return result.success();
}
private void printCacheStats() {
try (Writer sw = new StringWriter()) {
sw.write("Cache Statistics at the end of reindexing\n");
new CacheDisplay(
sw,
StreamSupport.stream(cacheMap.spliterator(), false)
.map(e -> new CacheInfo(e.getExportName(), e.get()))
.collect(Collectors.toList()))
.displayCaches();
System.out.print(sw.toString());
} catch (Exception e) {
System.out.format("Error displaying the cache statistics\n" + e.getMessage());
}
}
}