blob: f96786d3d261027996abdad908945afde37f501d [file] [log] [blame]
// Copyright (C) 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.googlesource.gerrit.modules.cache.chroniclemap;
import static com.googlesource.gerrit.modules.cache.chroniclemap.H2CacheCommand.H2_SUFFIX;
import static com.googlesource.gerrit.modules.cache.chroniclemap.H2CacheCommand.getStats;
import static com.googlesource.gerrit.modules.cache.chroniclemap.H2CacheCommand.jdbcUrl;
import static com.googlesource.gerrit.modules.cache.chroniclemap.HttpServletOps.checkAcceptHeader;
import static com.googlesource.gerrit.modules.cache.chroniclemap.HttpServletOps.setResponse;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.CachedProjectConfig;
import com.google.gerrit.extensions.auth.oauth.OAuthToken;
import com.google.gerrit.extensions.client.ChangeKind;
import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.httpd.WebSessionManager;
import com.google.gerrit.server.account.CachedAccountDetails;
import com.google.gerrit.server.cache.PersistentCacheDef;
import com.google.gerrit.server.cache.proto.Cache;
import com.google.gerrit.server.change.ChangeKindCacheImpl;
import com.google.gerrit.server.change.MergeabilityCacheImpl;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.TagSetHolder;
import com.google.gerrit.server.notedb.ChangeNotesCache;
import com.google.gerrit.server.notedb.ChangeNotesState;
import com.google.gerrit.server.patch.DiffSummary;
import com.google.gerrit.server.patch.DiffSummaryKey;
import com.google.gerrit.server.patch.IntraLineDiff;
import com.google.gerrit.server.patch.IntraLineDiffKey;
import com.google.gerrit.server.patch.PatchList;
import com.google.gerrit.server.patch.PatchListKey;
import com.google.gerrit.server.query.change.ConflictKey;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.jgit.lib.Config;
import org.h2.Driver;
@Singleton
public class H2MigrationServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
protected static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final ChronicleMapCacheConfig.Factory configFactory;
private final SitePaths site;
private final Config gerritConfig;
private final AdministerCachePermission adminCachePermission;
public static int DEFAULT_SIZE_MULTIPLIER = 3;
public static int DEFAULT_MAX_BLOAT_FACTOR = 3;
public static final String MAX_BLOAT_FACTOR_PARAM = "max-bloat-factor";
public static final String SIZE_MULTIPLIER_PARAM = "size-multiplier";
private final Set<PersistentCacheDef<?, ?>> persistentCacheDefs;
@Inject
H2MigrationServlet(
@GerritServerConfig Config cfg,
SitePaths site,
ChronicleMapCacheConfig.Factory configFactory,
AdministerCachePermission permissionBackend,
@Named("web_sessions") PersistentCacheDef<String, WebSessionManager.Val> webSessionsCacheDef,
@Named("accounts")
PersistentCacheDef<CachedAccountDetails.Key, CachedAccountDetails> accountsCacheDef,
@Named("oauth_tokens") PersistentCacheDef<Account.Id, OAuthToken> oauthTokenDef,
@Named("change_kind")
PersistentCacheDef<ChangeKindCacheImpl.Key, ChangeKind> changeKindCacheDef,
@Named("mergeability")
PersistentCacheDef<MergeabilityCacheImpl.EntryKey, Boolean> mergeabilityCacheDef,
@Named("pure_revert")
PersistentCacheDef<Cache.PureRevertKeyProto, Boolean> pureRevertCacheDef,
@Named("git_tags") PersistentCacheDef<String, TagSetHolder> gitTagsCacheDef,
@Named("change_notes")
PersistentCacheDef<ChangeNotesCache.Key, ChangeNotesState> changeNotesCacheDef,
@Named("diff") PersistentCacheDef<PatchListKey, PatchList> diffCacheDef,
@Named("diff_intraline")
PersistentCacheDef<IntraLineDiffKey, IntraLineDiff> diffIntraLineCacheDef,
@Named("diff_summary") PersistentCacheDef<DiffSummaryKey, DiffSummary> diffSummaryCacheDef,
@Named("persisted_projects")
PersistentCacheDef<Cache.ProjectCacheKeyProto, CachedProjectConfig>
persistedProjectsCacheDef,
@Named("conflicts") PersistentCacheDef<ConflictKey, Boolean> conflictsCacheDef) {
this.configFactory = configFactory;
this.site = site;
this.gerritConfig = cfg;
this.adminCachePermission = permissionBackend;
this.persistentCacheDefs =
Stream.of(
webSessionsCacheDef,
accountsCacheDef,
oauthTokenDef,
changeKindCacheDef,
mergeabilityCacheDef,
pureRevertCacheDef,
gitTagsCacheDef,
changeNotesCacheDef,
diffCacheDef,
diffIntraLineCacheDef,
diffSummaryCacheDef,
persistedProjectsCacheDef,
conflictsCacheDef)
.collect(Collectors.toSet());
}
@Override
protected void doPut(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
if (!checkAcceptHeader(req, rsp)) {
return;
}
if (!adminCachePermission.isCurrentUserAllowed()) {
setResponse(rsp, HttpServletResponse.SC_FORBIDDEN, "not permitted to administer caches");
return;
}
Optional<Path> cacheDir = getCacheDir();
int maxBloatFactor =
Optional.ofNullable(req.getParameter(MAX_BLOAT_FACTOR_PARAM))
.map(Integer::parseInt)
.orElse(DEFAULT_MAX_BLOAT_FACTOR);
int sizeMultiplier =
Optional.ofNullable(req.getParameter(SIZE_MULTIPLIER_PARAM))
.map(Integer::parseInt)
.orElse(DEFAULT_SIZE_MULTIPLIER);
if (!cacheDir.isPresent()) {
setResponse(
rsp,
HttpServletResponse.SC_BAD_REQUEST,
"Cannot run migration, cache directory is not configured");
return;
}
logger.atInfo().log("Migrating H2 caches to Chronicle-Map...");
logger.atInfo().log("* Size multiplier: " + sizeMultiplier);
logger.atInfo().log("* Max Bloat Factor: " + maxBloatFactor);
Config outputChronicleMapConfig = new Config();
try {
for (PersistentCacheDef<?, ?> in : persistentCacheDefs) {
Optional<Path> h2CacheFile = getH2CacheFile(cacheDir.get(), in.name());
Optional<ChronicleMapCacheConfig> chronicleMapConfig;
if (h2CacheFile.isPresent()) {
if (hasFullPersistentCacheConfiguration(in)) {
if (sizeMultiplier != DEFAULT_SIZE_MULTIPLIER) {
logger.atWarning().log(
"Size multiplier = %d ignored because of existing configuration found",
sizeMultiplier);
}
if (maxBloatFactor != DEFAULT_MAX_BLOAT_FACTOR) {
logger.atWarning().log(
"Max Bloat Factor = %d ignored because of existing configuration found",
maxBloatFactor);
}
File cacheFile =
ChronicleMapCacheFactory.fileName(cacheDir.get(), in.name(), in.version());
chronicleMapConfig =
Optional.of(
configFactory.create(
in.name(), cacheFile, in.expireAfterWrite(), in.refreshAfterWrite()));
} else {
if (hasPartialPersistentCacheConfiguration(in)) {
logger.atWarning().log(
"Existing configuration for cache %s found gerrit.config and will be ignored because incomplete",
in.name());
}
chronicleMapConfig =
optionalOf(getStats(h2CacheFile.get()))
.map(
(stats) ->
makeChronicleMapConfig(
configFactory,
cacheDir.get(),
in,
stats,
sizeMultiplier,
maxBloatFactor));
}
if (chronicleMapConfig.isPresent()) {
ChronicleMapCacheConfig cacheConfig = chronicleMapConfig.get();
ChronicleMapCacheImpl<?, ?> chronicleMapCache =
new ChronicleMapCacheImpl<>(in, cacheConfig);
doMigrate(h2CacheFile.get(), in, chronicleMapCache);
chronicleMapCache.close();
copyExistingCacheSettingsToConfig(outputChronicleMapConfig, cacheConfig);
}
}
}
} catch (Exception e) {
logger.atSevere().withCause(e).log("H2 to chronicle-map migration failed");
setResponse(rsp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
}
logger.atInfo().log("Migration completed");
setResponse(rsp, HttpServletResponse.SC_OK, outputChronicleMapConfig.toText());
}
private Optional<H2AggregateData> optionalOf(H2AggregateData stats) {
if (stats.isEmpty()) {
return Optional.empty();
}
return Optional.of(stats);
}
private boolean hasFullPersistentCacheConfiguration(PersistentCacheDef<?, ?> in) {
return gerritConfig.getLong("cache", in.name(), "avgKeySize", 0L) > 0
&& gerritConfig.getLong("cache", in.name(), "avgValueSize", 0L) > 0
&& gerritConfig.getLong("cache", in.name(), "maxEntries", 0L) > 0
&& gerritConfig.getInt("cache", in.name(), "maxBloatFactor", 0) > 0;
}
private boolean hasPartialPersistentCacheConfiguration(PersistentCacheDef<?, ?> in) {
return gerritConfig.getLong("cache", in.name(), "avgKeySize", 0L) > 0
|| gerritConfig.getLong("cache", in.name(), "avgValueSize", 0L) > 0
|| gerritConfig.getLong("cache", in.name(), "maxEntries", 0L) > 0
|| gerritConfig.getInt("cache", in.name(), "maxBloatFactor", 0) > 0;
}
protected Optional<Path> getCacheDir() throws IOException {
String name = gerritConfig.getString("cache", null, "directory");
if (name == null) {
return Optional.empty();
}
Path loc = site.resolve(name);
if (!Files.exists(loc)) {
throw new IOException(
String.format("disk cache is configured but doesn't exist: %s", loc.toAbsolutePath()));
}
if (!Files.isReadable(loc)) {
throw new IOException(String.format("Can't read from disk cache: %s", loc.toAbsolutePath()));
}
logger.atFine().log("Enabling disk cache %s", loc.toAbsolutePath());
return Optional.of(loc);
}
private Optional<Path> getH2CacheFile(Path cacheDir, String name) {
Path h2CacheFile = cacheDir.resolve(String.format("%s.%s", name, H2_SUFFIX));
if (Files.exists(h2CacheFile)) {
return Optional.of(h2CacheFile);
}
return Optional.empty();
}
protected static ChronicleMapCacheConfig makeChronicleMapConfig(
ChronicleMapCacheConfig.Factory configFactory,
Path cacheDir,
PersistentCacheDef<?, ?> in,
H2AggregateData stats,
int sizeMultiplier,
int maxBloatFactor) {
return configFactory.createWithValues(
in.configKey(),
ChronicleMapCacheFactory.fileName(cacheDir, in.name(), in.version()),
in.expireAfterWrite(),
in.refreshAfterWrite(),
stats.size() * sizeMultiplier,
stats.avgKeySize(),
stats.avgValueSize(),
maxBloatFactor);
}
private void doMigrate(
Path h2File, PersistentCacheDef<?, ?> in, ChronicleMapCacheImpl<?, ?> chronicleMapCache)
throws RestApiException {
String url = jdbcUrl(h2File);
try (Connection conn = Driver.load().connect(url, null)) {
PreparedStatement preparedStatement =
conn.prepareStatement("SELECT k, v, created FROM data WHERE version=?");
preparedStatement.setInt(1, in.version());
try (ResultSet r = preparedStatement.executeQuery()) {
while (r.next()) {
Object key =
isStringType(in.keyType())
? r.getString(1)
: in.keySerializer().deserialize(r.getBytes(1));
Object value =
isStringType(in.valueType())
? r.getString(2)
: in.valueSerializer().deserialize(r.getBytes(2));
Timestamp created = r.getTimestamp(3);
chronicleMapCache.putUnchecked(key, value, created);
}
}
} catch (Exception e) {
String message = String.format("FATAL: error migrating %s H2 cache", in.name());
logger.atSevere().withCause(e).log(message);
throw RestApiException.wrap(message, e);
}
}
private boolean isStringType(TypeLiteral<?> typeLiteral) {
return typeLiteral.getRawType().getSimpleName().equals("String");
}
private static void copyExistingCacheSettingsToConfig(
Config outputConfig, ChronicleMapCacheConfig cacheConfig) {
String cacheName = cacheConfig.getConfigKey();
outputConfig.setLong("cache", cacheName, "avgKeySize", cacheConfig.getAverageKeySize());
outputConfig.setLong("cache", cacheName, "avgValueSize", cacheConfig.getAverageValueSize());
outputConfig.setLong("cache", cacheName, "maxEntries", cacheConfig.getMaxEntries());
outputConfig.setLong("cache", cacheName, "maxBloatFactor", cacheConfig.getMaxBloatFactor());
}
}