Merge branch 'stable-3.7' into stable-3.8
* stable-3.7:
Revise the "init project" endpoint to manage project configuration
Extract the ser/des of HTTP payloads to utility class
Revise the "init project" client to accommodate the configuration
Improve log message when event is fired
Add JGit client test for unset mirror
Add mirror replication option for CGit client
Change-Id: I3b54519a94f3f20abe00cc8c7acfa5546015a756
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index 1ba47de..aafe12f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -219,7 +219,10 @@
private void fire(ReferenceUpdatedEvent event, ReplicationState state) {
if (!running) {
stateLog.warn(
- "Replication plugin did not finish startup before event, event replication is postponed",
+ String.format(
+ "Replication plugin did not finish startup before event, event replication is postponed"
+ + " for event %s",
+ event),
state);
beforeStartupEventsQueue.add(event);
@@ -518,7 +521,20 @@
private HttpResult initProject(
Project.NameKey project, URIish uri, FetchApiClient fetchClient, HttpResult result)
throws IOException, ClientProtocolException {
- HttpResult initProjectResult = fetchClient.initProject(project, uri);
+ RevisionData refsMetaConfigRevisionData =
+ revReaderProvider
+ .get()
+ .read(project, null, RefNames.REFS_CONFIG, 0)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ String.format(
+ "Project %s does not have %s", project, RefNames.REFS_CONFIG)));
+
+ List<RevisionData> refsMetaConfigDataList =
+ fetchWholeMetaHistory(project, RefNames.REFS_CONFIG, refsMetaConfigRevisionData);
+ HttpResult initProjectResult =
+ fetchClient.initProject(project, uri, System.currentTimeMillis(), refsMetaConfigDataList);
if (initProjectResult.isSuccessful()) {
result = fetchClient.callFetch(project, FetchOne.ALL_REFS, uri);
} else {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
index 8711379..e7289ac 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -14,25 +14,39 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
+import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.checkAcceptHeader;
import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.setResponse;
+import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+import static javax.servlet.http.HttpServletResponse.SC_CONFLICT;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import com.google.common.base.Strings;
import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.Url;
import com.google.gerrit.index.project.ProjectIndexer;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.permissions.GlobalPermission;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
+import com.google.gerrit.server.project.ProjectCache;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.LocalFS;
import com.googlesource.gerrit.plugins.replication.pull.GerritConfigOps;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
+import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
+import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
+import com.googlesource.gerrit.plugins.replication.pull.api.util.PayloadSerDes;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
import java.util.Optional;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@@ -51,17 +65,23 @@
private final Provider<CurrentUser> userProvider;
private final PermissionBackend permissionBackend;
private final ProjectIndexer projectIndexer;
+ private final ApplyObjectCommand applyObjectCommand;
+ private final ProjectCache projectCache;
@Inject
ProjectInitializationAction(
GerritConfigOps gerritConfigOps,
Provider<CurrentUser> userProvider,
PermissionBackend permissionBackend,
- ProjectIndexer projectIndexer) {
+ ProjectIndexer projectIndexer,
+ ApplyObjectCommand applyObjectCommand,
+ ProjectCache projectCache) {
this.gerritConfigOps = gerritConfigOps;
this.userProvider = userProvider;
this.permissionBackend = permissionBackend;
this.projectIndexer = projectIndexer;
+ this.applyObjectCommand = applyObjectCommand;
+ this.projectCache = projectCache;
}
@Override
@@ -73,47 +93,140 @@
return;
}
- String path = httpServletRequest.getRequestURI();
- String projectName = Url.decode(path.substring(path.lastIndexOf('/') + 1));
+ String gitRepositoryName = getGitRepositoryName(httpServletRequest);
try {
- if (initProject(projectName)) {
+ boolean initProjectStatus;
+ String contentType = httpServletRequest.getContentType();
+ if (checkContentType(contentType, MediaType.JSON_UTF_8)) {
+ // init project request includes project configuration in JSON format.
+ initProjectStatus = initProjectWithConfiguration(httpServletRequest, gitRepositoryName);
+ } else if (checkContentType(contentType, MediaType.PLAIN_TEXT_UTF_8)) {
+ // init project request does not include project configuration.
+ initProjectStatus = initProject(gitRepositoryName);
+ } else {
+ setResponse(
+ httpServletResponse,
+ SC_BAD_REQUEST,
+ String.format(
+ "Invalid Content Type. Only %s or %s is supported.",
+ MediaType.JSON_UTF_8.toString(), MediaType.PLAIN_TEXT_UTF_8.toString()));
+ return;
+ }
+
+ if (initProjectStatus) {
setResponse(
httpServletResponse,
HttpServletResponse.SC_CREATED,
- "Project " + projectName + " initialized");
+ "Project " + gitRepositoryName + " initialized");
return;
}
- } catch (AuthException | PermissionBackendException e) {
+
setResponse(
httpServletResponse,
- HttpServletResponse.SC_FORBIDDEN,
- "User not authorized to create project " + projectName);
- return;
+ HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
+ "Cannot initialize project " + gitRepositoryName);
+ } catch (BadRequestException | IllegalArgumentException e) {
+ logExceptionAndUpdateResponse(httpServletResponse, e, SC_BAD_REQUEST, gitRepositoryName);
+ } catch (RefUpdateException | MissingParentObjectException e) {
+ logExceptionAndUpdateResponse(httpServletResponse, e, SC_CONFLICT, gitRepositoryName);
+ } catch (AuthException | PermissionBackendException e) {
+ logExceptionAndUpdateResponse(httpServletResponse, e, SC_FORBIDDEN, gitRepositoryName);
}
-
- setResponse(
- httpServletResponse,
- HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
- "Cannot initialize project " + projectName);
}
- public boolean initProject(String projectName) throws AuthException, PermissionBackendException {
+ public boolean initProject(String gitRepositoryName)
+ throws AuthException, PermissionBackendException {
+ if (initProject(gitRepositoryName, true)) {
+ repLog.info("Init project API from {}", gitRepositoryName);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean initProjectWithConfiguration(
+ HttpServletRequest httpServletRequest, String gitRepositoryName)
+ throws AuthException, PermissionBackendException, IOException, BadRequestException,
+ MissingParentObjectException, RefUpdateException {
+
+ RevisionsInput input = PayloadSerDes.parseRevisionsInput(httpServletRequest);
+ validateInput(input);
+ if (!initProject(gitRepositoryName, false)) {
+ return false;
+ }
+
+ String projectName = gitRepositoryName.replace(".git", "");
+ applyObjectCommand.applyObjects(
+ Project.nameKey(projectName),
+ input.getRefName(),
+ input.getRevisionsData(),
+ input.getLabel(),
+ input.getEventCreatedOn());
+ projectCache.onCreateProject(Project.nameKey(projectName));
+ repLog.info(
+ "Init project API from {} for {}:{} - {}",
+ input.getLabel(),
+ projectName,
+ input.getRefName(),
+ Arrays.toString(input.getRevisionsData()));
+ return true;
+ }
+
+ private boolean initProject(String gitRepositoryName, boolean needsProjectReindexing)
+ throws AuthException, PermissionBackendException {
// When triggered internally(for example by consuming stream events) user is not provided
// and internal user is returned. Project creation should be always allowed for internal user.
if (!userProvider.get().isInternalUser()) {
permissionBackend.user(userProvider.get()).check(GlobalPermission.CREATE_PROJECT);
}
- Optional<URIish> maybeUri = gerritConfigOps.getGitRepositoryURI(projectName);
+ Optional<URIish> maybeUri = gerritConfigOps.getGitRepositoryURI(gitRepositoryName);
if (!maybeUri.isPresent()) {
- logger.atSevere().log("Cannot initialize project '%s'", projectName);
+ logger.atSevere().log("Cannot initialize project '%s'", gitRepositoryName);
return false;
}
LocalFS localFS = new LocalFS(maybeUri.get());
- Project.NameKey projectNameKey = Project.NameKey.parse(projectName);
+ Project.NameKey projectNameKey = Project.NameKey.parse(gitRepositoryName);
if (localFS.createProject(projectNameKey, RefNames.HEAD)) {
- projectIndexer.index(projectNameKey);
+ if (needsProjectReindexing) {
+ projectIndexer.index(projectNameKey);
+ }
return true;
}
return false;
}
+
+ private void validateInput(RevisionsInput input) {
+
+ if (Strings.isNullOrEmpty(input.getLabel())) {
+ throw new IllegalArgumentException("Source label cannot be null or empty");
+ }
+
+ if (!Objects.equals(input.getRefName(), RefNames.REFS_CONFIG)) {
+ throw new IllegalArgumentException(
+ String.format("Ref-update refname should be %s", RefNames.REFS_CONFIG));
+ }
+ input.validate();
+ }
+
+ private String getGitRepositoryName(HttpServletRequest httpServletRequest) {
+ String path = httpServletRequest.getRequestURI();
+ return Url.decode(path.substring(path.lastIndexOf('/') + 1));
+ }
+
+ private void logExceptionAndUpdateResponse(
+ HttpServletResponse httpServletResponse,
+ Exception e,
+ int statusCode,
+ String gitRepositoryName)
+ throws IOException {
+ repLog.error("Init Project API FAILED for {}", gitRepositoryName, e);
+ setResponse(httpServletResponse, statusCode, e.getMessage());
+ }
+
+ private boolean checkContentType(String contentType, MediaType mediaType) {
+ try {
+ return MediaType.parse(contentType).is(mediaType);
+ } catch (Exception e) {
+ return false;
+ }
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
index 6b903d8..f501e90 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -18,10 +18,8 @@
import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.checkAcceptHeader;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_CONFLICT;
-import static javax.servlet.http.HttpServletResponse.SC_CREATED;
import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
-import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
import com.google.common.flogger.FluentLogger;
@@ -38,28 +36,22 @@
import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
import com.google.gerrit.httpd.AllRequestFilter;
import com.google.gerrit.httpd.restapi.RestApiServlet;
-import com.google.gerrit.json.OutputFormat;
import com.google.gerrit.server.AnonymousUser;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectResource;
import com.google.gerrit.server.project.ProjectState;
-import com.google.gson.Gson;
import com.google.gson.JsonParseException;
-import com.google.gson.stream.JsonReader;
import com.google.gson.stream.MalformedJsonException;
import com.google.inject.Inject;
import com.google.inject.Provider;
-import com.google.inject.TypeLiteral;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.UnauthorizedAuthException;
-import java.io.BufferedReader;
-import java.io.EOFException;
+import com.googlesource.gerrit.plugins.replication.pull.api.util.PayloadSerDes;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
@@ -86,7 +78,6 @@
private UpdateHeadAction updateHEADAction;
private ProjectDeletionAction projectDeletionAction;
private ProjectCache projectCache;
- private Gson gson;
private String pluginName;
private final Provider<CurrentUser> currentUserProvider;
@@ -109,7 +100,6 @@
this.projectDeletionAction = projectDeletionAction;
this.projectCache = projectCache;
this.pluginName = pluginName;
- this.gson = OutputFormat.JSON.newGsonBuilder().create();
this.currentUserProvider = currentUserProvider;
}
@@ -126,13 +116,13 @@
try {
if (isFetchAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
- writeResponse(httpResponse, doFetch(httpRequest));
+ PayloadSerDes.writeResponse(httpResponse, doFetch(httpRequest));
} else if (isApplyObjectAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
- writeResponse(httpResponse, doApplyObject(httpRequest));
+ PayloadSerDes.writeResponse(httpResponse, doApplyObject(httpRequest));
} else if (isApplyObjectsAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
- writeResponse(httpResponse, doApplyObjects(httpRequest));
+ PayloadSerDes.writeResponse(httpResponse, doApplyObjects(httpRequest));
} else if (isInitProjectAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
if (!checkAcceptHeader(httpRequest, httpResponse)) {
@@ -141,10 +131,10 @@
doInitProject(httpRequest, httpResponse);
} else if (isUpdateHEADAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
- writeResponse(httpResponse, doUpdateHEAD(httpRequest));
+ PayloadSerDes.writeResponse(httpResponse, doUpdateHEAD(httpRequest));
} else if (isDeleteProjectAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
- writeResponse(httpResponse, doDeleteProject(httpRequest));
+ PayloadSerDes.writeResponse(httpResponse, doDeleteProject(httpRequest));
} else {
chain.doFilter(request, response);
}
@@ -199,7 +189,7 @@
@SuppressWarnings("unchecked")
private Response<String> doApplyObject(HttpServletRequest httpRequest)
throws RestApiException, IOException, PermissionBackendException {
- RevisionInput input = readJson(httpRequest, TypeLiteral.get(RevisionInput.class));
+ RevisionInput input = PayloadSerDes.parseRevisionInput(httpRequest);
IdString id = getProjectName(httpRequest).get();
return (Response<String>) applyObjectAction.apply(parseProjectResource(id), input);
@@ -208,7 +198,7 @@
@SuppressWarnings("unchecked")
private Response<String> doApplyObjects(HttpServletRequest httpRequest)
throws RestApiException, IOException, PermissionBackendException {
- RevisionsInput input = readJson(httpRequest, TypeLiteral.get(RevisionsInput.class));
+ RevisionsInput input = PayloadSerDes.parseRevisionsInput(httpRequest);
IdString id = getProjectName(httpRequest).get();
return (Response<String>) applyObjectsAction.apply(parseProjectResource(id), input);
@@ -216,7 +206,7 @@
@SuppressWarnings("unchecked")
private Response<String> doUpdateHEAD(HttpServletRequest httpRequest) throws Exception {
- HeadInput input = readJson(httpRequest, TypeLiteral.get(HeadInput.class));
+ HeadInput input = PayloadSerDes.parseHeadInput(httpRequest);
IdString id = getProjectName(httpRequest).get();
return (Response<String>) updateHEADAction.apply(parseProjectResource(id), input);
@@ -233,7 +223,7 @@
@SuppressWarnings("unchecked")
private Response<Map<String, Object>> doFetch(HttpServletRequest httpRequest)
throws IOException, RestApiException, PermissionBackendException {
- Input input = readJson(httpRequest, TypeLiteral.get(Input.class));
+ Input input = PayloadSerDes.parseInput(httpRequest);
IdString id = getProjectName(httpRequest).get();
return (Response<Map<String, Object>>) fetchAction.apply(parseProjectResource(id), input);
@@ -247,50 +237,6 @@
return new ProjectResource(project.get(), currentUserProvider.get());
}
- private <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response)
- throws IOException {
- String responseJson = gson.toJson(response);
- if (response.statusCode() == SC_OK || response.statusCode() == SC_CREATED) {
-
- httpResponse.setContentType("application/json");
- httpResponse.setStatus(response.statusCode());
- PrintWriter writer = httpResponse.getWriter();
- writer.print(new String(RestApiServlet.JSON_MAGIC));
- writer.print(responseJson);
- } else {
- httpResponse.sendError(response.statusCode(), responseJson);
- }
- }
-
- private <T> T readJson(HttpServletRequest httpRequest, TypeLiteral<T> typeLiteral)
- throws IOException, BadRequestException {
-
- try (BufferedReader br = httpRequest.getReader();
- JsonReader json = new JsonReader(br)) {
- try {
- json.setLenient(true);
-
- try {
- json.peek();
- } catch (EOFException e) {
- throw new BadRequestException("Expected JSON object", e);
- }
-
- return gson.fromJson(json, typeLiteral.getType());
- } finally {
- try {
- // Reader.close won't consume the rest of the input. Explicitly consume the request
- // body.
- br.skip(Long.MAX_VALUE);
- } catch (Exception e) {
- // ignore, e.g. trying to consume the rest of the input may fail if the request was
- // cancelled
- logger.atFine().withCause(e).log("Exception during the parsing of the request json");
- }
- }
- }
- }
-
/**
* Return project name from request URI. Request URI format:
* /a/projects/<project_name>/pull-replication~apply-object
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java
new file mode 100644
index 0000000..414af37
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java
@@ -0,0 +1,106 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api.util;
+
+import static javax.servlet.http.HttpServletResponse.SC_CREATED;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.extensions.api.projects.HeadInput;
+import com.google.gerrit.extensions.restapi.BadRequestException;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.httpd.restapi.RestApiServlet;
+import com.google.gerrit.json.OutputFormat;
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonReader;
+import com.google.inject.TypeLiteral;
+import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
+import java.io.BufferedReader;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.PrintWriter;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+public class PayloadSerDes {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final Gson gson = OutputFormat.JSON.newGsonBuilder().create();
+
+ public static RevisionInput parseRevisionInput(HttpServletRequest httpRequest)
+ throws BadRequestException, IOException {
+ return parse(httpRequest, TypeLiteral.get(RevisionInput.class));
+ }
+
+ public static RevisionsInput parseRevisionsInput(HttpServletRequest httpRequest)
+ throws BadRequestException, IOException {
+ return parse(httpRequest, TypeLiteral.get(RevisionsInput.class));
+ }
+
+ public static HeadInput parseHeadInput(HttpServletRequest httpRequest)
+ throws BadRequestException, IOException {
+ return parse(httpRequest, TypeLiteral.get(HeadInput.class));
+ }
+
+ public static FetchAction.Input parseInput(HttpServletRequest httpRequest)
+ throws BadRequestException, IOException {
+ return parse(httpRequest, TypeLiteral.get(FetchAction.Input.class));
+ }
+
+ public static <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response)
+ throws IOException {
+ String responseJson = gson.toJson(response);
+ if (response.statusCode() == SC_OK || response.statusCode() == SC_CREATED) {
+
+ httpResponse.setContentType("application/json");
+ httpResponse.setStatus(response.statusCode());
+ PrintWriter writer = httpResponse.getWriter();
+ writer.print(new String(RestApiServlet.JSON_MAGIC));
+ writer.print(responseJson);
+ } else {
+ httpResponse.sendError(response.statusCode(), responseJson);
+ }
+ }
+
+ private static <T> T parse(HttpServletRequest httpRequest, TypeLiteral<T> typeLiteral)
+ throws IOException, BadRequestException {
+
+ try (BufferedReader br = httpRequest.getReader();
+ JsonReader json = new JsonReader(br)) {
+ try {
+ json.setLenient(true);
+
+ try {
+ json.peek();
+ } catch (EOFException e) {
+ throw new BadRequestException("Expected JSON object", e);
+ }
+
+ return gson.fromJson(json, typeLiteral.getType());
+ } finally {
+ try {
+ // Reader.close won't consume the rest of the input. Explicitly consume the request
+ // body.
+ br.skip(Long.MAX_VALUE);
+ } catch (Exception e) {
+ // ignore, e.g. trying to consume the rest of the input may fail if the request was
+ // cancelled
+ logger.atFine().withCause(e).log("Exception during the parsing of the request json");
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
index 1991260..f7ed4cb 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -40,7 +40,22 @@
return callFetch(project, refName, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis()));
}
- HttpResult initProject(Project.NameKey project, URIish uri) throws IOException;
+ /**
+ * Replicates the creation of a project, including the configuration stored in refs/meta/config.
+ *
+ * @param project The unique name of the project.
+ * @param uri The destination URI where the project and its configuration should be replicated to.
+ * @param eventCreatedOn The timestamp indicating when the init project event occurred.
+ * @param refsMetaConfigRevisionData A history of revisions for the refs/meta/config ref.
+ * @return An HTTP result object providing information about the replication process.
+ * @throws IOException If an I/O error occurs during the replication.
+ */
+ HttpResult initProject(
+ Project.NameKey project,
+ URIish uri,
+ long eventCreatedOn,
+ List<RevisionData> refsMetaConfigRevisionData)
+ throws IOException;
HttpResult deleteProject(Project.NameKey project, URIish apiUri) throws IOException;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
index b606ba8..7607e4b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -23,6 +23,7 @@
import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
+import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.annotations.PluginName;
import com.google.gerrit.extensions.restapi.Url;
import com.google.gerrit.server.config.GerritInstanceId;
@@ -130,14 +131,29 @@
}
/* (non-Javadoc)
- * @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#initProject(com.google.gerrit.entities.Project.NameKey, org.eclipse.jgit.transport.URIish)
+ * @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#initProject(com.google.gerrit.entities.Project.NameKey, org.eclipse.jgit.transport.URIish, long, java.util.List<com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData>)
*/
@Override
- public HttpResult initProject(Project.NameKey project, URIish uri) throws IOException {
+ public HttpResult initProject(
+ NameKey project,
+ URIish uri,
+ long eventCreatedOn,
+ List<RevisionData> refsMetaConfigRevisionData)
+ throws IOException {
String url = formatInitProjectUrl(uri.toString(), project);
+
+ RevisionData[] inputData = new RevisionData[refsMetaConfigRevisionData.size()];
+ RevisionsInput input =
+ new RevisionsInput(
+ instanceId,
+ RefNames.REFS_CONFIG,
+ eventCreatedOn,
+ refsMetaConfigRevisionData.toArray(inputData));
+
HttpPut put = new HttpPut(url);
+ put.setEntity(new StringEntity(GSON.toJson(input)));
put.addHeader(new BasicHeader("Accept", MediaType.ANY_TEXT_TYPE.toString()));
- put.addHeader(new BasicHeader("Content-Type", MediaType.PLAIN_TEXT_UTF_8.toString()));
+ put.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString()));
return executeRequest(put, bearerTokenProvider.get(), uri);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java
index d3e45da..43ec9bf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java
@@ -44,6 +44,7 @@
private URIish uri;
private int timeout;
private final String taskIdHex;
+ private final boolean isMirror;
@Inject
public CGitFetch(
@@ -56,12 +57,17 @@
this.taskIdHex = taskIdHex;
this.uri = appendCredentials(uri, cpFactory.create(config.getRemoteConfig().getName()));
this.timeout = config.getRemoteConfig().getTimeout();
+ this.isMirror = config.getRemoteConfig().isMirror();
}
@Override
public List<RefUpdateState> fetch(List<RefSpec> refsSpec) throws IOException {
List<String> refs = refsSpec.stream().map(s -> s.toString()).collect(Collectors.toList());
- List<String> command = Lists.newArrayList("git", "fetch", uri.toPrivateASCIIString());
+ List<String> command = Lists.newArrayList("git", "fetch");
+ if (isMirror) {
+ command.add("--prune");
+ }
+ command.add(uri.toPrivateASCIIString());
command.addAll(refs);
ProcessBuilder pb = new ProcessBuilder().command(command).directory(localProjectDirectory);
repLog.info("[{}] Fetch references {} from {}", taskIdHex, refs, uri);
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index b7db2c0..61d62b4 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -526,8 +526,7 @@
remote.NAME.mirror
: If true, replication will remove local branches and tags that are
absent remotely or invisible to the replication (for example read access
-denied via `authGroup` option). Note that this option is currently
-implemented for the JGit client only.
+denied via `authGroup` option).
By default, false, do not remove remote branches or tags.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
index baacf20..3ca937c 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
@@ -15,6 +15,9 @@
package com.googlesource.gerrit.plugins.replication.pull;
import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.acceptance.GitUtil.deleteRef;
+import static com.google.gerrit.acceptance.GitUtil.pushHead;
+import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -22,35 +25,33 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
+import com.google.gerrit.acceptance.PushOneCommit;
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.SkipProjectClone;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Permission;
import com.google.gerrit.extensions.api.projects.BranchInput;
-import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.inject.Scopes;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
-import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
-import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
-import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
+import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.pull.fetch.BatchFetchClient;
import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
-import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
-import java.net.URISyntaxException;
import java.util.List;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.transport.PushResult;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.URIish;
+import org.junit.Before;
import org.junit.Test;
@SkipProjectClone
@@ -62,6 +63,16 @@
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
private static final String TEST_TASK_ID = "taskid";
+ @Inject private ProjectOperations projectOperations;
+
+ @Before
+ public void allowRefDeletion() {
+ projectOperations
+ .allProjectsForUpdate()
+ .add(allow(Permission.DELETE).ref("refs/*").group(adminGroupUuid()))
+ .update();
+ }
+
@Test
public void shouldFetchRef() throws Exception {
testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
@@ -224,29 +235,55 @@
}
}
+ @Test
+ public void shouldNotPruneRefsWhenMirrorIsUnset() throws Exception {
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+ String BRANCH_REF = Constants.R_HEADS + "anyBranch";
+ String TAG_REF = Constants.R_TAGS + "anyTag";
+
+ PushOneCommit.Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(BRANCH_REF);
+ branchPush.assertOkStatus();
+
+ PushResult tagPush = pushHead(testRepo, TAG_REF, false, false);
+ assertOkStatus(tagPush, TAG_REF);
+
+ try (Repository localRepo = repoManager.openRepository(project)) {
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ waitUntil(
+ () ->
+ checkedGetRef(localRepo, BRANCH_REF) != null
+ && checkedGetRef(localRepo, TAG_REF) != null);
+ assertThat(getRef(localRepo, BRANCH_REF)).isNotNull();
+ assertThat(getRef(localRepo, TAG_REF)).isNotNull();
+
+ PushResult deleteBranchResult = deleteRef(testRepo, BRANCH_REF);
+ assertOkStatus(deleteBranchResult, BRANCH_REF);
+
+ PushResult deleteTagResult = deleteRef(testRepo, TAG_REF);
+ assertOkStatus(deleteTagResult, TAG_REF);
+
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ waitUntil(
+ () ->
+ checkedGetRef(localRepo, BRANCH_REF) != null
+ && checkedGetRef(localRepo, TAG_REF) != null);
+ assertThat(getRef(localRepo, BRANCH_REF)).isNotNull();
+ assertThat(getRef(localRepo, TAG_REF)).isNotNull();
+ }
+ }
+
@SuppressWarnings("unused")
- private static class TestModule extends FactoryModule {
+ private static class TestModule extends FetchModule<CGitFetch> {
@Override
- protected void configure() {
+ Class<CGitFetch> clientClass() {
+ return CGitFetch.class;
+ }
+
+ @Override
+ Config cf() {
Config cf = new Config();
cf.setInt("remote", "test_config", "timeout", 0);
- try {
- RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config");
- SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf);
- bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
- bind(CredentialsFactory.class)
- .to(AutoReloadSecureCredentialsFactoryDecorator.class)
- .in(Scopes.SINGLETON);
-
- bind(SourceConfiguration.class).toInstance(sourceConfig);
- install(
- new FactoryModuleBuilder()
- .implement(Fetch.class, CGitFetch.class)
- .implement(Fetch.class, FetchClientImplementation.class, CGitFetch.class)
- .build(FetchFactory.class));
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
+ return cf;
}
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchWithMirrorIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchWithMirrorIT.java
new file mode 100644
index 0000000..8a3bae9
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchWithMirrorIT.java
@@ -0,0 +1,108 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.acceptance.GitUtil.deleteRef;
+import static com.google.gerrit.acceptance.GitUtil.pushHead;
+import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow;
+
+import com.google.gerrit.acceptance.PushOneCommit.Result;
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Permission;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetch;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.PushResult;
+import org.junit.Before;
+import org.junit.Test;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.CGitFetchWithMirrorIT$TestModule")
+public class CGitFetchWithMirrorIT extends FetchITBase {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+ private static final String TEST_TASK_ID = "taskid";
+
+ @Inject private ProjectOperations projectOperations;
+
+ @Before
+ public void allowRefDeletion() {
+ projectOperations
+ .allProjectsForUpdate()
+ .add(allow(Permission.DELETE).ref("refs/*").group(adminGroupUuid()))
+ .update();
+ }
+
+ @Test
+ public void shouldPruneRefsWhenMirrorIsTrue() throws Exception {
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+ String BRANCH_REF = Constants.R_HEADS + "anyBranch";
+ String TAG_REF = Constants.R_TAGS + "anyTag";
+
+ Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(BRANCH_REF);
+ branchPush.assertOkStatus();
+
+ PushResult tagPush = pushHead(testRepo, TAG_REF, false, false);
+ assertOkStatus(tagPush, TAG_REF);
+
+ try (Repository localRepo = repoManager.openRepository(project)) {
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ waitUntil(
+ () ->
+ checkedGetRef(localRepo, BRANCH_REF) != null
+ && checkedGetRef(localRepo, TAG_REF) != null);
+ assertThat(getRef(localRepo, BRANCH_REF)).isNotNull();
+ assertThat(getRef(localRepo, TAG_REF)).isNotNull();
+
+ PushResult deleteBranchResult = deleteRef(testRepo, BRANCH_REF);
+ assertOkStatus(deleteBranchResult, BRANCH_REF);
+
+ PushResult deleteTagResult = deleteRef(testRepo, TAG_REF);
+ assertOkStatus(deleteTagResult, TAG_REF);
+
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ waitUntil(
+ () ->
+ checkedGetRef(localRepo, BRANCH_REF) == null
+ && checkedGetRef(localRepo, TAG_REF) == null);
+ assertThat(getRef(localRepo, BRANCH_REF)).isNull();
+ assertThat(getRef(localRepo, TAG_REF)).isNull();
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static class TestModule extends FetchModule<CGitFetch> {
+ @Override
+ Class<CGitFetch> clientClass() {
+ return CGitFetch.class;
+ }
+
+ @Override
+ Config cf() {
+ Config cf = new Config();
+ cf.setInt("remote", "test_config", "timeout", 0);
+ cf.setBoolean("remote", "test_config", "mirror", true);
+ return cf;
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
index be9f902..3429983 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
@@ -14,18 +14,39 @@
package com.googlesource.gerrit.plugins.replication.pull;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+
+import com.google.common.collect.Lists;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.extensions.config.FactoryModule;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
+import com.google.inject.Scopes;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
+import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import java.io.IOException;
+import java.net.URISyntaxException;
import java.nio.file.Path;
import java.time.Duration;
import java.util.function.Supplier;
+import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.PushResult;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.RemoteRefUpdate;
+import org.eclipse.jgit.transport.URIish;
public abstract class FetchITBase extends LightweightPluginDaemonTest {
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
@@ -33,6 +54,7 @@
private static final int TEST_REPLICATION_DELAY = 60;
private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
+ private static final RefSpec ALL_REFS = new RefSpec("+refs/*:refs/*");
@Inject private SitePaths sitePaths;
@Inject private ProjectOperations projectOperations;
@@ -66,7 +88,49 @@
}
}
+ protected void fetchAllRefs(String taskId, Path remotePath, Repository localRepo)
+ throws URISyntaxException, IOException {
+ fetchFactory
+ .create(taskId, new URIish(remotePath.toString()), localRepo)
+ .fetch(Lists.newArrayList(ALL_REFS));
+ }
+
+ protected static void assertOkStatus(PushResult result, String ref) {
+ RemoteRefUpdate refUpdate = result.getRemoteUpdate(ref);
+ assertThat(refUpdate).isNotNull();
+ assertWithMessage(refUpdate.getMessage())
+ .that(refUpdate.getStatus())
+ .isEqualTo(RemoteRefUpdate.Status.OK);
+ }
+
Project.NameKey createTestProject(String name) {
return projectOperations.newProject().name(name).create();
}
+
+ protected abstract static class FetchModule<T extends Fetch> extends FactoryModule {
+ abstract Config cf();
+
+ abstract Class<T> clientClass();
+
+ @Override
+ protected void configure() {
+ try {
+ RemoteConfig remoteConfig = new RemoteConfig(cf(), "test_config");
+ SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf());
+ bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
+ bind(CredentialsFactory.class)
+ .to(AutoReloadSecureCredentialsFactoryDecorator.class)
+ .in(Scopes.SINGLETON);
+
+ bind(SourceConfiguration.class).toInstance(sourceConfig);
+ install(
+ new FactoryModuleBuilder()
+ .implement(Fetch.class, clientClass())
+ .implement(Fetch.class, FetchClientImplementation.class, clientClass())
+ .build(FetchFactory.class));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
index 77cf80d..b900d8a 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.replication.pull;
import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth.assertWithMessage;
import static com.google.gerrit.acceptance.GitUtil.deleteRef;
import static com.google.gerrit.acceptance.GitUtil.pushHead;
import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow;
@@ -27,31 +26,15 @@
import com.google.gerrit.acceptance.UseLocalDisk;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.entities.Permission;
-import com.google.gerrit.extensions.config.FactoryModule;
import com.google.inject.Inject;
-import com.google.inject.Scopes;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
-import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
-import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
-import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
-import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
-import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
-import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.List;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.PushResult;
import org.eclipse.jgit.transport.RefSpec;
-import org.eclipse.jgit.transport.RemoteConfig;
-import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.eclipse.jgit.transport.URIish;
import org.junit.Before;
import org.junit.Test;
@@ -64,7 +47,6 @@
public class JGitFetchIT extends FetchITBase {
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
private static final String TEST_TASK_ID = "taskid";
- private static final RefSpec ALL_REFS = new RefSpec("+refs/*:refs/*");
@Inject private ProjectOperations projectOperations;
@@ -89,12 +71,10 @@
}
@Test
- public void shouldPruneRefsWhenMirrorIsTrue() throws Exception {
+ public void shouldPruneRefsWhenMirrorIsUnset() throws Exception {
testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
- String branchName = "anyBranch";
- String branchRef = Constants.R_HEADS + branchName;
- String tagName = "anyTag";
- String tagRef = Constants.R_TAGS + tagName;
+ String branchRef = Constants.R_HEADS + "anyBranch";
+ String tagRef = Constants.R_TAGS + "anyTag";
PushOneCommit.Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(branchRef);
branchPush.assertOkStatus();
@@ -103,13 +83,8 @@
assertOkStatus(tagPush, tagRef);
try (Repository localRepo = repoManager.openRepository(project)) {
- List<RefUpdateState> fetchCreated = fetchAllRefs(localRepo);
- assertThat(fetchCreated.toString())
- .contains(new RefUpdateState(branchRef, RefUpdate.Result.NEW).toString());
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
assertThat(getRef(localRepo, branchRef)).isNotNull();
-
- assertThat(fetchCreated.toString())
- .contains(new RefUpdateState(tagRef, RefUpdate.Result.NEW).toString());
assertThat(getRef(localRepo, tagRef)).isNotNull();
PushResult deleteBranchResult = deleteRef(testRepo, branchRef);
@@ -118,55 +93,24 @@
PushResult deleteTagResult = deleteRef(testRepo, tagRef);
assertOkStatus(deleteTagResult, tagRef);
- List<RefUpdateState> fetchDeleted = fetchAllRefs(localRepo);
- assertThat(fetchDeleted.toString())
- .contains(new RefUpdateState(branchRef, RefUpdate.Result.FORCED).toString());
- assertThat(getRef(localRepo, branchRef)).isNull();
-
- assertThat(fetchDeleted.toString())
- .contains(new RefUpdateState(tagRef, RefUpdate.Result.FORCED).toString());
- assertThat(getRef(localRepo, tagRef)).isNull();
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ assertThat(getRef(localRepo, branchRef)).isNotNull();
+ assertThat(getRef(localRepo, tagRef)).isNotNull();
}
}
- private List<RefUpdateState> fetchAllRefs(Repository localRepo)
- throws URISyntaxException, IOException {
- Fetch fetch = fetchFactory.create(TEST_TASK_ID, new URIish(testRepoPath.toString()), localRepo);
- return fetch.fetch(Lists.newArrayList(ALL_REFS));
- }
-
- private static void assertOkStatus(PushResult result, String ref) {
- RemoteRefUpdate refUpdate = result.getRemoteUpdate(ref);
- assertThat(refUpdate).isNotNull();
- assertWithMessage(refUpdate.getMessage())
- .that(refUpdate.getStatus())
- .isEqualTo(RemoteRefUpdate.Status.OK);
- }
-
@SuppressWarnings("unused")
- private static class TestModule extends FactoryModule {
+ private static class TestModule extends FetchModule<JGitFetch> {
@Override
- protected void configure() {
+ Class<JGitFetch> clientClass() {
+ return JGitFetch.class;
+ }
+
+ @Override
+ Config cf() {
Config cf = new Config();
cf.setInt("remote", "test_config", "timeout", 0);
- cf.setBoolean("remote", "test_config", "mirror", true);
- try {
- RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config");
- SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf);
- bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
- bind(CredentialsFactory.class)
- .to(AutoReloadSecureCredentialsFactoryDecorator.class)
- .in(Scopes.SINGLETON);
-
- bind(SourceConfiguration.class).toInstance(sourceConfig);
- install(
- new FactoryModuleBuilder()
- .implement(Fetch.class, JGitFetch.class)
- .implement(Fetch.class, FetchClientImplementation.class, JGitFetch.class)
- .build(FetchFactory.class));
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
+ return cf;
}
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchWithMirrorIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchWithMirrorIT.java
new file mode 100644
index 0000000..2df700f
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchWithMirrorIT.java
@@ -0,0 +1,100 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.acceptance.GitUtil.deleteRef;
+import static com.google.gerrit.acceptance.GitUtil.pushHead;
+import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow;
+
+import com.google.gerrit.acceptance.PushOneCommit;
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Permission;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.PushResult;
+import org.junit.Before;
+import org.junit.Test;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.JGitFetchWithMirrorIT$TestModule")
+public class JGitFetchWithMirrorIT extends FetchITBase {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+ private static final String TEST_TASK_ID = "taskid";
+
+ @Inject private ProjectOperations projectOperations;
+
+ @Before
+ public void allowRefDeletion() {
+ projectOperations
+ .allProjectsForUpdate()
+ .add(allow(Permission.DELETE).ref("refs/*").group(adminGroupUuid()))
+ .update();
+ }
+
+ @Test
+ public void shouldPruneRefsWhenMirrorIsTrue() throws Exception {
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+ String branchRef = Constants.R_HEADS + "anyBranch";
+ String tagRef = Constants.R_TAGS + "anyTag";
+
+ PushOneCommit.Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(branchRef);
+ branchPush.assertOkStatus();
+
+ PushResult tagPush = pushHead(testRepo, tagRef, false, false);
+ assertOkStatus(tagPush, tagRef);
+
+ try (Repository localRepo = repoManager.openRepository(project)) {
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ assertThat(getRef(localRepo, branchRef)).isNotNull();
+ assertThat(getRef(localRepo, tagRef)).isNotNull();
+
+ PushResult deleteBranchResult = deleteRef(testRepo, branchRef);
+ assertOkStatus(deleteBranchResult, branchRef);
+
+ PushResult deleteTagResult = deleteRef(testRepo, tagRef);
+ assertOkStatus(deleteTagResult, tagRef);
+
+ fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo);
+ assertThat(getRef(localRepo, branchRef)).isNull();
+ assertThat(getRef(localRepo, tagRef)).isNull();
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static class TestModule extends FetchModule<JGitFetch> {
+ @Override
+ Class<JGitFetch> clientClass() {
+ return JGitFetch.class;
+ }
+
+ @Override
+ Config cf() {
+ Config cf = new Config();
+ cf.setInt("remote", "test_config", "timeout", 0);
+ cf.setBoolean("remote", "test_config", "mirror", true);
+ return cf;
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
index 29bf7e4..5717a8b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -309,7 +309,11 @@
getInstance(SourcesCollection.class).getByRemoteName(TEST_REPLICATION_REMOTE).get();
FetchApiClient client = getInstance(FetchApiClient.Factory.class).create(source);
- client.initProject(projectToCreate, new URIish(source.getApis().get(0)));
+ client.initProject(
+ projectToCreate,
+ new URIish(source.getApis().get(0)),
+ System.currentTimeMillis(),
+ Collections.emptyList());
waitUntil(() -> repoManager.list().contains(projectToCreate));
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 603528d..a07aa55 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -33,6 +33,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.api.changes.NotifyHandling;
import com.google.gerrit.extensions.common.AccountInfo;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
@@ -147,7 +148,8 @@
.when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any()))
.thenReturn(httpResult);
when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(fetchHttpResult);
- when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult);
+ when(fetchRestApiClient.initProject(any(), any(), anyLong(), any()))
+ .thenReturn(successfulHttpResult);
when(successfulHttpResult.isSuccessful()).thenReturn(true);
when(httpResult.isSuccessful()).thenReturn(true);
when(fetchHttpResult.isSuccessful()).thenReturn(true);
@@ -206,7 +208,7 @@
objectUnderTest.start();
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient).initProject(any(), any());
+ verify(fetchRestApiClient).initProject(any(), any(), anyLong(), any());
}
@Test
@@ -219,7 +221,22 @@
objectUnderTest.start();
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient, never()).initProject(any(), any());
+ verify(fetchRestApiClient, never()).initProject(any(), any(), anyLong(), any());
+ }
+
+ @Test
+ public void shouldNotCallInitProjectWhenProjectWithoutConfiguration() throws Exception {
+ Event event = new TestEvent("refs/changes/01/1/meta");
+ when(httpResult.isSuccessful()).thenReturn(false);
+ when(httpResult.isProjectMissing(any())).thenReturn(true);
+ when(source.isCreateMissingRepositories()).thenReturn(true);
+ when(revReader.read(any(), any(), eq(RefNames.REFS_CONFIG), anyInt()))
+ .thenReturn(Optional.empty());
+
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient, never()).initProject(any(), any(), anyLong(), any());
}
@Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
index 25aa2a7..65ccdfb 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -24,6 +24,7 @@
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.io.CharStreams;
+import com.google.common.net.MediaType;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
@@ -37,6 +38,7 @@
import java.nio.ByteBuffer;
import java.util.Collections;
import org.apache.http.Header;
+import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
@@ -94,6 +96,17 @@
+ "\",\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"sha1\":\""
+ blobObjectId
+ "\",\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}";
+
+ String expectedInitProjectWithConfigPayload =
+ "{\"label\":\"Replication\",\"ref_name\":\"refs/meta/config\",\"event_created_on\":"
+ + eventCreatedOn
+ + ",\"revisions_data\":[{\"commit_object\":{\"sha1\":\""
+ + commitObjectId
+ + "\",\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"sha1\":\""
+ + treeObjectId
+ + "\",\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"sha1\":\""
+ + blobObjectId
+ + "\",\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}]}";
String commitObject =
"tree "
+ treeObjectId
@@ -374,18 +387,26 @@
@Test
public void shouldCallInitProjectEndpoint() throws Exception {
-
- objectUnderTest.initProject(Project.nameKey("test_repo"), new URIish(api));
+ objectUnderTest.initProject(
+ Project.nameKey("test_repo"),
+ new URIish(api),
+ eventCreatedOn,
+ Collections.singletonList(createSampleRevisionData()));
verify(httpClient, times(1)).execute(httpPutCaptor.capture(), any());
-
HttpPut httpPut = httpPutCaptor.getValue();
+ String payload =
+ CharStreams.toString(
+ new InputStreamReader(httpPut.getEntity().getContent(), Charsets.UTF_8));
assertThat(httpPut.getURI().getHost()).isEqualTo("gerrit-host");
+ assertThat(httpPut.getHeaders(HttpHeaders.CONTENT_TYPE)[0].getValue())
+ .isEqualTo(MediaType.JSON_UTF_8.toString());
assertThat(httpPut.getURI().getPath())
.isEqualTo(
String.format(
"%s/plugins/pull-replication/init-project/test_repo.git",
urlAuthenticationPrefix()));
+ assertThat(payload).isEqualTo(expectedInitProjectWithConfigPayload);
assertAuthentication(httpPut);
}
