Merge "Remove events from beforeStartupEventsQueue in-memory queue" into stable-3.6
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index fe6a17c..ef95596 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -521,7 +521,20 @@ private HttpResult initProject( Project.NameKey project, URIish uri, FetchApiClient fetchClient, HttpResult result) throws IOException, ClientProtocolException { - HttpResult initProjectResult = fetchClient.initProject(project, uri); + RevisionData refsMetaConfigRevisionData = + revReaderProvider + .get() + .read(project, null, RefNames.REFS_CONFIG, 0) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Project %s does not have %s", project, RefNames.REFS_CONFIG))); + + List<RevisionData> refsMetaConfigDataList = + fetchWholeMetaHistory(project, RefNames.REFS_CONFIG, refsMetaConfigRevisionData); + HttpResult initProjectResult = + fetchClient.initProject(project, uri, System.currentTimeMillis(), refsMetaConfigDataList); if (initProjectResult.isSuccessful()) { result = fetchClient.callFetch(project, FetchOne.ALL_REFS, uri); } else {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java index 8711379..e7289ac 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -14,25 +14,39 @@ package com.googlesource.gerrit.plugins.replication.pull.api; +import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.checkAcceptHeader; import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.setResponse; +import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; +import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; +import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.Url; import com.google.gerrit.index.project.ProjectIndexer; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.permissions.GlobalPermission; import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.PermissionBackendException; +import com.google.gerrit.server.project.ProjectCache; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.LocalFS; import com.googlesource.gerrit.plugins.replication.pull.GerritConfigOps; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import com.googlesource.gerrit.plugins.replication.pull.api.util.PayloadSerDes; import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; import java.util.Optional; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -51,17 +65,23 @@ private final Provider<CurrentUser> userProvider; private final PermissionBackend permissionBackend; private final ProjectIndexer projectIndexer; + private final ApplyObjectCommand applyObjectCommand; + private final ProjectCache projectCache; @Inject ProjectInitializationAction( GerritConfigOps gerritConfigOps, Provider<CurrentUser> userProvider, PermissionBackend permissionBackend, - ProjectIndexer projectIndexer) { + ProjectIndexer projectIndexer, + ApplyObjectCommand applyObjectCommand, + ProjectCache projectCache) { this.gerritConfigOps = gerritConfigOps; this.userProvider = userProvider; this.permissionBackend = permissionBackend; this.projectIndexer = projectIndexer; + this.applyObjectCommand = applyObjectCommand; + this.projectCache = projectCache; } @Override @@ -73,47 +93,140 @@ return; } - String path = httpServletRequest.getRequestURI(); - String projectName = Url.decode(path.substring(path.lastIndexOf('/') + 1)); + String gitRepositoryName = getGitRepositoryName(httpServletRequest); try { - if (initProject(projectName)) { + boolean initProjectStatus; + String contentType = httpServletRequest.getContentType(); + if (checkContentType(contentType, MediaType.JSON_UTF_8)) { + // init project request includes project configuration in JSON format. + initProjectStatus = initProjectWithConfiguration(httpServletRequest, gitRepositoryName); + } else if (checkContentType(contentType, MediaType.PLAIN_TEXT_UTF_8)) { + // init project request does not include project configuration. + initProjectStatus = initProject(gitRepositoryName); + } else { + setResponse( + httpServletResponse, + SC_BAD_REQUEST, + String.format( + "Invalid Content Type. Only %s or %s is supported.", + MediaType.JSON_UTF_8.toString(), MediaType.PLAIN_TEXT_UTF_8.toString())); + return; + } + + if (initProjectStatus) { setResponse( httpServletResponse, HttpServletResponse.SC_CREATED, - "Project " + projectName + " initialized"); + "Project " + gitRepositoryName + " initialized"); return; } - } catch (AuthException | PermissionBackendException e) { + setResponse( httpServletResponse, - HttpServletResponse.SC_FORBIDDEN, - "User not authorized to create project " + projectName); - return; + HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Cannot initialize project " + gitRepositoryName); + } catch (BadRequestException | IllegalArgumentException e) { + logExceptionAndUpdateResponse(httpServletResponse, e, SC_BAD_REQUEST, gitRepositoryName); + } catch (RefUpdateException | MissingParentObjectException e) { + logExceptionAndUpdateResponse(httpServletResponse, e, SC_CONFLICT, gitRepositoryName); + } catch (AuthException | PermissionBackendException e) { + logExceptionAndUpdateResponse(httpServletResponse, e, SC_FORBIDDEN, gitRepositoryName); } - - setResponse( - httpServletResponse, - HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - "Cannot initialize project " + projectName); } - public boolean initProject(String projectName) throws AuthException, PermissionBackendException { + public boolean initProject(String gitRepositoryName) + throws AuthException, PermissionBackendException { + if (initProject(gitRepositoryName, true)) { + repLog.info("Init project API from {}", gitRepositoryName); + return true; + } + return false; + } + + private boolean initProjectWithConfiguration( + HttpServletRequest httpServletRequest, String gitRepositoryName) + throws AuthException, PermissionBackendException, IOException, BadRequestException, + MissingParentObjectException, RefUpdateException { + + RevisionsInput input = PayloadSerDes.parseRevisionsInput(httpServletRequest); + validateInput(input); + if (!initProject(gitRepositoryName, false)) { + return false; + } + + String projectName = gitRepositoryName.replace(".git", ""); + applyObjectCommand.applyObjects( + Project.nameKey(projectName), + input.getRefName(), + input.getRevisionsData(), + input.getLabel(), + input.getEventCreatedOn()); + projectCache.onCreateProject(Project.nameKey(projectName)); + repLog.info( + "Init project API from {} for {}:{} - {}", + input.getLabel(), + projectName, + input.getRefName(), + Arrays.toString(input.getRevisionsData())); + return true; + } + + private boolean initProject(String gitRepositoryName, boolean needsProjectReindexing) + throws AuthException, PermissionBackendException { // When triggered internally(for example by consuming stream events) user is not provided // and internal user is returned. Project creation should be always allowed for internal user. if (!userProvider.get().isInternalUser()) { permissionBackend.user(userProvider.get()).check(GlobalPermission.CREATE_PROJECT); } - Optional<URIish> maybeUri = gerritConfigOps.getGitRepositoryURI(projectName); + Optional<URIish> maybeUri = gerritConfigOps.getGitRepositoryURI(gitRepositoryName); if (!maybeUri.isPresent()) { - logger.atSevere().log("Cannot initialize project '%s'", projectName); + logger.atSevere().log("Cannot initialize project '%s'", gitRepositoryName); return false; } LocalFS localFS = new LocalFS(maybeUri.get()); - Project.NameKey projectNameKey = Project.NameKey.parse(projectName); + Project.NameKey projectNameKey = Project.NameKey.parse(gitRepositoryName); if (localFS.createProject(projectNameKey, RefNames.HEAD)) { - projectIndexer.index(projectNameKey); + if (needsProjectReindexing) { + projectIndexer.index(projectNameKey); + } return true; } return false; } + + private void validateInput(RevisionsInput input) { + + if (Strings.isNullOrEmpty(input.getLabel())) { + throw new IllegalArgumentException("Source label cannot be null or empty"); + } + + if (!Objects.equals(input.getRefName(), RefNames.REFS_CONFIG)) { + throw new IllegalArgumentException( + String.format("Ref-update refname should be %s", RefNames.REFS_CONFIG)); + } + input.validate(); + } + + private String getGitRepositoryName(HttpServletRequest httpServletRequest) { + String path = httpServletRequest.getRequestURI(); + return Url.decode(path.substring(path.lastIndexOf('/') + 1)); + } + + private void logExceptionAndUpdateResponse( + HttpServletResponse httpServletResponse, + Exception e, + int statusCode, + String gitRepositoryName) + throws IOException { + repLog.error("Init Project API FAILED for {}", gitRepositoryName, e); + setResponse(httpServletResponse, statusCode, e.getMessage()); + } + + private boolean checkContentType(String contentType, MediaType mediaType) { + try { + return MediaType.parse(contentType).is(mediaType); + } catch (Exception e) { + return false; + } + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java index 5a4393c..d0e5c10 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -18,10 +18,8 @@ import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.checkAcceptHeader; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; -import static javax.servlet.http.HttpServletResponse.SC_CREATED; import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; -import static javax.servlet.http.HttpServletResponse.SC_OK; import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED; import com.google.common.flogger.FluentLogger; @@ -38,28 +36,22 @@ import com.google.gerrit.extensions.restapi.UnprocessableEntityException; import com.google.gerrit.httpd.AllRequestFilter; import com.google.gerrit.httpd.restapi.RestApiServlet; -import com.google.gerrit.json.OutputFormat; import com.google.gerrit.server.AnonymousUser; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectResource; import com.google.gerrit.server.project.ProjectState; -import com.google.gson.Gson; import com.google.gson.JsonParseException; -import com.google.gson.stream.JsonReader; import com.google.gson.stream.MalformedJsonException; import com.google.inject.Inject; import com.google.inject.Provider; -import com.google.inject.TypeLiteral; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; import com.googlesource.gerrit.plugins.replication.pull.api.exception.UnauthorizedAuthException; -import java.io.BufferedReader; -import java.io.EOFException; +import com.googlesource.gerrit.plugins.replication.pull.api.util.PayloadSerDes; import java.io.IOException; -import java.io.PrintWriter; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -86,7 +78,6 @@ private UpdateHeadAction updateHEADAction; private ProjectDeletionAction projectDeletionAction; private ProjectCache projectCache; - private Gson gson; private String pluginName; private final Provider<CurrentUser> currentUserProvider; @@ -109,7 +100,6 @@ this.projectDeletionAction = projectDeletionAction; this.projectCache = projectCache; this.pluginName = pluginName; - this.gson = OutputFormat.JSON.newGsonBuilder().create(); this.currentUserProvider = currentUserProvider; } @@ -126,13 +116,13 @@ try { if (isFetchAction(httpRequest)) { failIfcurrentUserIsAnonymous(); - writeResponse(httpResponse, doFetch(httpRequest)); + PayloadSerDes.writeResponse(httpResponse, doFetch(httpRequest)); } else if (isApplyObjectAction(httpRequest)) { failIfcurrentUserIsAnonymous(); - writeResponse(httpResponse, doApplyObject(httpRequest)); + PayloadSerDes.writeResponse(httpResponse, doApplyObject(httpRequest)); } else if (isApplyObjectsAction(httpRequest)) { failIfcurrentUserIsAnonymous(); - writeResponse(httpResponse, doApplyObjects(httpRequest)); + PayloadSerDes.writeResponse(httpResponse, doApplyObjects(httpRequest)); } else if (isInitProjectAction(httpRequest)) { failIfcurrentUserIsAnonymous(); if (!checkAcceptHeader(httpRequest, httpResponse)) { @@ -141,10 +131,10 @@ doInitProject(httpRequest, httpResponse); } else if (isUpdateHEADAction(httpRequest)) { failIfcurrentUserIsAnonymous(); - writeResponse(httpResponse, doUpdateHEAD(httpRequest)); + PayloadSerDes.writeResponse(httpResponse, doUpdateHEAD(httpRequest)); } else if (isDeleteProjectAction(httpRequest)) { failIfcurrentUserIsAnonymous(); - writeResponse(httpResponse, doDeleteProject(httpRequest)); + PayloadSerDes.writeResponse(httpResponse, doDeleteProject(httpRequest)); } else { chain.doFilter(request, response); } @@ -199,7 +189,7 @@ @SuppressWarnings("unchecked") private Response<String> doApplyObject(HttpServletRequest httpRequest) throws RestApiException, IOException, PermissionBackendException { - RevisionInput input = readJson(httpRequest, TypeLiteral.get(RevisionInput.class)); + RevisionInput input = PayloadSerDes.parseRevisionInput(httpRequest); IdString id = getProjectName(httpRequest).get(); return (Response<String>) applyObjectAction.apply(parseProjectResource(id), input); @@ -208,7 +198,7 @@ @SuppressWarnings("unchecked") private Response<String> doApplyObjects(HttpServletRequest httpRequest) throws RestApiException, IOException, PermissionBackendException { - RevisionsInput input = readJson(httpRequest, TypeLiteral.get(RevisionsInput.class)); + RevisionsInput input = PayloadSerDes.parseRevisionsInput(httpRequest); IdString id = getProjectName(httpRequest).get(); return (Response<String>) applyObjectsAction.apply(parseProjectResource(id), input); @@ -216,7 +206,7 @@ @SuppressWarnings("unchecked") private Response<String> doUpdateHEAD(HttpServletRequest httpRequest) throws Exception { - HeadInput input = readJson(httpRequest, TypeLiteral.get(HeadInput.class)); + HeadInput input = PayloadSerDes.parseHeadInput(httpRequest); IdString id = getProjectName(httpRequest).get(); return (Response<String>) updateHEADAction.apply(parseProjectResource(id), input); @@ -233,7 +223,7 @@ @SuppressWarnings("unchecked") private Response<Map<String, Object>> doFetch(HttpServletRequest httpRequest) throws IOException, RestApiException, PermissionBackendException { - Input input = readJson(httpRequest, TypeLiteral.get(Input.class)); + Input input = PayloadSerDes.parseInput(httpRequest); IdString id = getProjectName(httpRequest).get(); return (Response<Map<String, Object>>) fetchAction.apply(parseProjectResource(id), input); @@ -247,50 +237,6 @@ return new ProjectResource(project.get(), currentUserProvider.get()); } - private <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response) - throws IOException { - String responseJson = gson.toJson(response); - if (response.statusCode() == SC_OK || response.statusCode() == SC_CREATED) { - - httpResponse.setContentType("application/json"); - httpResponse.setStatus(response.statusCode()); - PrintWriter writer = httpResponse.getWriter(); - writer.print(new String(RestApiServlet.JSON_MAGIC)); - writer.print(responseJson); - } else { - httpResponse.sendError(response.statusCode(), responseJson); - } - } - - private <T> T readJson(HttpServletRequest httpRequest, TypeLiteral<T> typeLiteral) - throws IOException, BadRequestException { - - try (BufferedReader br = httpRequest.getReader(); - JsonReader json = new JsonReader(br)) { - try { - json.setLenient(true); - - try { - json.peek(); - } catch (EOFException e) { - throw new BadRequestException("Expected JSON object", e); - } - - return gson.fromJson(json, typeLiteral.getType()); - } finally { - try { - // Reader.close won't consume the rest of the input. Explicitly consume the request - // body. - br.skip(Long.MAX_VALUE); - } catch (Exception e) { - // ignore, e.g. trying to consume the rest of the input may fail if the request was - // cancelled - logger.atFine().withCause(e).log("Exception during the parsing of the request json"); - } - } - } - } - /** * Return project name from request URI. Request URI format: * /a/projects/<project_name>/pull-replication~apply-object
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java new file mode 100644 index 0000000..414af37 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/util/PayloadSerDes.java
@@ -0,0 +1,106 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.util; + +import static javax.servlet.http.HttpServletResponse.SC_CREATED; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.extensions.api.projects.HeadInput; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.httpd.restapi.RestApiServlet; +import com.google.gerrit.json.OutputFormat; +import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; +import com.google.inject.TypeLiteral; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.IOException; +import java.io.PrintWriter; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class PayloadSerDes { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final Gson gson = OutputFormat.JSON.newGsonBuilder().create(); + + public static RevisionInput parseRevisionInput(HttpServletRequest httpRequest) + throws BadRequestException, IOException { + return parse(httpRequest, TypeLiteral.get(RevisionInput.class)); + } + + public static RevisionsInput parseRevisionsInput(HttpServletRequest httpRequest) + throws BadRequestException, IOException { + return parse(httpRequest, TypeLiteral.get(RevisionsInput.class)); + } + + public static HeadInput parseHeadInput(HttpServletRequest httpRequest) + throws BadRequestException, IOException { + return parse(httpRequest, TypeLiteral.get(HeadInput.class)); + } + + public static FetchAction.Input parseInput(HttpServletRequest httpRequest) + throws BadRequestException, IOException { + return parse(httpRequest, TypeLiteral.get(FetchAction.Input.class)); + } + + public static <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response) + throws IOException { + String responseJson = gson.toJson(response); + if (response.statusCode() == SC_OK || response.statusCode() == SC_CREATED) { + + httpResponse.setContentType("application/json"); + httpResponse.setStatus(response.statusCode()); + PrintWriter writer = httpResponse.getWriter(); + writer.print(new String(RestApiServlet.JSON_MAGIC)); + writer.print(responseJson); + } else { + httpResponse.sendError(response.statusCode(), responseJson); + } + } + + private static <T> T parse(HttpServletRequest httpRequest, TypeLiteral<T> typeLiteral) + throws IOException, BadRequestException { + + try (BufferedReader br = httpRequest.getReader(); + JsonReader json = new JsonReader(br)) { + try { + json.setLenient(true); + + try { + json.peek(); + } catch (EOFException e) { + throw new BadRequestException("Expected JSON object", e); + } + + return gson.fromJson(json, typeLiteral.getType()); + } finally { + try { + // Reader.close won't consume the rest of the input. Explicitly consume the request + // body. + br.skip(Long.MAX_VALUE); + } catch (Exception e) { + // ignore, e.g. trying to consume the rest of the input may fail if the request was + // cancelled + logger.atFine().withCause(e).log("Exception during the parsing of the request json"); + } + } + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java index 1991260..f7ed4cb 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -40,7 +40,22 @@ return callFetch(project, refName, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis())); } - HttpResult initProject(Project.NameKey project, URIish uri) throws IOException; + /** + * Replicates the creation of a project, including the configuration stored in refs/meta/config. + * + * @param project The unique name of the project. + * @param uri The destination URI where the project and its configuration should be replicated to. + * @param eventCreatedOn The timestamp indicating when the init project event occurred. + * @param refsMetaConfigRevisionData A history of revisions for the refs/meta/config ref. + * @return An HTTP result object providing information about the replication process. + * @throws IOException If an I/O error occurs during the replication. + */ + HttpResult initProject( + Project.NameKey project, + URIish uri, + long eventCreatedOn, + List<RevisionData> refsMetaConfigRevisionData) + throws IOException; HttpResult deleteProject(Project.NameKey project, URIish apiUri) throws IOException;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index b606ba8..7607e4b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -23,6 +23,7 @@ import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.Project.NameKey; +import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.annotations.PluginName; import com.google.gerrit.extensions.restapi.Url; import com.google.gerrit.server.config.GerritInstanceId; @@ -130,14 +131,29 @@ } /* (non-Javadoc) - * @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#initProject(com.google.gerrit.entities.Project.NameKey, org.eclipse.jgit.transport.URIish) + * @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#initProject(com.google.gerrit.entities.Project.NameKey, org.eclipse.jgit.transport.URIish, long, java.util.List<com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData>) */ @Override - public HttpResult initProject(Project.NameKey project, URIish uri) throws IOException { + public HttpResult initProject( + NameKey project, + URIish uri, + long eventCreatedOn, + List<RevisionData> refsMetaConfigRevisionData) + throws IOException { String url = formatInitProjectUrl(uri.toString(), project); + + RevisionData[] inputData = new RevisionData[refsMetaConfigRevisionData.size()]; + RevisionsInput input = + new RevisionsInput( + instanceId, + RefNames.REFS_CONFIG, + eventCreatedOn, + refsMetaConfigRevisionData.toArray(inputData)); + HttpPut put = new HttpPut(url); + put.setEntity(new StringEntity(GSON.toJson(input))); put.addHeader(new BasicHeader("Accept", MediaType.ANY_TEXT_TYPE.toString())); - put.addHeader(new BasicHeader("Content-Type", MediaType.PLAIN_TEXT_UTF_8.toString())); + put.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); return executeRequest(put, bearerTokenProvider.get(), uri); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java index d3e45da..43ec9bf 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/CGitFetch.java
@@ -44,6 +44,7 @@ private URIish uri; private int timeout; private final String taskIdHex; + private final boolean isMirror; @Inject public CGitFetch( @@ -56,12 +57,17 @@ this.taskIdHex = taskIdHex; this.uri = appendCredentials(uri, cpFactory.create(config.getRemoteConfig().getName())); this.timeout = config.getRemoteConfig().getTimeout(); + this.isMirror = config.getRemoteConfig().isMirror(); } @Override public List<RefUpdateState> fetch(List<RefSpec> refsSpec) throws IOException { List<String> refs = refsSpec.stream().map(s -> s.toString()).collect(Collectors.toList()); - List<String> command = Lists.newArrayList("git", "fetch", uri.toPrivateASCIIString()); + List<String> command = Lists.newArrayList("git", "fetch"); + if (isMirror) { + command.add("--prune"); + } + command.add(uri.toPrivateASCIIString()); command.addAll(refs); ProcessBuilder pb = new ProcessBuilder().command(command).directory(localProjectDirectory); repLog.info("[{}] Fetch references {} from {}", taskIdHex, refs, uri);
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index b7db2c0..61d62b4 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -526,8 +526,7 @@ remote.NAME.mirror : If true, replication will remove local branches and tags that are absent remotely or invisible to the replication (for example read access -denied via `authGroup` option). Note that this option is currently -implemented for the JGit client only. +denied via `authGroup` option). By default, false, do not remove remote branches or tags.
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java index baacf20..3ca937c 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
@@ -15,6 +15,9 @@ package com.googlesource.gerrit.plugins.replication.pull; import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.acceptance.GitUtil.deleteRef; +import static com.google.gerrit.acceptance.GitUtil.pushHead; +import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -22,35 +25,33 @@ import static org.mockito.Mockito.when; import com.google.common.collect.Lists; +import com.google.gerrit.acceptance.PushOneCommit; import com.google.gerrit.acceptance.PushOneCommit.Result; import com.google.gerrit.acceptance.SkipProjectClone; import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Permission; import com.google.gerrit.extensions.api.projects.BranchInput; -import com.google.gerrit.extensions.config.FactoryModule; -import com.google.inject.Scopes; -import com.google.inject.assistedinject.FactoryModuleBuilder; -import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator; -import com.googlesource.gerrit.plugins.replication.CredentialsFactory; -import com.googlesource.gerrit.plugins.replication.ReplicationConfig; -import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; +import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.pull.fetch.BatchFetchClient; import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetch; import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch; -import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation; import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory; import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; -import java.net.URISyntaxException; import java.util.List; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.transport.PushResult; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; +import org.junit.Before; import org.junit.Test; @SkipProjectClone @@ -62,6 +63,16 @@ private static final String TEST_REPLICATION_SUFFIX = "suffix1"; private static final String TEST_TASK_ID = "taskid"; + @Inject private ProjectOperations projectOperations; + + @Before + public void allowRefDeletion() { + projectOperations + .allProjectsForUpdate() + .add(allow(Permission.DELETE).ref("refs/*").group(adminGroupUuid())) + .update(); + } + @Test public void shouldFetchRef() throws Exception { testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); @@ -224,29 +235,55 @@ } } + @Test + public void shouldNotPruneRefsWhenMirrorIsUnset() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + String BRANCH_REF = Constants.R_HEADS + "anyBranch"; + String TAG_REF = Constants.R_TAGS + "anyTag"; + + PushOneCommit.Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(BRANCH_REF); + branchPush.assertOkStatus(); + + PushResult tagPush = pushHead(testRepo, TAG_REF, false, false); + assertOkStatus(tagPush, TAG_REF); + + try (Repository localRepo = repoManager.openRepository(project)) { + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + waitUntil( + () -> + checkedGetRef(localRepo, BRANCH_REF) != null + && checkedGetRef(localRepo, TAG_REF) != null); + assertThat(getRef(localRepo, BRANCH_REF)).isNotNull(); + assertThat(getRef(localRepo, TAG_REF)).isNotNull(); + + PushResult deleteBranchResult = deleteRef(testRepo, BRANCH_REF); + assertOkStatus(deleteBranchResult, BRANCH_REF); + + PushResult deleteTagResult = deleteRef(testRepo, TAG_REF); + assertOkStatus(deleteTagResult, TAG_REF); + + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + waitUntil( + () -> + checkedGetRef(localRepo, BRANCH_REF) != null + && checkedGetRef(localRepo, TAG_REF) != null); + assertThat(getRef(localRepo, BRANCH_REF)).isNotNull(); + assertThat(getRef(localRepo, TAG_REF)).isNotNull(); + } + } + @SuppressWarnings("unused") - private static class TestModule extends FactoryModule { + private static class TestModule extends FetchModule<CGitFetch> { @Override - protected void configure() { + Class<CGitFetch> clientClass() { + return CGitFetch.class; + } + + @Override + Config cf() { Config cf = new Config(); cf.setInt("remote", "test_config", "timeout", 0); - try { - RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config"); - SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf); - bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class); - bind(CredentialsFactory.class) - .to(AutoReloadSecureCredentialsFactoryDecorator.class) - .in(Scopes.SINGLETON); - - bind(SourceConfiguration.class).toInstance(sourceConfig); - install( - new FactoryModuleBuilder() - .implement(Fetch.class, CGitFetch.class) - .implement(Fetch.class, FetchClientImplementation.class, CGitFetch.class) - .build(FetchFactory.class)); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + return cf; } } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchWithMirrorIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchWithMirrorIT.java new file mode 100644 index 0000000..8a3bae9 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchWithMirrorIT.java
@@ -0,0 +1,108 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.acceptance.GitUtil.deleteRef; +import static com.google.gerrit.acceptance.GitUtil.pushHead; +import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow; + +import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Permission; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetch; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.PushResult; +import org.junit.Before; +import org.junit.Test; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.CGitFetchWithMirrorIT$TestModule") +public class CGitFetchWithMirrorIT extends FetchITBase { + private static final String TEST_REPLICATION_SUFFIX = "suffix1"; + private static final String TEST_TASK_ID = "taskid"; + + @Inject private ProjectOperations projectOperations; + + @Before + public void allowRefDeletion() { + projectOperations + .allProjectsForUpdate() + .add(allow(Permission.DELETE).ref("refs/*").group(adminGroupUuid())) + .update(); + } + + @Test + public void shouldPruneRefsWhenMirrorIsTrue() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + String BRANCH_REF = Constants.R_HEADS + "anyBranch"; + String TAG_REF = Constants.R_TAGS + "anyTag"; + + Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(BRANCH_REF); + branchPush.assertOkStatus(); + + PushResult tagPush = pushHead(testRepo, TAG_REF, false, false); + assertOkStatus(tagPush, TAG_REF); + + try (Repository localRepo = repoManager.openRepository(project)) { + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + waitUntil( + () -> + checkedGetRef(localRepo, BRANCH_REF) != null + && checkedGetRef(localRepo, TAG_REF) != null); + assertThat(getRef(localRepo, BRANCH_REF)).isNotNull(); + assertThat(getRef(localRepo, TAG_REF)).isNotNull(); + + PushResult deleteBranchResult = deleteRef(testRepo, BRANCH_REF); + assertOkStatus(deleteBranchResult, BRANCH_REF); + + PushResult deleteTagResult = deleteRef(testRepo, TAG_REF); + assertOkStatus(deleteTagResult, TAG_REF); + + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + waitUntil( + () -> + checkedGetRef(localRepo, BRANCH_REF) == null + && checkedGetRef(localRepo, TAG_REF) == null); + assertThat(getRef(localRepo, BRANCH_REF)).isNull(); + assertThat(getRef(localRepo, TAG_REF)).isNull(); + } + } + + @SuppressWarnings("unused") + private static class TestModule extends FetchModule<CGitFetch> { + @Override + Class<CGitFetch> clientClass() { + return CGitFetch.class; + } + + @Override + Config cf() { + Config cf = new Config(); + cf.setInt("remote", "test_config", "timeout", 0); + cf.setBoolean("remote", "test_config", "mirror", true); + return cf; + } + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java index be9f902..3429983 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
@@ -14,18 +14,39 @@ package com.googlesource.gerrit.plugins.replication.pull; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.common.collect.Lists; import com.google.common.flogger.FluentLogger; import com.google.gerrit.acceptance.LightweightPluginDaemonTest; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.config.FactoryModule; import com.google.gerrit.server.config.SitePaths; import com.google.inject.Inject; +import com.google.inject.Scopes; +import com.google.inject.assistedinject.FactoryModuleBuilder; +import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator; +import com.googlesource.gerrit.plugins.replication.CredentialsFactory; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; +import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; +import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch; +import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation; import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory; +import java.io.IOException; +import java.net.URISyntaxException; import java.nio.file.Path; import java.time.Duration; import java.util.function.Supplier; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.PushResult; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.RemoteRefUpdate; +import org.eclipse.jgit.transport.URIish; public abstract class FetchITBase extends LightweightPluginDaemonTest { private static final String TEST_REPLICATION_SUFFIX = "suffix1"; @@ -33,6 +54,7 @@ private static final int TEST_REPLICATION_DELAY = 60; private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2); + private static final RefSpec ALL_REFS = new RefSpec("+refs/*:refs/*"); @Inject private SitePaths sitePaths; @Inject private ProjectOperations projectOperations; @@ -66,7 +88,49 @@ } } + protected void fetchAllRefs(String taskId, Path remotePath, Repository localRepo) + throws URISyntaxException, IOException { + fetchFactory + .create(taskId, new URIish(remotePath.toString()), localRepo) + .fetch(Lists.newArrayList(ALL_REFS)); + } + + protected static void assertOkStatus(PushResult result, String ref) { + RemoteRefUpdate refUpdate = result.getRemoteUpdate(ref); + assertThat(refUpdate).isNotNull(); + assertWithMessage(refUpdate.getMessage()) + .that(refUpdate.getStatus()) + .isEqualTo(RemoteRefUpdate.Status.OK); + } + Project.NameKey createTestProject(String name) { return projectOperations.newProject().name(name).create(); } + + protected abstract static class FetchModule<T extends Fetch> extends FactoryModule { + abstract Config cf(); + + abstract Class<T> clientClass(); + + @Override + protected void configure() { + try { + RemoteConfig remoteConfig = new RemoteConfig(cf(), "test_config"); + SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf()); + bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class); + bind(CredentialsFactory.class) + .to(AutoReloadSecureCredentialsFactoryDecorator.class) + .in(Scopes.SINGLETON); + + bind(SourceConfiguration.class).toInstance(sourceConfig); + install( + new FactoryModuleBuilder() + .implement(Fetch.class, clientClass()) + .implement(Fetch.class, FetchClientImplementation.class, clientClass()) + .build(FetchFactory.class)); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java index 77cf80d..b900d8a 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
@@ -15,7 +15,6 @@ package com.googlesource.gerrit.plugins.replication.pull; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; import static com.google.gerrit.acceptance.GitUtil.deleteRef; import static com.google.gerrit.acceptance.GitUtil.pushHead; import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow; @@ -27,31 +26,15 @@ import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.entities.Permission; -import com.google.gerrit.extensions.config.FactoryModule; import com.google.inject.Inject; -import com.google.inject.Scopes; -import com.google.inject.assistedinject.FactoryModuleBuilder; -import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator; -import com.googlesource.gerrit.plugins.replication.CredentialsFactory; -import com.googlesource.gerrit.plugins.replication.ReplicationConfig; -import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch; -import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation; -import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory; import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch; import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException; -import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.List; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.PushResult; import org.eclipse.jgit.transport.RefSpec; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.RemoteRefUpdate; import org.eclipse.jgit.transport.URIish; import org.junit.Before; import org.junit.Test; @@ -64,7 +47,6 @@ public class JGitFetchIT extends FetchITBase { private static final String TEST_REPLICATION_SUFFIX = "suffix1"; private static final String TEST_TASK_ID = "taskid"; - private static final RefSpec ALL_REFS = new RefSpec("+refs/*:refs/*"); @Inject private ProjectOperations projectOperations; @@ -89,12 +71,10 @@ } @Test - public void shouldPruneRefsWhenMirrorIsTrue() throws Exception { + public void shouldPruneRefsWhenMirrorIsUnset() throws Exception { testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); - String branchName = "anyBranch"; - String branchRef = Constants.R_HEADS + branchName; - String tagName = "anyTag"; - String tagRef = Constants.R_TAGS + tagName; + String branchRef = Constants.R_HEADS + "anyBranch"; + String tagRef = Constants.R_TAGS + "anyTag"; PushOneCommit.Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(branchRef); branchPush.assertOkStatus(); @@ -103,13 +83,8 @@ assertOkStatus(tagPush, tagRef); try (Repository localRepo = repoManager.openRepository(project)) { - List<RefUpdateState> fetchCreated = fetchAllRefs(localRepo); - assertThat(fetchCreated.toString()) - .contains(new RefUpdateState(branchRef, RefUpdate.Result.NEW).toString()); + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); assertThat(getRef(localRepo, branchRef)).isNotNull(); - - assertThat(fetchCreated.toString()) - .contains(new RefUpdateState(tagRef, RefUpdate.Result.NEW).toString()); assertThat(getRef(localRepo, tagRef)).isNotNull(); PushResult deleteBranchResult = deleteRef(testRepo, branchRef); @@ -118,55 +93,24 @@ PushResult deleteTagResult = deleteRef(testRepo, tagRef); assertOkStatus(deleteTagResult, tagRef); - List<RefUpdateState> fetchDeleted = fetchAllRefs(localRepo); - assertThat(fetchDeleted.toString()) - .contains(new RefUpdateState(branchRef, RefUpdate.Result.FORCED).toString()); - assertThat(getRef(localRepo, branchRef)).isNull(); - - assertThat(fetchDeleted.toString()) - .contains(new RefUpdateState(tagRef, RefUpdate.Result.FORCED).toString()); - assertThat(getRef(localRepo, tagRef)).isNull(); + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + assertThat(getRef(localRepo, branchRef)).isNotNull(); + assertThat(getRef(localRepo, tagRef)).isNotNull(); } } - private List<RefUpdateState> fetchAllRefs(Repository localRepo) - throws URISyntaxException, IOException { - Fetch fetch = fetchFactory.create(TEST_TASK_ID, new URIish(testRepoPath.toString()), localRepo); - return fetch.fetch(Lists.newArrayList(ALL_REFS)); - } - - private static void assertOkStatus(PushResult result, String ref) { - RemoteRefUpdate refUpdate = result.getRemoteUpdate(ref); - assertThat(refUpdate).isNotNull(); - assertWithMessage(refUpdate.getMessage()) - .that(refUpdate.getStatus()) - .isEqualTo(RemoteRefUpdate.Status.OK); - } - @SuppressWarnings("unused") - private static class TestModule extends FactoryModule { + private static class TestModule extends FetchModule<JGitFetch> { @Override - protected void configure() { + Class<JGitFetch> clientClass() { + return JGitFetch.class; + } + + @Override + Config cf() { Config cf = new Config(); cf.setInt("remote", "test_config", "timeout", 0); - cf.setBoolean("remote", "test_config", "mirror", true); - try { - RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config"); - SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf); - bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class); - bind(CredentialsFactory.class) - .to(AutoReloadSecureCredentialsFactoryDecorator.class) - .in(Scopes.SINGLETON); - - bind(SourceConfiguration.class).toInstance(sourceConfig); - install( - new FactoryModuleBuilder() - .implement(Fetch.class, JGitFetch.class) - .implement(Fetch.class, FetchClientImplementation.class, JGitFetch.class) - .build(FetchFactory.class)); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + return cf; } } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchWithMirrorIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchWithMirrorIT.java new file mode 100644 index 0000000..2df700f --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchWithMirrorIT.java
@@ -0,0 +1,100 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.acceptance.GitUtil.deleteRef; +import static com.google.gerrit.acceptance.GitUtil.pushHead; +import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow; + +import com.google.gerrit.acceptance.PushOneCommit; +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Permission; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.PushResult; +import org.junit.Before; +import org.junit.Test; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.JGitFetchWithMirrorIT$TestModule") +public class JGitFetchWithMirrorIT extends FetchITBase { + private static final String TEST_REPLICATION_SUFFIX = "suffix1"; + private static final String TEST_TASK_ID = "taskid"; + + @Inject private ProjectOperations projectOperations; + + @Before + public void allowRefDeletion() { + projectOperations + .allProjectsForUpdate() + .add(allow(Permission.DELETE).ref("refs/*").group(adminGroupUuid())) + .update(); + } + + @Test + public void shouldPruneRefsWhenMirrorIsTrue() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + String branchRef = Constants.R_HEADS + "anyBranch"; + String tagRef = Constants.R_TAGS + "anyTag"; + + PushOneCommit.Result branchPush = pushFactory.create(user.newIdent(), testRepo).to(branchRef); + branchPush.assertOkStatus(); + + PushResult tagPush = pushHead(testRepo, tagRef, false, false); + assertOkStatus(tagPush, tagRef); + + try (Repository localRepo = repoManager.openRepository(project)) { + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + assertThat(getRef(localRepo, branchRef)).isNotNull(); + assertThat(getRef(localRepo, tagRef)).isNotNull(); + + PushResult deleteBranchResult = deleteRef(testRepo, branchRef); + assertOkStatus(deleteBranchResult, branchRef); + + PushResult deleteTagResult = deleteRef(testRepo, tagRef); + assertOkStatus(deleteTagResult, tagRef); + + fetchAllRefs(TEST_TASK_ID, testRepoPath, localRepo); + assertThat(getRef(localRepo, branchRef)).isNull(); + assertThat(getRef(localRepo, tagRef)).isNull(); + } + } + + @SuppressWarnings("unused") + private static class TestModule extends FetchModule<JGitFetch> { + @Override + Class<JGitFetch> clientClass() { + return JGitFetch.class; + } + + @Override + Config cf() { + Config cf = new Config(); + cf.setInt("remote", "test_config", "timeout", 0); + cf.setBoolean("remote", "test_config", "mirror", true); + return cf; + } + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java index 29bf7e4..5717a8b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -309,7 +309,11 @@ getInstance(SourcesCollection.class).getByRemoteName(TEST_REPLICATION_REMOTE).get(); FetchApiClient client = getInstance(FetchApiClient.Factory.class).create(source); - client.initProject(projectToCreate, new URIish(source.getApis().get(0))); + client.initProject( + projectToCreate, + new URIish(source.getApis().get(0)), + System.currentTimeMillis(), + Collections.emptyList()); waitUntil(() -> repoManager.list().contains(projectToCreate)); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 603528d..a07aa55 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.api.changes.NotifyHandling; import com.google.gerrit.extensions.common.AccountInfo; import com.google.gerrit.extensions.events.ProjectDeletedListener; @@ -147,7 +148,8 @@ .when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(fetchHttpResult); - when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult); + when(fetchRestApiClient.initProject(any(), any(), anyLong(), any())) + .thenReturn(successfulHttpResult); when(successfulHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isSuccessful()).thenReturn(true); when(fetchHttpResult.isSuccessful()).thenReturn(true); @@ -206,7 +208,7 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).initProject(any(), any()); + verify(fetchRestApiClient).initProject(any(), any(), anyLong(), any()); } @Test @@ -219,7 +221,22 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient, never()).initProject(any(), any()); + verify(fetchRestApiClient, never()).initProject(any(), any(), anyLong(), any()); + } + + @Test + public void shouldNotCallInitProjectWhenProjectWithoutConfiguration() throws Exception { + Event event = new TestEvent("refs/changes/01/1/meta"); + when(httpResult.isSuccessful()).thenReturn(false); + when(httpResult.isProjectMissing(any())).thenReturn(true); + when(source.isCreateMissingRepositories()).thenReturn(true); + when(revReader.read(any(), any(), eq(RefNames.REFS_CONFIG), anyInt())) + .thenReturn(Optional.empty()); + + objectUnderTest.start(); + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, never()).initProject(any(), any(), anyLong(), any()); } @Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java index 25aa2a7..65ccdfb 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -24,6 +24,7 @@ import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.io.CharStreams; +import com.google.common.net.MediaType; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; import com.googlesource.gerrit.plugins.replication.CredentialsFactory; @@ -37,6 +38,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import org.apache.http.Header; +import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; @@ -94,6 +96,17 @@ + "\",\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"sha1\":\"" + blobObjectId + "\",\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}"; + + String expectedInitProjectWithConfigPayload = + "{\"label\":\"Replication\",\"ref_name\":\"refs/meta/config\",\"event_created_on\":" + + eventCreatedOn + + ",\"revisions_data\":[{\"commit_object\":{\"sha1\":\"" + + commitObjectId + + "\",\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"sha1\":\"" + + treeObjectId + + "\",\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"sha1\":\"" + + blobObjectId + + "\",\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}]}"; String commitObject = "tree " + treeObjectId @@ -374,18 +387,26 @@ @Test public void shouldCallInitProjectEndpoint() throws Exception { - - objectUnderTest.initProject(Project.nameKey("test_repo"), new URIish(api)); + objectUnderTest.initProject( + Project.nameKey("test_repo"), + new URIish(api), + eventCreatedOn, + Collections.singletonList(createSampleRevisionData())); verify(httpClient, times(1)).execute(httpPutCaptor.capture(), any()); - HttpPut httpPut = httpPutCaptor.getValue(); + String payload = + CharStreams.toString( + new InputStreamReader(httpPut.getEntity().getContent(), Charsets.UTF_8)); assertThat(httpPut.getURI().getHost()).isEqualTo("gerrit-host"); + assertThat(httpPut.getHeaders(HttpHeaders.CONTENT_TYPE)[0].getValue()) + .isEqualTo(MediaType.JSON_UTF_8.toString()); assertThat(httpPut.getURI().getPath()) .isEqualTo( String.format( "%s/plugins/pull-replication/init-project/test_repo.git", urlAuthenticationPrefix())); + assertThat(payload).isEqualTo(expectedInitProjectWithConfigPayload); assertAuthentication(httpPut); }