Add event creation time to the apply object payload Add eventCreatedOn field to the apply object REST API json payload. This allows to send timestamp of the event which triggered the apply object call. This information is useful for the receiver side to understand which events were already processed. Adding new field to the apply object REST API is backward compatible. Json payload generated without eventsCreatedOn is deserialised correctly and eventCreatedOn field is set to 0. Previous version of pull-replication ignores extra fields in the json payload. Bug: Issue 16885 Change-Id: I73c66cfc07285d3e1c2d0c4e190d56e21599a89d
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index c7a44d0..e99580a 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -170,6 +170,7 @@ event.refUpdate.get().project, ObjectId.fromString(event.refUpdate.get().newRev), event.getRefName(), + event.eventCreatedOn, ZEROS_OBJECTID.equals(event.refUpdate.get().newRev)); } } @@ -199,16 +200,22 @@ return !refsFilter.match(refName); } - private void fire(String projectName, ObjectId objectId, String refName, boolean isDelete) { + private void fire( + String projectName, + ObjectId objectId, + String refName, + long eventCreatedOn, + boolean isDelete) { ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher.get())); - fire(Project.nameKey(projectName), objectId, refName, isDelete, state); + fire(Project.nameKey(projectName), objectId, refName, eventCreatedOn, isDelete, state); state.markAllFetchTasksScheduled(); } private void fire( - Project.NameKey project, + NameKey project, ObjectId objectId, String refName, + long eventCreatedOn, boolean isDelete, ReplicationState state) { if (!running) { @@ -216,7 +223,7 @@ "Replication plugin did not finish startup before event, event replication is postponed", state); beforeStartupEventsQueue.add( - ReferenceUpdatedEvent.create(project.get(), refName, objectId, isDelete)); + ReferenceUpdatedEvent.create(project.get(), refName, objectId, eventCreatedOn, isDelete)); return; } ForkJoinPool fetchCallsPool = null; @@ -224,7 +231,7 @@ fetchCallsPool = new ForkJoinPool(sources.get().getAll().size()); final Consumer<Source> callFunction = - callFunction(project, objectId, refName, isDelete, state); + callFunction(project, objectId, refName, eventCreatedOn, isDelete, state); fetchCallsPool .submit(() -> sources.get().getAll().parallelStream().forEach(callFunction)) .get(fetchCallsTimeout, TimeUnit.MILLISECONDS); @@ -246,9 +253,11 @@ NameKey project, ObjectId objectId, String refName, + long eventCreatedOn, boolean isDelete, ReplicationState state) { - CallFunction call = getCallFunction(project, objectId, refName, isDelete, state); + CallFunction call = + getCallFunction(project, objectId, refName, eventCreatedOn, isDelete, state); return (source) -> { boolean callSuccessful; @@ -273,10 +282,12 @@ NameKey project, ObjectId objectId, String refName, + long eventCreatedOn, boolean isDelete, ReplicationState state) { if (isDelete) { - return ((source) -> callSendObject(source, project, refName, isDelete, null, state)); + return ((source) -> + callSendObject(source, project, refName, eventCreatedOn, isDelete, null, state)); } try { @@ -291,7 +302,13 @@ if (revisionData.isPresent()) { return ((source) -> callSendObject( - source, project, refName, isDelete, Arrays.asList(revisionData.get()), state)); + source, + project, + refName, + eventCreatedOn, + isDelete, + Arrays.asList(revisionData.get()), + state)); } } catch (InvalidObjectIdException | IOException e) { stateLog.error( @@ -307,8 +324,9 @@ private boolean callSendObject( Source source, - Project.NameKey project, + NameKey project, String refName, + long eventCreatedOn, boolean isDelete, List<RevisionData> revision, ReplicationState state) @@ -328,8 +346,9 @@ Context<String> apiTimer = applyObjectMetrics.startEnd2End(source.getRemoteConfigName()); HttpResult result = isDelete - ? fetchClient.callSendObject(project, refName, isDelete, null, uri) - : fetchClient.callSendObjects(project, refName, revision, uri); + ? fetchClient.callSendObject( + project, refName, eventCreatedOn, isDelete, null, uri) + : fetchClient.callSendObjects(project, refName, eventCreatedOn, revision, uri); boolean resultSuccessful = result.isSuccessful(); repLog.info( "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:" @@ -360,7 +379,8 @@ project, refName, allRevisions); - return callSendObject(source, project, refName, isDelete, allRevisions, state); + return callSendObject( + source, project, refName, eventCreatedOn, isDelete, allRevisions, state); } throw new MissingParentObjectException( @@ -501,7 +521,12 @@ String eventKey = String.format("%s:%s", event.projectName(), event.refName()); if (!eventsReplayed.contains(eventKey)) { repLog.info("Firing pending task {}", event); - fire(event.projectName(), event.objectId(), event.refName(), event.isDelete()); + fire( + event.projectName(), + event.objectId(), + event.refName(), + event.eventCreatedOn(), + event.isDelete()); eventsReplayed.add(eventKey); } } @@ -522,9 +547,13 @@ abstract static class ReferenceUpdatedEvent { static ReferenceUpdatedEvent create( - String projectName, String refName, ObjectId objectId, boolean isDelete) { + String projectName, + String refName, + ObjectId objectId, + long eventCreatedOn, + boolean isDelete) { return new AutoValue_ReplicationQueue_ReferenceUpdatedEvent( - projectName, refName, objectId, isDelete); + projectName, refName, objectId, eventCreatedOn, isDelete); } public abstract String projectName(); @@ -533,6 +562,8 @@ public abstract ObjectId objectId(); + public abstract long eventCreatedOn(); + public abstract boolean isDelete(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java index c18e11d..6883f2c 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -23,11 +23,14 @@ private String refName; + private long eventCreatedOn; private RevisionData revisionData; - public RevisionInput(String label, String refName, RevisionData revisionData) { + public RevisionInput( + String label, String refName, long eventCreatedOn, RevisionData revisionData) { this.label = label; this.refName = refName; + this.eventCreatedOn = eventCreatedOn; this.revisionData = revisionData; } @@ -43,6 +46,10 @@ return revisionData; } + public long getEventCreatedOn() { + return eventCreatedOn; + } + public void validate() { validate(refName, revisionData); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java index 2361f6b..d4626f4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java
@@ -21,11 +21,14 @@ private String refName; + private long eventCreatedOn; private RevisionData[] revisionsData; - public RevisionsInput(String label, String refName, RevisionData[] revisionsData) { + public RevisionsInput( + String label, String refName, long eventCreatedOn, RevisionData[] revisionsData) { this.label = label; this.refName = refName; + this.eventCreatedOn = eventCreatedOn; this.revisionsData = revisionsData; } @@ -37,6 +40,10 @@ return refName; } + public long getEventCreatedOn() { + return eventCreatedOn; + } + public RevisionData[] getRevisionsData() { return revisionsData; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java index 6000eb9..1991260 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -17,6 +17,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.Project.NameKey; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import java.io.IOException; @@ -46,14 +47,19 @@ HttpResult updateHead(Project.NameKey project, String newHead, URIish apiUri) throws IOException; HttpResult callSendObject( - Project.NameKey project, + NameKey project, String refName, + long eventCreatedOn, boolean isDelete, RevisionData revisionData, URIish targetUri) throws ClientProtocolException, IOException; HttpResult callSendObjects( - Project.NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri) + NameKey project, + String refName, + long eventCreatedOn, + List<RevisionData> revisionData, + URIish targetUri) throws ClientProtocolException, IOException; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index 0afbecf..c3fec5e 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -171,8 +171,9 @@ */ @Override public HttpResult callSendObject( - Project.NameKey project, + NameKey project, String refName, + long eventCreatedOn, boolean isDelete, @Nullable RevisionData revisionData, URIish targetUri) @@ -184,7 +185,7 @@ } else { requireNull(revisionData, "DELETE ref-updates cannot be associated with a RevisionData"); } - RevisionInput input = new RevisionInput(instanceId, refName, revisionData); + RevisionInput input = new RevisionInput(instanceId, refName, eventCreatedOn, revisionData); String url = formatUrl(targetUri.toString(), project, "apply-object"); @@ -196,14 +197,20 @@ @Override public HttpResult callSendObjects( - NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri) + NameKey project, + String refName, + long eventCreatedOn, + List<RevisionData> revisionData, + URIish targetUri) throws ClientProtocolException, IOException { if (revisionData.size() == 1) { - return callSendObject(project, refName, false, revisionData.get(0), targetUri); + return callSendObject( + project, refName, eventCreatedOn, false, revisionData.get(0), targetUri); } RevisionData[] inputData = new RevisionData[revisionData.size()]; - RevisionsInput input = new RevisionsInput(instanceId, refName, revisionData.toArray(inputData)); + RevisionsInput input = + new RevisionsInput(instanceId, refName, eventCreatedOn, revisionData.toArray(inputData)); String url = formatUrl(targetUri.toString(), project, "apply-objects"); HttpPost post = new HttpPost(url);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index d989c4d..0a48eaf 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import static java.nio.file.Files.createTempDirectory; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.anyString; @@ -137,10 +138,12 @@ when(fetchClientFactory.create(any())).thenReturn(fetchRestApiClient); lenient() - .when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any())) + .when( + fetchRestApiClient.callSendObject( + any(), anyString(), anyLong(), anyBoolean(), any(), any())) .thenReturn(httpResult); lenient() - .when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + .when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(fetchHttpResult); when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult); @@ -172,7 +175,7 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); } @Test @@ -183,7 +186,8 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient, never()).callSendObjects(any(), anyString(), any(), any()); + verify(fetchRestApiClient, never()) + .callSendObjects(any(), anyString(), anyLong(), any(), any()); } @Test @@ -218,7 +222,7 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); } @Test @@ -255,7 +259,7 @@ when(httpResult.isSuccessful()).thenReturn(false); when(httpResult.isParentObjectMissing()).thenReturn(true); - when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); objectUnderTest.onEvent(event); @@ -271,13 +275,13 @@ when(httpResult.isSuccessful()).thenReturn(false, true); when(httpResult.isParentObjectMissing()).thenReturn(true, false); - when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); objectUnderTest.onEvent(event); verify(fetchRestApiClient, times(2)) - .callSendObjects(any(), anyString(), revisionsDataCaptor.capture(), any()); + .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any()); List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); assertThat(revisionsDataValues).hasSize(2);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java index 814ba76..e575eb9 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -48,6 +48,9 @@ @RunWith(MockitoJUnitRunner.class) public class ApplyObjectActionTest { + + private static final long DUMMY_EVENT_TIMESTAMP = 1684875939; + ApplyObjectAction applyObjectAction; String label = "instance-2-label"; String url = "file:///gerrit-host/instance-1/git/${name}.git"; @@ -55,7 +58,6 @@ String refMetaName = "refs/meta/version"; String location = "http://gerrit-host/a/config/server/tasks/08d173e9"; int taskId = 1234; - private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb"; @@ -95,7 +97,8 @@ @Test public void shouldReturnCreatedResponseCode() throws RestApiException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); Response<?> response = applyObjectAction.apply(projectResource, inputParams); @@ -109,6 +112,7 @@ new RevisionInput( label, refMetaName, + DUMMY_EVENT_TIMESTAMP, createSampleRevisionDataBlob( new RevisionObjectData(sampleBlobObjectId, Constants.OBJ_BLOB, blobData))); @@ -120,7 +124,8 @@ @SuppressWarnings("cast") @Test public void shouldReturnSourceUrlAndrefNameAsAResponseBody() throws Exception { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); Response<?> response = applyObjectAction.apply(projectResource, inputParams); assertThat((RevisionInput) response.value()).isEqualTo(inputParams); @@ -128,28 +133,32 @@ @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenMissingLabel() throws Exception { - RevisionInput inputParams = new RevisionInput(null, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(null, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenEmptyLabel() throws Exception { - RevisionInput inputParams = new RevisionInput("", refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput("", refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenMissingRefName() throws Exception { - RevisionInput inputParams = new RevisionInput(label, null, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, null, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenEmptyRefName() throws Exception { - RevisionInput inputParams = new RevisionInput(label, "", createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, "", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @@ -161,7 +170,8 @@ RevisionObjectData treeData = new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {}); RevisionInput inputParams = - new RevisionInput(label, refName, createSampleRevisionData(commitData, treeData)); + new RevisionInput( + label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData(commitData, treeData)); applyObjectAction.apply(projectResource, inputParams); } @@ -172,7 +182,8 @@ new RevisionObjectData( sampleCommitObjectId, Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); RevisionInput inputParams = - new RevisionInput(label, refName, createSampleRevisionData(commitData, null)); + new RevisionInput( + label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData(commitData, null)); applyObjectAction.apply(projectResource, inputParams); } @@ -180,7 +191,8 @@ @Test(expected = AuthException.class) public void shouldThrowAuthExceptionWhenCallFetchActionCapabilityNotAssigned() throws RestApiException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); when(preConditions.canCallFetchApi()).thenReturn(false); @@ -190,7 +202,8 @@ @Test(expected = ResourceConflictException.class) public void shouldThrowResourceConflictExceptionWhenMissingParentObject() throws RestApiException, IOException, RefUpdateException, MissingParentObjectException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); doThrow( new MissingParentObjectException(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java index a2389d7..cdb238e 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -74,6 +74,7 @@ String pluginName = "pull-replication"; String instanceId = "Replication"; String refName = RefNames.REFS_HEADS + "master"; + long eventCreatedOn = 1684875939; String expectedPayload = "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":false}"; @@ -87,7 +88,9 @@ String blobObjectId = "bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8"; String expectedSendObjectPayload = - "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"event_created_on\":" + + eventCreatedOn + + ",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + commitObjectId + "\",\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"sha1\":\"" + treeObjectId @@ -265,6 +268,7 @@ objectUnderTest.callSendObject( Project.nameKey("test_repo"), refName, + eventCreatedOn, IS_REF_UPDATE, createSampleRevisionData(), new URIish(api)); @@ -287,6 +291,7 @@ objectUnderTest.callSendObject( Project.nameKey("test_repo"), refName, + eventCreatedOn, IS_REF_UPDATE, createSampleRevisionData(), new URIish(api));