Always fallback to fetch when ApplyObject REST-API fails

Previously the Git fetch was invoked only when the ApplyObject
failed because of missing parent commit. However, there are other
cases when the ApplyObject may fail and *must* be covered by
a fallback to Git fetch.

One example of these failures is a forced update of a branch
or a lock failure.

Falling back to a fetch allows to complete the replication anyway
by relying on the Git protocol to resolve the complete fetch.

Change-Id: I2a7b94d0088c65dbade3e3bdc0beae14d46e6410

src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index 431255c..3db867b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -232,9 +232,19 @@
     CallFunction call = getCallFunction(project, objectId, refName, isDelete, state);
 
     return (source) -> {
+      boolean callSuccessful;
       try {
-        call.call(source);
-      } catch (MissingParentObjectException e) {
+        callSuccessful = call.call(source);
+      } catch (Exception e) {
+        repLog.warn(
+            String.format(
+                "Failed to apply object %s on project %s:%s, falling back to git fetch",
+                objectId.name(), project, refName),
+            e);
+        callSuccessful = false;
+      }
+
+      if (!callSuccessful) {
         callFetch(source, project, refName, state);
       }
     };
@@ -274,7 +284,7 @@
     return (source) -> callFetch(source, project, refName, state);
   }
 
-  private void callSendObject(
+  private boolean callSendObject(
       Source source,
       Project.NameKey project,
       String refName,
@@ -282,6 +292,7 @@
       RevisionData revision,
       ReplicationState state)
       throws MissingParentObjectException {
+    boolean resultIsSuccessful = true;
     if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
       for (String apiUrl : source.getApis()) {
         try {
@@ -316,6 +327,8 @@
                   project, refName, source.getRemoteConfigName());
             }
           }
+
+          resultIsSuccessful &= result.isSuccessful();
         } catch (URISyntaxException e) {
           repLog.warn(
               "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}",
@@ -325,6 +338,7 @@
               revision,
               e);
           stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
+          resultIsSuccessful = false;
         } catch (IOException e) {
           repLog.warn(
               "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}",
@@ -340,13 +354,17 @@
                   apiUrl, e.getMessage()),
               e,
               state);
+          resultIsSuccessful = false;
         }
       }
     }
+
+    return resultIsSuccessful;
   }
 
-  private void callFetch(
+  private boolean callFetch(
       Source source, Project.NameKey project, String refName, ReplicationState state) {
+    boolean resultIsSuccessful = true;
     if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
       for (String apiUrl : source.getApis()) {
         try {
@@ -374,8 +392,11 @@
                     apiUrl, result.getMessage().orElse("unknown")),
                 state);
           }
+
+          resultIsSuccessful &= result.isSuccessful();
         } catch (URISyntaxException e) {
           stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
+          resultIsSuccessful = false;
         } catch (Exception e) {
           stateLog.error(
               String.format(
@@ -384,9 +405,12 @@
                   apiUrl, e.getMessage()),
               e,
               state);
+          resultIsSuccessful = false;
         }
       }
     }
+
+    return resultIsSuccessful;
   }
 
   public boolean retry(int attempt, int maxRetries) {
@@ -453,6 +477,6 @@
 
   @FunctionalInterface
   private interface CallFunction {
-    void call(Source source) throws MissingParentObjectException;
+    boolean call(Source source) throws MissingParentObjectException;
   }
 }

src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java

diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 56b6ad1..b8ff766 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -80,6 +80,8 @@
   @Mock AccountInfo accountInfo;
   @Mock RevisionReader revReader;
   @Mock RevisionData revisionData;
+  @Mock HttpResult successfulHttpResult;
+  @Mock HttpResult fetchHttpResult;
   @Mock HttpResult httpResult;
   ApplyObjectMetrics applyObjectMetrics;
   FetchReplicationMetrics fetchMetrics;
@@ -110,8 +112,12 @@
     when(fetchClientFactory.create(any())).thenReturn(fetchRestApiClient);
     when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any()))
         .thenReturn(httpResult);
-    when(fetchRestApiClient.callFetch(any(), anyString(), any(), anyLong())).thenReturn(httpResult);
+    when(fetchRestApiClient.callFetch(any(), anyString(), any(), anyLong()))
+        .thenReturn(fetchHttpResult);
+    when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult);
+    when(successfulHttpResult.isSuccessful()).thenReturn(true);
     when(httpResult.isSuccessful()).thenReturn(true);
+    when(fetchHttpResult.isSuccessful()).thenReturn(true);
     when(httpResult.isProjectMissing(any())).thenReturn(false);
 
     applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker());