Handle kill command in replication tasks also while running Issue description: When replication tasks is scheduled and it gets killed with ssh kill command further replication attempts get ignored and as a result replication to particular mirror is no longer possible (Gerrit needs to be restarted or plugin reloaded). Solution: Step1: Derive PushOne from CanceledWhileRunning interface so that it gets informed that it was canceled while operation was in progress (which results in any resource operation failure). Step2: check in resource failure catch if it occurred while task was canceled. If that is the case don't reschedule. Change-Id: I4882fe1334efedf060a5ff5fef89d9d33c88d75c Signed-off-by: Jacek Centkowski <geminica.programs@gmail.com>
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index 8164c09..3e615d5 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -269,6 +269,13 @@ } } + void pushWasCanceled(PushOne pushOp) { + synchronized (stateLock) { + URIish uri = pushOp.getURI(); + pending.remove(uri); + } + } + /** * It schedules again a PushOp instance. * <p> @@ -334,8 +341,7 @@ // when notifying it is starting (with pending lock protection), // it will see it was canceled and then it will do nothing with // pending list and it will not execute its run implementation. - - pendingPushOp.cancel(); + pendingPushOp.canceledByReplication(); pending.remove(uri); pushOp.addRefs(pendingPushOp.getRefs());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index 44399b1..48586eb 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -32,6 +32,7 @@ import com.google.gerrit.server.git.SearchingChangeCacheImpl; import com.google.gerrit.server.git.TagCache; import com.google.gerrit.server.git.VisibleRefFilter; +import com.google.gerrit.server.git.WorkQueue.CanceledWhileRunning; import com.google.gerrit.server.notedb.ChangeNotes; import com.google.gerrit.server.project.NoSuchProjectException; import com.google.gerrit.server.project.ProjectControl; @@ -73,6 +74,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicBoolean; /** * A push to remote operation started by {@link GitReferenceUpdatedListener}. @@ -80,7 +82,7 @@ * Instance members are protected by the lock within PushQueue. Callers must * take that lock to ensure they are working with a current view of the object. */ -class PushOne implements ProjectRunnable { +class PushOne implements ProjectRunnable, CanceledWhileRunning { private final ReplicationStateListener stateLog; static final String ALL_REFS = "..all.."; static final String ID_MDC_KEY = "pushOneId"; @@ -115,6 +117,7 @@ private final int id; private final long createdAt; private final ReplicationMetrics metrics; + private final AtomicBoolean canceledWhileRunning; @Inject PushOne(GitRepositoryManager grm, @@ -150,6 +153,20 @@ stateLog = sl; createdAt = System.nanoTime(); metrics = m; + canceledWhileRunning = new AtomicBoolean(false); + } + + @Override + public void cancel() { + repLog.info("Replication {} was canceled", getURI()); + canceledByReplication(); + pool.pushWasCanceled(this); + } + + @Override + public void setCanceledWhileRunning() { + repLog.info("Replication {} was canceled while being executed", getURI()); + canceledWhileRunning.set(true); } @Override @@ -186,7 +203,7 @@ retryCount++; } - void cancel() { + void canceledByReplication() { canceled = true; } @@ -323,7 +340,6 @@ createRepository(); } catch (NotSupportedException e) { stateLog.error("Cannot replicate to " + uri, e, getStatesAsArray()); - } catch (TransportException e) { Throwable cause = e.getCause(); if (cause instanceof JSchException @@ -337,15 +353,23 @@ // The remote push operation should be retried. if (lockRetryCount <= maxLockRetries) { - pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR); + if (canceledWhileRunning.get()) { + logCanceledWhileRunningException(e); + } else { + pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR); + } } else { repLog.error("Giving up after " + lockRetryCount + " of this error during replication to " + e.getMessage()); } } else { - repLog.error("Cannot replicate to " + uri, e); - // The remote push operation should be retried. - pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR); + if (canceledWhileRunning.get()) { + logCanceledWhileRunningException(e); + } else { + repLog.error("Cannot replicate to " + uri, e); + // The remote push operation should be retried. + pool.reschedule(this, Destination.RetryReason.TRANSPORT_ERROR); + } } } catch (IOException e) { stateLog.error("Cannot replicate to " + uri, e, getStatesAsArray()); @@ -359,6 +383,11 @@ } } + private void logCanceledWhileRunningException(TransportException e) { + repLog.info("Cannot replicate to " + uri + "." + + " It was canceled while running", e); + } + private void createRepository() { if (pool.isCreateMissingRepos()) { try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java index fc11773..223424b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -113,7 +113,7 @@ int discarded = config.shutdown(); if (discarded > 0) { repLog.warn(String.format( - "Cancelled %d replication events during shutdown", discarded)); + "Canceled %d replication events during shutdown", discarded)); } }