Feature: Retry with exponential backoff when Octopus Deploy is unavailable (#186)

* Feature: Retry with exponential backoff when Octopus Deploy is unavailable (#186)

When the Octopus Deploy server is unavailable (connection refused, 502/503/504,
DNS failure, timeouts), the TeamCity plugin now retries CLI execution with
exponential backoff instead of failing the build immediately.

- OctopusErrorClassifier: distinguishes transient errors (retry) from permanent
  errors (fail fast) by parsing CLI stdout/stderr patterns for both .NET and Go CLIs
- OctopusCliRetryExecutor: retry loop with exponential backoff (5s initial,
  2m max per attempt, 15m total timeout), configurable via env vars
  (OCTOPUS_RETRY_ENABLED, OCTOPUS_RETRY_TIMEOUT, OCTOPUS_RETRY_INITIAL_DELAY,
  OCTOPUS_RETRY_MAX_DELAY)
- CaptureWriter: Output.Writer decorator for capturing CLI output for error
  classification in the legacy build process path
- Both legacy (OctopusBuildProcess) and new CLI (CLIBuildProcess) paths use the
  shared retry executor

* fix: resolve symlink mismatch in Zip Slip check on macOS

The extractTarGzResource() Zip Slip security check compared
destDir.resolve(entry).normalize() against destDir.toRealPath(). On macOS,
/var is a symlink to /private/var, so toRealPath() resolved to
/private/var/folders/... while the entry path kept /var/folders/...,
causing startsWith() to fail and throw a false-positive Zip Slip error.

Fix: resolve destDir to its real path once upfront, then use the resolved
path consistently for both entry resolution and the security check.

* docs: add code comments explaining retry workflow for debugging

Adds Javadoc and inline comments to the retry infrastructure classes
(OctopusCliRetryExecutor, OctopusErrorClassifier, CaptureWriter) and
both build process integration points (OctopusBuildProcess, CLIBuildProcess)
to help engineers unfamiliar with the codebase understand the retry flow.

Author: cstrzadala

octopus-agent/src/main/java/octopus/teamcity/agent/CaptureWriter.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2000-2012 Octopus Deploy Pty. Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package octopus.teamcity.agent;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Decorates an {@link Output.Writer} to capture all written lines for later inspection, while still
+ * forwarding each line to the delegate (typically the TeamCity build log).
+ *
+ * <p>Used by the legacy CLI path ({@link OctopusBuildProcess}) to collect stdout/stderr so that the
+ * combined output can be passed to {@link OctopusErrorClassifier} for transient-vs-permanent
+ * failure classification after the CLI process exits.
+ *
+ * <p>The Go CLI path ({@link octopus.teamcity.agent.cli.CLIBuildProcess}) does not use this class
+ * because it merges stdout/stderr via {@code ProcessBuilder.redirectErrorStream(true)} and captures
+ * output inline with a StringBuilder.
+ *
+ * <p>Not thread-safe; callers must ensure writes are complete before calling {@link
+ * #getCapturedOutput()}.
+ */
+public class CaptureWriter implements Output.Writer {
+  private final Output.Writer delegate;
+  private final List<String> lines = new ArrayList<String>();
+
+  public CaptureWriter(Output.Writer delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public void write(String text) {
+    delegate.write(text);
+    lines.add(text);
+  }
+
+  public String getCapturedOutput() {
+    StringBuilder sb = new StringBuilder();
+    for (String line : lines) {
+      sb.append(line).append('\n');
+    }
+    return sb.toString();
+  }
+}

octopus-agent/src/main/java/octopus/teamcity/agent/EmbeddedResourceExtractor.java

@@ -140,6 +140,7 @@ public void extractCliTo(String destinationPath, String octopusVersion, String o
 
   public void extractTarGzResource(String resourcePath, Path destDir) throws IOException {
     Files.createDirectories(destDir);
+    Path realDestDir = destDir.toRealPath();
 
     try (InputStream fi = getClass().getResourceAsStream(resourcePath)) {
       if (fi == null) {
@@ -153,9 +154,9 @@ public void extractTarGzResource(String resourcePath, Path destDir) throws IOExc
         TarArchiveEntry entry;
         byte[] buffer = new byte[4096];
         while ((entry = tarIn.getNextTarEntry()) != null) {
-          Path entryPath = destDir.resolve(entry.getName()).normalize();
+          Path entryPath = realDestDir.resolve(entry.getName()).normalize();
 
-          if (!entryPath.startsWith(destDir.toRealPath())) {
+          if (!entryPath.startsWith(realDestDir)) {
             throw new IOException(
                 "Zip Slip detected! Entry is outside of the target directory: " + entry.getName());
           }

octopus-agent/src/main/java/octopus/teamcity/agent/OctopusBuildProcess.java

@@ -39,12 +39,12 @@
 public abstract class OctopusBuildProcess implements BuildProcess {
   private final AgentRunningBuild runningBuild;
   private final BuildRunnerContext context;
-  private Process process;
+  private volatile Process process;
   private File extractedTo;
-  private Output.ReaderThread standardError;
-  private Output.ReaderThread standardOutput;
   private boolean isFinished;
   private final BuildProgressLogger logger;
+  private OctopusCliRetryExecutor.CliResult cliResult;
+  private OctopusCliRetryExecutor retryExecutor;
 
   protected OctopusBuildProcess(
       @NotNull AgentRunningBuild runningBuild, @NotNull BuildRunnerContext context) {
@@ -58,6 +58,9 @@ protected OctopusBuildProcess(
   public void start() throws RunBuildException {
     extractOctoExe();
 
+    retryExecutor =
+        new OctopusCliRetryExecutor(logger, context.getBuildParameters().getEnvironmentVariables());
+
     OctopusCommandBuilder arguments = createCommand();
     startOcto(arguments);
   }
@@ -118,48 +121,83 @@ private void startOcto(final OctopusCommandBuilder command) throws RunBuildExcep
     logger.progressMessage(getLogMessage());
 
     try {
-      final String octopusVersion = getSelectedOctopusVersion();
-
-      final ArrayList<String> arguments = new ArrayList<>();
-      if (useExe) {
-        arguments.add(new File(extractedTo, octopusVersion + "/octo.exe").getAbsolutePath());
-      } else if (useOcto) {
-        arguments.add("octo");
-      } else {
-        arguments.add("dotnet");
-        String dllPath = new File(extractedTo, octopusVersion + "/Core/octo.dll").getAbsolutePath();
-        arguments.add(dllPath);
-      }
-
-      arguments.addAll(Arrays.asList(realCommand));
-
-      final String extensionVersion = getClass().getPackage().getImplementationVersion();
-
-      final ProcessBuilder builder = new ProcessBuilder();
-
-      Map<String, String> programEnvironmentVariables =
-          context.getBuildParameters().getEnvironmentVariables();
-      Map<String, String> environment = builder.environment();
-      environment.put("OCTOEXTENSION", extensionVersion);
-      environment.putAll(programEnvironmentVariables);
+      final Boolean finalUseExe = useExe;
+      final Boolean finalUseOcto = useOcto;
 
-      process = builder.command(arguments).directory(context.getWorkingDirectory()).start();
+      // Execute the CLI process, wrapped in retry logic. On transient failures (connection
+      // refused, 502/503/504, DNS errors, timeouts), the retry executor will re-launch the
+      // entire CLI process with exponential backoff. The CaptureWriter on stdout/stderr
+      // collects output so the error classifier can inspect it after each attempt.
+      cliResult =
+          retryExecutor.executeWithRetry(
+              () -> executeOctoProcess(realCommand, finalUseExe, finalUseOcto));
 
-      final LoggingProcessListener listener = new LoggingProcessListener(logger);
+      logger.message("octo exit code: " + cliResult.getExitCode());
 
-      standardError = new Output.ReaderThread(process.getErrorStream(), listener::onErrorOutput);
-      standardOutput =
-          new Output.ReaderThread(process.getInputStream(), listener::onStandardOutput);
-
-      standardError.start();
-      standardOutput.start();
     } catch (IOException e) {
       final String message = "Error from octo: " + e.getMessage();
       Logger.getInstance(getClass().getName()).error(message, e);
       throw new RunBuildException(message);
+    } catch (InterruptedException e) {
+      isFinished = true;
+      final String message = "Unable to wait for octo: " + e.getMessage();
+      Logger.getInstance(getClass().getName()).error(message, e);
+      throw new RunBuildException(message);
     }
   }
 
+  private OctopusCliRetryExecutor.CliResult executeOctoProcess(
+      String[] realCommand, boolean useExe, boolean useOcto)
+      throws IOException, InterruptedException {
+    final String octopusVersion = getSelectedOctopusVersion();
+
+    final ArrayList<String> arguments = new ArrayList<>();
+    if (useExe) {
+      arguments.add(new File(extractedTo, octopusVersion + "/octo.exe").getAbsolutePath());
+    } else if (useOcto) {
+      arguments.add("octo");
+    } else {
+      arguments.add("dotnet");
+      String dllPath = new File(extractedTo, octopusVersion + "/Core/octo.dll").getAbsolutePath();
+      arguments.add(dllPath);
+    }
+
+    arguments.addAll(Arrays.asList(realCommand));
+
+    final String extensionVersion = getClass().getPackage().getImplementationVersion();
+
+    final ProcessBuilder builder = new ProcessBuilder();
+
+    Map<String, String> programEnvironmentVariables =
+        context.getBuildParameters().getEnvironmentVariables();
+    Map<String, String> environment = builder.environment();
+    environment.put("OCTOEXTENSION", extensionVersion);
+    environment.putAll(programEnvironmentVariables);
+
+    process = builder.command(arguments).directory(context.getWorkingDirectory()).start();
+
+    final LoggingProcessListener listener = new LoggingProcessListener(logger);
+
+    CaptureWriter stderrCapture = new CaptureWriter(listener::onErrorOutput);
+    CaptureWriter stdoutCapture = new CaptureWriter(listener::onStandardOutput);
+
+    Output.ReaderThread standardError =
+        new Output.ReaderThread(process.getErrorStream(), stderrCapture);
+    Output.ReaderThread standardOutput =
+        new Output.ReaderThread(process.getInputStream(), stdoutCapture);
+
+    standardError.start();
+    standardOutput.start();
+
+    int exitCode = process.waitFor();
+
+    standardError.join();
+    standardOutput.join();
+
+    String combinedOutput = stdoutCapture.getCapturedOutput() + stderrCapture.getCapturedOutput();
+    return new OctopusCliRetryExecutor.CliResult(exitCode, combinedOutput);
+  }
+
   private String getSelectedOctopusVersion() {
     final Map<String, String> parameters = getContext().getRunnerParameters();
     final OctopusConstants constants = OctopusConstants.Instance;
@@ -199,26 +237,10 @@ public void interrupt() {
   @Override
   @NotNull
   public BuildFinishedStatus waitFor() throws RunBuildException {
-    int exitCode;
-
-    try {
-      exitCode = process.waitFor();
-
-      standardError.join();
-      standardOutput.join();
-
-      logger.message("octo exit code: " + exitCode);
-      logger.activityFinished("Octopus Deploy", BLOCK_TYPE_BUILD_STEP);
-
-      isFinished = true;
-    } catch (InterruptedException e) {
-      isFinished = true;
-      final String message = "Unable to wait for octo: " + e.getMessage();
-      Logger.getInstance(getClass().getName()).error(message, e);
-      throw new RunBuildException(message);
-    }
+    logger.activityFinished("Octopus Deploy", BLOCK_TYPE_BUILD_STEP);
+    isFinished = true;
 
-    if (exitCode == 0) {
+    if (cliResult != null && cliResult.getExitCode() == 0) {
       return BuildFinishedStatus.FINISHED_SUCCESS;
     }