From 8d66686a7b8ca88a91129bdbc36efa6427608778 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Thu, 26 Feb 2026 14:12:48 +0000
Subject: [PATCH 01/13] qhouyee/83/backup-service: bump SNAPSHOT version

---
 stack-clients/docker-compose.yml       | 2 +-
 stack-clients/pom.xml                  | 2 +-
 stack-data-uploader/docker-compose.yml | 2 +-
 stack-data-uploader/pom.xml            | 4 ++--
 stack-manager/docker-compose.yml       | 2 +-
 stack-manager/pom.xml                  | 4 ++--
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/stack-clients/docker-compose.yml b/stack-clients/docker-compose.yml
index 22782cfe..76c37239 100644
--- a/stack-clients/docker-compose.yml
+++ b/stack-clients/docker-compose.yml
@@ -1,6 +1,6 @@
 services:
   stack-client:
-    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.57.0
+    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.58.0-backup-service-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password
diff --git a/stack-clients/pom.xml b/stack-clients/pom.xml
index 7250a7bd..adb1c8dc 100644
--- a/stack-clients/pom.xml
+++ b/stack-clients/pom.xml
@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-clients</artifactId>
-    <version>1.57.0</version>
+    <version>1.58.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Clients</name>
     <url>https://theworldavatar.io</url>
diff --git a/stack-data-uploader/docker-compose.yml b/stack-data-uploader/docker-compose.yml
index b3fe8efb..e7bece24 100644
--- a/stack-data-uploader/docker-compose.yml
+++ b/stack-data-uploader/docker-compose.yml
@@ -1,6 +1,6 @@
 services:
   stack-data-uploader:
-    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.57.0
+    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.58.0-backup-service-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password
diff --git a/stack-data-uploader/pom.xml b/stack-data-uploader/pom.xml
index f304441b..edc916dd 100644
--- a/stack-data-uploader/pom.xml
+++ b/stack-data-uploader/pom.xml
@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-data-uploader</artifactId>
-    <version>1.57.0</version>
+    <version>1.58.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Data Uploader</name>
     <url>https://theworldavatar.io</url>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.cmclinnovations</groupId>
             <artifactId>stack-clients</artifactId>
-            <version>1.57.0</version>
+            <version>1.58.0-backup-service-SNAPSHOT</version>
         </dependency>
 
         <dependency>
diff --git a/stack-manager/docker-compose.yml b/stack-manager/docker-compose.yml
index e4914f5b..f6e4cdcf 100644
--- a/stack-manager/docker-compose.yml
+++ b/stack-manager/docker-compose.yml
@@ -1,6 +1,6 @@
 services:
   stack-manager:
-    image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.57.0
+    image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.58.0-backup-service-SNAPSHOT
     environment:
       EXTERNAL_PORT: "${EXTERNAL_PORT-3838}"
       STACK_BASE_DIR: "${STACK_BASE_DIR}"
diff --git a/stack-manager/pom.xml b/stack-manager/pom.xml
index d63a4fce..0239a225 100644
--- a/stack-manager/pom.xml
+++ b/stack-manager/pom.xml
@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-manager</artifactId>
-    <version>1.57.0</version>
+    <version>1.58.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Manager</name>
     <url>https://theworldavatar.io</url>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.cmclinnovations</groupId>
             <artifactId>stack-clients</artifactId>
-            <version>1.57.0</version>
+            <version>1.58.0-backup-service-SNAPSHOT</version>
         </dependency>
 
         <dependency>

From 9fde447028fe9fe95b1ea8aba2437a73b500c665 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 3 Mar 2026 14:04:10 +0000
Subject: [PATCH 02/13] qhouyee/83/backup-service: added kopia stack service
 config as a default

---
 .../stack/services/built-ins/kopia.json       | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json

diff --git a/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
new file mode 100644
index 00000000..3c8e9484
--- /dev/null
+++ b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
@@ -0,0 +1,21 @@
+{
+  "type": "kopia",
+  "ServiceSpec": {
+    "Name": "kopia",
+    "TaskTemplate": {
+      "ContainerSpec": {
+        "Image": "kopia/kopia:latest",
+        "Command": [
+          "tail",
+          "-f",
+          "/dev/null"
+        ],
+        "Secrets": [
+          {
+            "SecretName": "kopia_password"
+          }
+        ]
+      }
+    }
+  }
+}
\ No newline at end of file

From e67d6a0a78417a4a33c1d08c07e94f08862e46c6 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 3 Mar 2026 14:08:33 +0000
Subject: [PATCH 03/13] qhouyee/83/backup-service: added kopia service to set
 up filesystem repository on container creation

---
 .../stack/clients/utils/JsonHelper.java       | 21 ++++++++
 .../stack/services/KopiaService.java          | 48 +++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java
index 1a887cd4..a5389b35 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java
@@ -1,11 +1,15 @@
 package com.cmclinnovations.stack.clients.utils;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.text.MessageFormat;
 
 import javax.annotation.Nonnull;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.json.JsonMapper;
 
@@ -31,4 +35,21 @@ public static final String handleFileValues(String value) {
         }
         return value;
     }
+
+    /**
+     * Parses a JSON file into a JSON object.
+     * 
+     * @param file Path to the file.
+     */
+    public static final JsonNode readFile(String file) {
+        try {
+            String fileContents = Files.readString(Path.of(file));
+            return getMapper().readTree(fileContents);
+        } catch (MalformedURLException ex) {
+            throw new IllegalArgumentException(MessageFormat.format("Invalid file path: {0}", file));
+        } catch (IOException ex) {
+            throw new UncheckedIOException(
+                    MessageFormat.format("Failed to read file: {0}", ex.getMessage()), ex);
+        }
+    }
 }
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
new file mode 100644
index 00000000..8348c6ef
--- /dev/null
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -0,0 +1,48 @@
+package com.cmclinnovations.stack.services;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cmclinnovations.stack.clients.utils.JsonHelper;
+import com.cmclinnovations.stack.services.config.ServiceConfig;
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class KopiaService extends ContainerService {
+    public static final String TYPE = "kopia";
+
+    private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+    private static final String STORAGE_KEY = "storage";
+
+    private static final String PASSWORD_SECRETS_FLAG = "--password=$(cat /run/secrets/kopia_password)";
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class);
+
+    public KopiaService(String stackName, ServiceConfig config) {
+        super(stackName, config);
+    }
+
+    @Override
+    public void doFirstTimePostStartUpConfiguration() {
+        JsonNode storageConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+        String storageType = storageConfig.get("type").asText();
+        JsonNode storageConfigOptions = storageConfig.get("config");
+        List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", "create", storageType));
+        switch (storageType) {
+            case "filesystem":
+                String storagePath = storageConfigOptions.get("path").asText();
+                createRepoCommandArgs.add("--path=" + storagePath);
+                break;
+            default:
+                LOGGER.warn(
+                        "Unsupported storage type '{}' for automatic stack setup. Please manually configure the storage",
+                        storageType);
+                break;
+        }
+
+        createRepoCommandArgs.add(PASSWORD_SECRETS_FLAG);
+        super.executeCommand(createRepoCommandArgs.toArray(new String[0]));
+    }
+}

From 1bf4ddb5dbbc48b1bb65df19eda517e3336774d3 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 3 Mar 2026 16:20:29 +0000
Subject: [PATCH 04/13] qhouyee/83/backup-service: mount all stack volumes into
 kopia container for backups

---
 .../stack/clients/docker/DockerClient.java    | 17 +++++++++--
 .../stack/services/ContainerService.java      |  4 +++
 .../stack/services/KopiaService.java          | 29 +++++++++++++++++++
 3 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java
index 85391b99..41766d3f 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java
@@ -15,12 +15,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
-import java.util.NoSuchElementException;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -40,9 +40,11 @@
 import com.github.dockerjava.api.command.InspectContainerCmd;
 import com.github.dockerjava.api.command.InspectExecCmd;
 import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
 import com.github.dockerjava.api.command.ListConfigsCmd;
 import com.github.dockerjava.api.command.ListContainersCmd;
 import com.github.dockerjava.api.command.ListSecretsCmd;
+import com.github.dockerjava.api.command.ListVolumesCmd;
 import com.github.dockerjava.api.command.RemoveConfigCmd;
 import com.github.dockerjava.api.command.RemoveSecretCmd;
 import com.github.dockerjava.api.model.Config;
@@ -553,7 +555,7 @@ public boolean isContainerUp(String containerName) {
 
     public String getContainerId(String containerName) {
         return getContainer(containerName).map(Container::getId)
-                .orElseThrow(() -> new NoSuchElementException("Cannot get container "+containerName+"."));
+                .orElseThrow(() -> new NoSuchElementException("Cannot get container " + containerName + "."));
     }
 
     private Map<String, List<String>> convertToConfigFilterMap(String configName, Map<String, String> labelMap) {
@@ -593,6 +595,17 @@ public Optional<Config> getConfig(List<Config> configs, String configName) {
 
     }
 
+    public List<InspectVolumeResponse> getVolumes() {
+        try (ListVolumesCmd listVolumesCmd = internalClient.listVolumesCmd()) {
+            return listVolumesCmd
+                    .exec()
+                    .getVolumes()
+                    .stream()
+                    .filter(v -> v.getName() != null && v.getName().startsWith(StackClient.getStackName()))
+                    .collect(Collectors.toList());
+        }
+    }
+
     public List<Config> getConfigs() {
         try (ListConfigsCmd listConfigsCmd = internalClient.listConfigsCmd()) {
             return listConfigsCmd
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java
index e07c4b8d..0dc55104 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java
@@ -25,6 +25,7 @@
 import com.cmclinnovations.stack.services.config.ServiceConfig;
 import com.github.dockerjava.api.command.InspectImageCmd;
 import com.github.dockerjava.api.command.InspectImageResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
 import com.github.dockerjava.api.model.ContainerSpec;
 import com.github.dockerjava.api.model.ServiceSpec;
 import com.github.dockerjava.api.model.TaskSpec;
@@ -240,4 +241,7 @@ public String getDNSIPAddress() {
         return dockerClient.getDNSIPAddress();
     }
 
+    protected List<InspectVolumeResponse> getVolumes() {
+        return dockerClient.getVolumes();
+    }
 }
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index 8348c6ef..62266ff9 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -6,14 +6,20 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.cmclinnovations.stack.clients.core.StackClient;
 import com.cmclinnovations.stack.clients.utils.JsonHelper;
 import com.cmclinnovations.stack.services.config.ServiceConfig;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.model.ContainerSpec;
+import com.github.dockerjava.api.model.Mount;
+import com.github.dockerjava.api.model.MountType;
 
 public class KopiaService extends ContainerService {
     public static final String TYPE = "kopia";
 
     private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+    private static final String VOLUME_DIR = "/data/";
     private static final String STORAGE_KEY = "storage";
 
     private static final String PASSWORD_SECRETS_FLAG = "--password=$(cat /run/secrets/kopia_password)";
@@ -24,6 +30,29 @@ public KopiaService(String stackName, ServiceConfig config) {
         super(stackName, config);
     }
 
+    @Override
+    public void doPreStartUpConfiguration() {
+        // Mount all volumes on the stack (except kopia) for backups
+        List<InspectVolumeResponse> volumes = super.getVolumes();
+        List<Mount> mounts = new ArrayList<>();
+
+        for (InspectVolumeResponse vol : volumes) {
+            if (!vol.getName().endsWith(TYPE)) {
+                // Remove stack name prefix (STACK_) from volume name
+                String volName = vol.getName().substring(StackClient.getStackName().length() + 1);
+                String destinationPath = VOLUME_DIR + volName;
+                mounts.add(new Mount()
+                        .withType(MountType.VOLUME) // Use Docker volume for named volumes
+                        .withSource(volName)
+                        .withTarget(destinationPath)
+                        .withReadOnly(false));
+            }
+        }
+
+        ContainerSpec containerSpec = super.getContainerSpec();
+        containerSpec.withMounts(mounts);
+    }
+
     @Override
     public void doFirstTimePostStartUpConfiguration() {
         JsonNode storageConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);

From bca9b23eb981024ca02f58bd57a0952ea9f59769 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 3 Mar 2026 17:24:47 +0000
Subject: [PATCH 05/13] qhouyee/83/backup-service: create kopia repository only
 if it doesnt exist by attempting to connect to the repo

---
 .../stack/services/KopiaService.java          | 35 ++++++++++++++++---
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index 62266ff9..6382f788 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -1,5 +1,6 @@
 package com.cmclinnovations.stack.services;
 
+import java.io.ByteArrayOutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -16,11 +17,15 @@
 import com.github.dockerjava.api.model.MountType;
 
 public class KopiaService extends ContainerService {
+    private final JsonNode storageConfig;
+
     public static final String TYPE = "kopia";
 
     private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
     private static final String VOLUME_DIR = "/data/";
     private static final String STORAGE_KEY = "storage";
+    private static final String CREATE_REPO_ACTION = "create";
+    private static final String CONNECT_REPO_ACTION = "connect";
 
     private static final String PASSWORD_SECRETS_FLAG = "--password=$(cat /run/secrets/kopia_password)";
 
@@ -28,6 +33,7 @@ public class KopiaService extends ContainerService {
 
     public KopiaService(String stackName, ServiceConfig config) {
         super(stackName, config);
+        this.storageConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
     }
 
     @Override
@@ -55,10 +61,27 @@ public void doPreStartUpConfiguration() {
 
     @Override
     public void doFirstTimePostStartUpConfiguration() {
-        JsonNode storageConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
-        String storageType = storageConfig.get("type").asText();
-        JsonNode storageConfigOptions = storageConfig.get("config");
-        List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", "create", storageType));
+        if (!connectRepository()) {
+            LOGGER.info("No existing Kopia repository found, creating new repository.");
+            createRepository();
+        }
+    }
+
+    private void createRepository() {
+        ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
+        createOrConnectRepository(CREATE_REPO_ACTION, errorStream);
+    }
+
+    private boolean connectRepository() {
+        ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
+        createOrConnectRepository(CONNECT_REPO_ACTION, errorStream);
+        return errorStream.toString().startsWith("Connected to repository");
+    }
+
+    private void createOrConnectRepository(String action, ByteArrayOutputStream errorStream) {
+        String storageType = this.storageConfig.get("type").asText();
+        JsonNode storageConfigOptions = this.storageConfig.get("config");
+        List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", action, storageType));
         switch (storageType) {
             case "filesystem":
                 String storagePath = storageConfigOptions.get("path").asText();
@@ -72,6 +95,8 @@ public void doFirstTimePostStartUpConfiguration() {
         }
 
         createRepoCommandArgs.add(PASSWORD_SECRETS_FLAG);
-        super.executeCommand(createRepoCommandArgs.toArray(new String[0]));
+        super.createComplexCommand(createRepoCommandArgs.toArray(new String[0]))
+                .withErrorStream(errorStream)
+                .exec();
     }
 }

From 7b162e137ec7aebb9e356a7b048b3a705a872180 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Thu, 5 Mar 2026 16:21:07 +0000
Subject: [PATCH 06/13] qhouyee/83/backup-service: read kopia password from
 secrets; gen command separately in a reusable method

---
 .../stack/services/KopiaService.java          | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index 6382f788..a45bc82e 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -16,24 +16,29 @@
 import com.github.dockerjava.api.model.Mount;
 import com.github.dockerjava.api.model.MountType;
 
+import uk.ac.cam.cares.jps.base.util.FileUtil;
+
 public class KopiaService extends ContainerService {
     private final JsonNode storageConfig;
 
     public static final String TYPE = "kopia";
 
-    private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
     private static final String VOLUME_DIR = "/data/";
+    private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
+    private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+
     private static final String STORAGE_KEY = "storage";
     private static final String CREATE_REPO_ACTION = "create";
     private static final String CONNECT_REPO_ACTION = "connect";
 
-    private static final String PASSWORD_SECRETS_FLAG = "--password=$(cat /run/secrets/kopia_password)";
+    private String passwordFlag;
 
     private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class);
 
     public KopiaService(String stackName, ServiceConfig config) {
         super(stackName, config);
         this.storageConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+        this.passwordFlag = "--password=" + FileUtil.readFileLocally(KOPIA_PASSWORD_PATH);
     }
 
     @Override
@@ -79,6 +84,13 @@ private boolean connectRepository() {
     }
 
     private void createOrConnectRepository(String action, ByteArrayOutputStream errorStream) {
+        List<String> createRepoCommandArgs = this.genCreateOrConnectRepositoryCommand(action);
+        super.createComplexCommand(createRepoCommandArgs.toArray(new String[0]))
+                .withErrorStream(errorStream)
+                .exec();
+    }
+
+    private List<String> genCreateOrConnectRepositoryCommand(String action) {
         String storageType = this.storageConfig.get("type").asText();
         JsonNode storageConfigOptions = this.storageConfig.get("config");
         List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", action, storageType));
@@ -94,9 +106,7 @@ private void createOrConnectRepository(String action, ByteArrayOutputStream erro
                 break;
         }
 
-        createRepoCommandArgs.add(PASSWORD_SECRETS_FLAG);
-        super.createComplexCommand(createRepoCommandArgs.toArray(new String[0]))
-                .withErrorStream(errorStream)
-                .exec();
+        createRepoCommandArgs.add(this.passwordFlag);
+        return createRepoCommandArgs;
     }
 }

From 2ac8e0e93242c97154187943e52eab875a0604e3 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Fri, 6 Mar 2026 12:01:23 +0000
Subject: [PATCH 07/13] qhouyee/83/backup-service: use a custom kopia image
 with crone installed; added scheduled backup script generation

---
 .../stack/services/KopiaService.java          | 25 +++++++++++++++++++
 .../stack/services/built-ins/kopia.json       |  7 +-----
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index a45bc82e..a0eff07c 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -1,6 +1,7 @@
 package com.cmclinnovations.stack.services;
 
 import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -26,6 +27,7 @@ public class KopiaService extends ContainerService {
     private static final String VOLUME_DIR = "/data/";
     private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
     private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+    private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh";
 
     private static final String STORAGE_KEY = "storage";
     private static final String CREATE_REPO_ACTION = "create";
@@ -70,6 +72,8 @@ public void doFirstTimePostStartUpConfiguration() {
             LOGGER.info("No existing Kopia repository found, creating new repository.");
             createRepository();
         }
+
+        this.genScheduledBackups();
     }
 
     private void createRepository() {
@@ -109,4 +113,25 @@ private List<String> genCreateOrConnectRepositoryCommand(String action) {
         createRepoCommandArgs.add(this.passwordFlag);
         return createRepoCommandArgs;
     }
+
+    private void genScheduledBackups() {
+        LOGGER.info("Creating initial snapshot for the repository...");
+        super.createComplexCommand(TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR, this.passwordFlag)
+                .exec();
+
+        LOGGER.info("Generating scheduled backup script...");
+        StringBuilder scriptContents = new StringBuilder();
+        scriptContents.append("#!/bin/bash\n\n")
+                // Add a warning that this file is auto-generated
+                .append("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n")
+                .append(
+                        String.join(" ", this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION)))
+                .append("\n")
+                .append("kopia snapshot create --all ").append(this.passwordFlag);
+        super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8));
+        // Requires executable permission for root user for the crone job
+        super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH)
+                .withUser("root")
+                .exec();
+    }
 }
diff --git a/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
index 3c8e9484..03766a86 100644
--- a/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
+++ b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
@@ -4,12 +4,7 @@
     "Name": "kopia",
     "TaskTemplate": {
       "ContainerSpec": {
-        "Image": "kopia/kopia:latest",
-        "Command": [
-          "tail",
-          "-f",
-          "/dev/null"
-        ],
+        "Image": "ghcr.io/theworldavatar/kopia:0.1.0-backup-service-SNAPSHOT",
         "Secrets": [
           {
             "SecretName": "kopia_password"

From 5de4475017af6bd161e416f30358565ac46bd47d Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Mon, 9 Mar 2026 16:13:29 +0000
Subject: [PATCH 08/13] qhouyee/83/backup-service: extended for sftp connection

---
 .../stack/services/KopiaService.java          | 26 ++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index a0eff07c..c3c571e4 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -1,10 +1,14 @@
 package com.cmclinnovations.stack.services;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.exception.UncheckedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,6 +31,7 @@ public class KopiaService extends ContainerService {
     private static final String VOLUME_DIR = "/data/";
     private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
     private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+    private static final String SFTP_SSH_KEY_PATH = "/tmp/ssh_key";
     private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh";
 
     private static final String STORAGE_KEY = "storage";
@@ -99,8 +104,27 @@ private List<String> genCreateOrConnectRepositoryCommand(String action) {
         JsonNode storageConfigOptions = this.storageConfig.get("config");
         List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", action, storageType));
         switch (storageType) {
-            case "filesystem":
+            case "sftp":
                 String storagePath = storageConfigOptions.get("path").asText();
+                String host = storageConfigOptions.get("host").asText();
+                String user = storageConfigOptions.get("username").asText();
+                if (action.equals(CREATE_REPO_ACTION)) {
+                    super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~/.ssh/known_hosts");
+                }
+                createRepoCommandArgs.add("--path=" + storagePath);
+                createRepoCommandArgs.add("--host=" + host);
+                createRepoCommandArgs.add("--username=" + user);
+                try {
+                    String keyFilePath = storageConfigOptions.get("keyfile").asText();
+                    super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath)));
+                    createRepoCommandArgs.add("--keyfile=" + SFTP_SSH_KEY_PATH);
+                } catch (IOException e) {
+                    throw new UncheckedException(e);
+                }
+                createRepoCommandArgs.add("--known-hosts=/root/.ssh/known_hosts");
+                break;
+            case "filesystem":
+                storagePath = storageConfigOptions.get("path").asText();
                 createRepoCommandArgs.add("--path=" + storagePath);
                 break;
             default:

From bc5c04c9f8bc4c1dbf273202cf36c0859f6f90b4 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Mon, 9 Mar 2026 17:19:04 +0000
Subject: [PATCH 09/13] qhouyee/83/backup-service: execute all commands as part
 of a script to create/connect repo and handle snapshots

---
 .../stack/services/KopiaService.java          | 62 +++++++------------
 1 file changed, 21 insertions(+), 41 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index c3c571e4..efcaaa30 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -1,6 +1,5 @@
 package com.cmclinnovations.stack.services;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -73,29 +72,31 @@ public void doPreStartUpConfiguration() {
 
     @Override
     public void doFirstTimePostStartUpConfiguration() {
-        if (!connectRepository()) {
-            LOGGER.info("No existing Kopia repository found, creating new repository.");
-            createRepository();
-        }
-
         this.genScheduledBackups();
     }
 
-    private void createRepository() {
-        ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
-        createOrConnectRepository(CREATE_REPO_ACTION, errorStream);
-    }
-
-    private boolean connectRepository() {
-        ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
-        createOrConnectRepository(CONNECT_REPO_ACTION, errorStream);
-        return errorStream.toString().startsWith("Connected to repository");
-    }
+    private void genScheduledBackups() {
+        LOGGER.info("Generating scheduled backup script...");
+        String connectRepoCommand = String.join(" ", this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION));
+        String initSnapshotCommand = String.join(" ", TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR,
+                this.passwordFlag);
 
-    private void createOrConnectRepository(String action, ByteArrayOutputStream errorStream) {
-        List<String> createRepoCommandArgs = this.genCreateOrConnectRepositoryCommand(action);
-        super.createComplexCommand(createRepoCommandArgs.toArray(new String[0]))
-                .withErrorStream(errorStream)
+        StringBuilder scriptContents = new StringBuilder();
+        scriptContents.append("#!/bin/bash\n\n")
+                // Add a warning that this file is auto-generated
+                .append("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n")
+                .append("if ! ").append(connectRepoCommand)
+                .append("; then\n\t")
+                .append(
+                        String.join(" ", this.genCreateOrConnectRepositoryCommand(CREATE_REPO_ACTION)))
+                .append("\n\t").append(connectRepoCommand)
+                .append("\n\t").append(initSnapshotCommand).append("\n")
+                .append("else\n\tkopia snapshot create --all ").append(this.passwordFlag)
+                .append("\nfi");
+        super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8));
+        // Requires executable permission for root user for the crone job
+        super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH)
+                .withUser("root")
                 .exec();
     }
 
@@ -137,25 +138,4 @@ private List<String> genCreateOrConnectRepositoryCommand(String action) {
         createRepoCommandArgs.add(this.passwordFlag);
         return createRepoCommandArgs;
     }
-
-    private void genScheduledBackups() {
-        LOGGER.info("Creating initial snapshot for the repository...");
-        super.createComplexCommand(TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR, this.passwordFlag)
-                .exec();
-
-        LOGGER.info("Generating scheduled backup script...");
-        StringBuilder scriptContents = new StringBuilder();
-        scriptContents.append("#!/bin/bash\n\n")
-                // Add a warning that this file is auto-generated
-                .append("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n")
-                .append(
-                        String.join(" ", this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION)))
-                .append("\n")
-                .append("kopia snapshot create --all ").append(this.passwordFlag);
-        super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8));
-        // Requires executable permission for root user for the crone job
-        super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH)
-                .withUser("root")
-                .exec();
-    }
 }

From a89ef7fb84179cbb6e2f045488316363ae9e7ded Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Mon, 9 Mar 2026 18:06:59 +0000
Subject: [PATCH 10/13] qhouyee/83/backup-service: add documentation for backup

---
 stack-manager/README.md | 60 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/stack-manager/README.md b/stack-manager/README.md
index 4e63b3a6..d375dfc3 100644
--- a/stack-manager/README.md
+++ b/stack-manager/README.md
@@ -414,6 +414,66 @@ graph TB
 
 ```
 
+## Volume backup
+
+The stack includes an automated backup solution using [Kopia](https://kopia.io/). It is configured to perform daily snapshots of all volumes attached to the stack to a target destination.
+
+> [!IMPORTANT]  
+> Kopia can only access and back up volumes within the same stack
+
+### Prerequisites
+
+1) `stack.json`: Include the `kopia` service
+2) `kopia_password`: A [Docker secret](#secrets) containing the master password for the Kopia repository
+3) `./inputs/data/kopia/repository.config`: Contains the connection details for the backup destination (known as a Kopia repository); Note that this directory need not be mounted to the kopia service
+
+### Repository configuration
+
+Kopia requires the creation of a [repository](https://kopia.io/docs/repositories/) that specifies the destination of backups. Note that this automated implementation only supports a filesystem or sftp repository connections using the custom JSON-based `repository.config`. The skeleton template is as follows:
+
+```json
+{
+  "storage": {
+    "type": "sftp OR filesystem",
+    "config": {
+      ... // config options
+    }
+  }
+}
+```
+
+#### 1. Filesystem
+
+```json
+{
+  "storage": {
+    "type": "filesystem",
+    "config": {
+        "path": "/backup", // local or network mounted path to storage location
+    }
+  }
+}
+```
+
+#### 2. SFTP
+
+For the `SFTP` option, users must include a `ssh_key` file in the `./inputs/data/kopia/` directory containing the `SSH` key to access the repository.
+
+```json
+{
+  "storage": {
+    "type": "sftp",
+    "config": {
+        "path": "/backup", // path to destination
+        "host": "XXX.XXX.X.XX", // domain or ip address
+        "port": 22,
+        "username": "user",
+        "keyfile": "/inputs/data/kopia/ssh_key", // do NOT change
+    }
+  }
+}
+```
+
 ## Example - including a visualisation
 
 This example explains how to spin up a TWA-VF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated and secrets `mapbox_username`, and `mapbox_api_key` to be created.

From 6ea74ea582c8ef0893019298f08c452df7095e37 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 10 Mar 2026 10:29:36 +0000
Subject: [PATCH 11/13] qhouyee/83/backup-service: create known hosts file only
 if it does not exists

---
 .../com/cmclinnovations/stack/services/KopiaService.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index efcaaa30..73526895 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -31,6 +31,8 @@ public class KopiaService extends ContainerService {
     private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
     private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
     private static final String SFTP_SSH_KEY_PATH = "/tmp/ssh_key";
+    private static final String KNOWN_HOSTS_PATH = "/.ssh/known_hosts";
+    private static final String ROOT_KNOWN_HOSTS_PATH = "/root" + KNOWN_HOSTS_PATH;
     private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh";
 
     private static final String STORAGE_KEY = "storage";
@@ -109,8 +111,8 @@ private List<String> genCreateOrConnectRepositoryCommand(String action) {
                 String storagePath = storageConfigOptions.get("path").asText();
                 String host = storageConfigOptions.get("host").asText();
                 String user = storageConfigOptions.get("username").asText();
-                if (action.equals(CREATE_REPO_ACTION)) {
-                    super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~/.ssh/known_hosts");
+                if (action.equals(CREATE_REPO_ACTION) && !super.fileExists(ROOT_KNOWN_HOSTS_PATH)) {
+                    super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~" + KNOWN_HOSTS_PATH);
                 }
                 createRepoCommandArgs.add("--path=" + storagePath);
                 createRepoCommandArgs.add("--host=" + host);

From 59cd41327a6e54606a37767455a4ed4fc08bc97f Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 10 Mar 2026 11:22:26 +0000
Subject: [PATCH 12/13] qhouyee/83/backup-service: clean up; copy ssh key once
 into container

---
 .../stack/services/KopiaService.java          | 70 ++++++++++++-------
 1 file changed, 46 insertions(+), 24 deletions(-)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index 73526895..aa26e6c0 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -23,7 +23,6 @@
 import uk.ac.cam.cares.jps.base.util.FileUtil;
 
 public class KopiaService extends ContainerService {
-    private final JsonNode storageConfig;
 
     public static final String TYPE = "kopia";
 
@@ -39,13 +38,17 @@ public class KopiaService extends ContainerService {
     private static final String CREATE_REPO_ACTION = "create";
     private static final String CONNECT_REPO_ACTION = "connect";
 
-    private String passwordFlag;
+    private final String storageType;
+    private final String passwordFlag;
+    private final JsonNode storageConfig;
 
     private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class);
 
     public KopiaService(String stackName, ServiceConfig config) {
         super(stackName, config);
-        this.storageConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+        JsonNode repoConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+        this.storageConfig = repoConfig.get("config");
+        this.storageType = repoConfig.get("type").asText();
         this.passwordFlag = "--password=" + FileUtil.readFileLocally(KOPIA_PASSWORD_PATH);
     }
 
@@ -83,11 +86,10 @@ private void genScheduledBackups() {
         String initSnapshotCommand = String.join(" ", TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR,
                 this.passwordFlag);
 
-        StringBuilder scriptContents = new StringBuilder();
-        scriptContents.append("#!/bin/bash\n\n")
-                // Add a warning that this file is auto-generated
-                .append("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n")
-                .append("if ! ").append(connectRepoCommand)
+        // Add a warning that this file is auto-generated
+        StringBuilder scriptContents = new StringBuilder(
+                "#!/bin/bash\n# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n");
+        scriptContents.append("if ! ").append(connectRepoCommand)
                 .append("; then\n\t")
                 .append(
                         String.join(" ", this.genCreateOrConnectRepositoryCommand(CREATE_REPO_ACTION)))
@@ -103,41 +105,61 @@ private void genScheduledBackups() {
     }
 
     private List<String> genCreateOrConnectRepositoryCommand(String action) {
-        String storageType = this.storageConfig.get("type").asText();
-        JsonNode storageConfigOptions = this.storageConfig.get("config");
-        List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", action, storageType));
-        switch (storageType) {
+        List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", action, this.storageType));
+        switch (this.storageType) {
             case "sftp":
-                String storagePath = storageConfigOptions.get("path").asText();
-                String host = storageConfigOptions.get("host").asText();
-                String user = storageConfigOptions.get("username").asText();
+                this.genSSHKeyFile(this.storageConfig.get("keyfile").asText());
+
+                String storagePath = this.storageConfig.get("path").asText();
+                String host = this.storageConfig.get("host").asText();
+                String user = this.storageConfig.get("username").asText();
                 if (action.equals(CREATE_REPO_ACTION) && !super.fileExists(ROOT_KNOWN_HOSTS_PATH)) {
                     super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~" + KNOWN_HOSTS_PATH);
                 }
                 createRepoCommandArgs.add("--path=" + storagePath);
                 createRepoCommandArgs.add("--host=" + host);
                 createRepoCommandArgs.add("--username=" + user);
-                try {
-                    String keyFilePath = storageConfigOptions.get("keyfile").asText();
-                    super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath)));
-                    createRepoCommandArgs.add("--keyfile=" + SFTP_SSH_KEY_PATH);
-                } catch (IOException e) {
-                    throw new UncheckedException(e);
+                createRepoCommandArgs.add("--keyfile=" + SFTP_SSH_KEY_PATH);
+                createRepoCommandArgs.add("--known-hosts=" + ROOT_KNOWN_HOSTS_PATH);
+                if (this.requiresExternalSSH()) {
+                    createRepoCommandArgs.add("--external");
                 }
-                createRepoCommandArgs.add("--known-hosts=/root/.ssh/known_hosts");
                 break;
             case "filesystem":
-                storagePath = storageConfigOptions.get("path").asText();
+                storagePath = this.storageConfig.get("path").asText();
                 createRepoCommandArgs.add("--path=" + storagePath);
                 break;
             default:
                 LOGGER.warn(
                         "Unsupported storage type '{}' for automatic stack setup. Please manually configure the storage",
-                        storageType);
+                        this.storageType);
                 break;
         }
 
         createRepoCommandArgs.add(this.passwordFlag);
         return createRepoCommandArgs;
     }
+
+    /**
+     * Checks if the repository requires external SSH key access due to passphrase
+     * protection.
+     */
+    private boolean requiresExternalSSH() {
+        return this.storageConfig.has("externalSSH") && this.storageConfig.get("externalSSH").asBoolean();
+    }
+
+    /**
+     * Generates an SSH key file in the Kopia container if it doesn't already exist.
+     * 
+     * @param keyFilePath Path to the SSH key file
+     */
+    private void genSSHKeyFile(String keyFilePath) {
+        if (!super.fileExists(keyFilePath)) {
+            try {
+                super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath)));
+            } catch (IOException e) {
+                throw new UncheckedException(e);
+            }
+        }
+    }
 }

From 723325abdb6a0058b5d19927ef356f385332ce99 Mon Sep 17 00:00:00 2001
From: qhouyee <quekhouyee@gmail.com>
Date: Tue, 10 Mar 2026 11:23:17 +0000
Subject: [PATCH 13/13] qhouyee/83/backup-service: handle encrypted ssh key
 access with passphrase

---
 .../java/com/cmclinnovations/stack/services/KopiaService.java | 4 ++++
 stack-manager/README.md                                       | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
index aa26e6c0..8380694c 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -89,6 +89,10 @@ private void genScheduledBackups() {
         // Add a warning that this file is auto-generated
         StringBuilder scriptContents = new StringBuilder(
                 "#!/bin/bash\n# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n");
+        if (this.storageType.equals("sftp") && this.requiresExternalSSH()) {
+            scriptContents.append("eval `keychain --eval --agents ssh ")
+                    .append(SFTP_SSH_KEY_PATH).append("`\n\n");
+        }
         scriptContents.append("if ! ").append(connectRepoCommand)
                 .append("; then\n\t")
                 .append(
diff --git a/stack-manager/README.md b/stack-manager/README.md
index d375dfc3..99891928 100644
--- a/stack-manager/README.md
+++ b/stack-manager/README.md
@@ -469,11 +469,14 @@ For the `SFTP` option, users must include a `ssh_key` file in the `./inputs/data
         "port": 22,
         "username": "user",
         "keyfile": "/inputs/data/kopia/ssh_key", // do NOT change
+        "externalSSH": false // optional and will default to false
     }
   }
 }
 ```
 
+If the `SSH` key is encrypted by a passphrase, please set the `externalSSH` flag to true and manually add the private passphrase in a terminal via the `keychain /tmp/ssh_key` command.
+
 ## Example - including a visualisation
 
 This example explains how to spin up a TWA-VF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated and secrets `mapbox_username`, and `mapbox_api_key` to be created.