Qhouyee/83/backup service

stack-clients/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-client:
-    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.57.0
+    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.58.0-backup-service-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-clients/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-clients</artifactId>
-    <version>1.57.0</version>
+    <version>1.58.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Clients</name>
     <url>https://theworldavatar.io</url>

stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java

@@ -15,12 +15,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
-import java.util.NoSuchElementException;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -40,9 +40,11 @@
 import com.github.dockerjava.api.command.InspectContainerCmd;
 import com.github.dockerjava.api.command.InspectExecCmd;
 import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
 import com.github.dockerjava.api.command.ListConfigsCmd;
 import com.github.dockerjava.api.command.ListContainersCmd;
 import com.github.dockerjava.api.command.ListSecretsCmd;
+import com.github.dockerjava.api.command.ListVolumesCmd;
 import com.github.dockerjava.api.command.RemoveConfigCmd;
 import com.github.dockerjava.api.command.RemoveSecretCmd;
 import com.github.dockerjava.api.model.Config;
@@ -553,7 +555,7 @@ public boolean isContainerUp(String containerName) {
 
     public String getContainerId(String containerName) {
         return getContainer(containerName).map(Container::getId)
-                .orElseThrow(() -> new NoSuchElementException("Cannot get container "+containerName+"."));
+                .orElseThrow(() -> new NoSuchElementException("Cannot get container " + containerName + "."));
     }
 
     private Map<String, List<String>> convertToConfigFilterMap(String configName, Map<String, String> labelMap) {
@@ -593,6 +595,17 @@ public Optional<Config> getConfig(List<Config> configs, String configName) {
 
     }
 
+    public List<InspectVolumeResponse> getVolumes() {
+        try (ListVolumesCmd listVolumesCmd = internalClient.listVolumesCmd()) {
+            return listVolumesCmd
+                    .exec()
+                    .getVolumes()
+                    .stream()
+                    .filter(v -> v.getName() != null && v.getName().startsWith(StackClient.getStackName()))
+                    .collect(Collectors.toList());
+        }
+    }
+
     public List<Config> getConfigs() {
         try (ListConfigsCmd listConfigsCmd = internalClient.listConfigsCmd()) {
             return listConfigsCmd

stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java

@@ -1,11 +1,15 @@
 package com.cmclinnovations.stack.clients.utils;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.text.MessageFormat;
 
 import javax.annotation.Nonnull;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.json.JsonMapper;
 
@@ -31,4 +35,21 @@ public static final String handleFileValues(String value) {
         }
         return value;
     }
+
+    /**
+     * Parses a JSON file into a JSON object.
+     * 
+     * @param file Path to the file.
+     */
+    public static final JsonNode readFile(String file) {
+        try {
+            String fileContents = Files.readString(Path.of(file));
+            return getMapper().readTree(fileContents);
+        } catch (MalformedURLException ex) {
+            throw new IllegalArgumentException(MessageFormat.format("Invalid file path: {0}", file));
+        } catch (IOException ex) {
+            throw new UncheckedIOException(
+                    MessageFormat.format("Failed to read file: {0}", ex.getMessage()), ex);
+        }
+    }
 }

stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java

@@ -25,6 +25,7 @@
 import com.cmclinnovations.stack.services.config.ServiceConfig;
 import com.github.dockerjava.api.command.InspectImageCmd;
 import com.github.dockerjava.api.command.InspectImageResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
 import com.github.dockerjava.api.model.ContainerSpec;
 import com.github.dockerjava.api.model.ServiceSpec;
 import com.github.dockerjava.api.model.TaskSpec;
@@ -240,4 +241,7 @@ public String getDNSIPAddress() {
         return dockerClient.getDNSIPAddress();
     }
 
+    protected List<InspectVolumeResponse> getVolumes() {
+        return dockerClient.getVolumes();
+    }
 }

stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java

@@ -0,0 +1,169 @@
+package com.cmclinnovations.stack.services;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.exception.UncheckedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cmclinnovations.stack.clients.core.StackClient;
+import com.cmclinnovations.stack.clients.utils.JsonHelper;
+import com.cmclinnovations.stack.services.config.ServiceConfig;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.model.ContainerSpec;
+import com.github.dockerjava.api.model.Mount;
+import com.github.dockerjava.api.model.MountType;
+
+import uk.ac.cam.cares.jps.base.util.FileUtil;
+
+public class KopiaService extends ContainerService {
+
+    public static final String TYPE = "kopia";
+
+    private static final String VOLUME_DIR = "/data/";
+    private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
+    private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+    private static final String SFTP_SSH_KEY_PATH = "/tmp/ssh_key";
+    private static final String KNOWN_HOSTS_PATH = "/.ssh/known_hosts";
+    private static final String ROOT_KNOWN_HOSTS_PATH = "/root" + KNOWN_HOSTS_PATH;
+    private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh";
+
+    private static final String STORAGE_KEY = "storage";
+    private static final String CREATE_REPO_ACTION = "create";
+    private static final String CONNECT_REPO_ACTION = "connect";
+
+    private final String storageType;
+    private final String passwordFlag;
+    private final JsonNode storageConfig;
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class);
+
+    public KopiaService(String stackName, ServiceConfig config) {
+        super(stackName, config);
+        JsonNode repoConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+        this.storageConfig = repoConfig.get("config");
+        this.storageType = repoConfig.get("type").asText();
+        this.passwordFlag = "--password=" + FileUtil.readFileLocally(KOPIA_PASSWORD_PATH);
+    }
+
+    @Override
+    public void doPreStartUpConfiguration() {
+        // Mount all volumes on the stack (except kopia) for backups
+        List<InspectVolumeResponse> volumes = super.getVolumes();
+        List<Mount> mounts = new ArrayList<>();
+
+        for (InspectVolumeResponse vol : volumes) {
+            if (!vol.getName().endsWith(TYPE)) {
+                // Remove stack name prefix (STACK_) from volume name
+                String volName = vol.getName().substring(StackClient.getStackName().length() + 1);
+                String destinationPath = VOLUME_DIR + volName;
+                mounts.add(new Mount()
+                        .withType(MountType.VOLUME) // Use Docker volume for named volumes
+                        .withSource(volName)
+                        .withTarget(destinationPath)
+                        .withReadOnly(false));
+            }
+        }
+
+        ContainerSpec containerSpec = super.getContainerSpec();
+        containerSpec.withMounts(mounts);
+    }
+
+    @Override
+    public void doFirstTimePostStartUpConfiguration() {
+        this.genScheduledBackups();
+    }
+
+    private void genScheduledBackups() {
+        LOGGER.info("Generating scheduled backup script...");
+        String connectRepoCommand = String.join(" ", this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION));
+        String initSnapshotCommand = String.join(" ", TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR,
+                this.passwordFlag);
+
+        // Add a warning that this file is auto-generated
+        StringBuilder scriptContents = new StringBuilder(
+                "#!/bin/bash\n# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY\n\n");
+        if (this.storageType.equals("sftp") && this.requiresExternalSSH()) {
+            scriptContents.append("eval `keychain --eval --agents ssh ")
+                    .append(SFTP_SSH_KEY_PATH).append("`\n\n");
+        }
+        scriptContents.append("if ! ").append(connectRepoCommand)
+                .append("; then\n\t")
+                .append(
+                        String.join(" ", this.genCreateOrConnectRepositoryCommand(CREATE_REPO_ACTION)))
+                .append("\n\t").append(connectRepoCommand)
+                .append("\n\t").append(initSnapshotCommand).append("\n")
+                .append("else\n\tkopia snapshot create --all ").append(this.passwordFlag)
+                .append("\nfi");
+        super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8));
+        // Requires executable permission for root user for the crone job
+        super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH)
+                .withUser("root")
+                .exec();
+    }
+
+    private List<String> genCreateOrConnectRepositoryCommand(String action) {
+        List<String> createRepoCommandArgs = new ArrayList<>(List.of(TYPE, "repository", action, this.storageType));
+        switch (this.storageType) {
+            case "sftp":
+                this.genSSHKeyFile(this.storageConfig.get("keyfile").asText());
+
+                String storagePath = this.storageConfig.get("path").asText();
+                String host = this.storageConfig.get("host").asText();
+                String user = this.storageConfig.get("username").asText();
+                if (action.equals(CREATE_REPO_ACTION) && !super.fileExists(ROOT_KNOWN_HOSTS_PATH)) {
+                    super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~" + KNOWN_HOSTS_PATH);
+                }
+                createRepoCommandArgs.add("--path=" + storagePath);
+                createRepoCommandArgs.add("--host=" + host);
+                createRepoCommandArgs.add("--username=" + user);
+                createRepoCommandArgs.add("--keyfile=" + SFTP_SSH_KEY_PATH);
+                createRepoCommandArgs.add("--known-hosts=" + ROOT_KNOWN_HOSTS_PATH);
+                if (this.requiresExternalSSH()) {
+                    createRepoCommandArgs.add("--external");
+                }
+                break;
+            case "filesystem":
+                storagePath = this.storageConfig.get("path").asText();
+                createRepoCommandArgs.add("--path=" + storagePath);
+                break;
+            default:
+                LOGGER.warn(
+                        "Unsupported storage type '{}' for automatic stack setup. Please manually configure the storage",
+                        this.storageType);
+                break;
+        }
+
+        createRepoCommandArgs.add(this.passwordFlag);
+        return createRepoCommandArgs;
+    }
+
+    /**
+     * Checks if the repository requires external SSH key access due to passphrase
+     * protection.
+     */
+    private boolean requiresExternalSSH() {
+        return this.storageConfig.has("externalSSH") && this.storageConfig.get("externalSSH").asBoolean();
+    }
+
+    /**
+     * Generates an SSH key file in the Kopia container if it doesn't already exist.
+     * 
+     * @param keyFilePath Path to the SSH key file
+     */
+    private void genSSHKeyFile(String keyFilePath) {
+        if (!super.fileExists(keyFilePath)) {
+            try {
+                super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath)));
+            } catch (IOException e) {
+                throw new UncheckedException(e);
+            }
+        }
+    }
+}

stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json

@@ -0,0 +1,16 @@
+{
+  "type": "kopia",
+  "ServiceSpec": {
+    "Name": "kopia",
+    "TaskTemplate": {
+      "ContainerSpec": {
+        "Image": "ghcr.io/theworldavatar/kopia:0.1.0-backup-service-SNAPSHOT",
+        "Secrets": [
+          {
+            "SecretName": "kopia_password"
+          }
+        ]
+      }
+    }
+  }
+}

stack-data-uploader/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-data-uploader:
-    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.57.0
+    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.58.0-backup-service-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-data-uploader/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-data-uploader</artifactId>
-    <version>1.57.0</version>
+    <version>1.58.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Data Uploader</name>
     <url>https://theworldavatar.io</url>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.cmclinnovations</groupId>
             <artifactId>stack-clients</artifactId>
-            <version>1.57.0</version>
+            <version>1.58.0-backup-service-SNAPSHOT</version>
         </dependency>
 
         <dependency>