From dc7beab1aef2887425e97a192f5d293f5e0b6170 Mon Sep 17 00:00:00 2001 From: Alexander Laye Date: Mon, 16 Mar 2026 14:04:29 -0400 Subject: [PATCH 1/3] add failover docs Signed-off-by: Alexander Laye --- .github/agents/documentation-agent.md | 2 +- .../failover-procedures.md | 342 ++++++++++ .../multi-region-deployment/overview.md | 181 ++++++ .../preview/multi-region-deployment/setup.md | 605 ++++++++++++++++++ .../aks-fleet-deployment/README.md | 2 +- mkdocs.yml | 4 + 6 files changed, 1134 insertions(+), 2 deletions(-) create mode 100644 docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md create mode 100644 docs/operator-public-documentation/preview/multi-region-deployment/overview.md create mode 100644 docs/operator-public-documentation/preview/multi-region-deployment/setup.md diff --git a/.github/agents/documentation-agent.md b/.github/agents/documentation-agent.md index 8f0a1d18..8915af38 100644 --- a/.github/agents/documentation-agent.md +++ b/.github/agents/documentation-agent.md @@ -1,6 +1,6 @@ --- description: 'Agent for documentation tasks in the DocumentDB Kubernetes Operator project.' -tools: [execute, read, terminal] +tools: [execute, read, edit] --- # Documentation Agent Instructions diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md b/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md new file mode 100644 index 00000000..e1139348 --- /dev/null +++ b/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md @@ -0,0 +1,342 @@ +--- +title: Multi-Region Failover Procedures +description: Step-by-step runbooks for planned and unplanned DocumentDB failovers across regions, including verification and rollback procedures. +tags: + - multi-region + - failover + - disaster-recovery + - operations +--- + +# Multi-Region Failover Procedures + +This guide provides runbooks for switching the primary DocumentDB cluster between regions during planned maintenance or unplanned outages. + +## Overview + +A **failover** promotes a replica cluster to become the new primary, making it accept write operations. The previous primary (if still available) becomes a replica replicating from the new primary. + +**When to perform failover:** + +- **Planned maintenance:** Region maintenance, infrastructure upgrades, cost optimization +- **Disaster recovery:** Primary region outage, network partition, catastrophic failure +- **Performance optimization:** Move primary closer to write-heavy workload +- **Testing:** Validate disaster recovery procedures + +## Failover Types + +### Planned Failover + +This is a failover where the primary is safely demoted, writes are all flushed, and then the new primary is +promoted. This kind of failover has no data loss, a set window where writes aren't accepted, and the same number +of replicas before and after. + +### Unplanned Failover (Disaster Recovery) + +This is a failover where the primary becomes unavailable and has to be forced out of the DocumentDB cluster +entirely. Downtime is relative to how long it takes to detect the degradation of the primary, as well as the +time it takes to spin up HA on the new primary (if HA is enabled). There is a potential for lost writes to +the primary, but with High-Availability the client should know when writes weren't committed to remotes. +By the end of this type of failover, there will be one fewer region in the DocumentDB cluster and likely steps +will be needed to add the new region. See the [add region playground guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/fleet-add-region/README.md) +for an example. + +## Prerequisites + +Before performing any failover: + +- [ ] **Replica health:** Target replica cluster is running and replication is current +- [ ] **Network access:** You have kubectl access to all clusters involved +- [ ] **Backup available:** Recent backup exists for rollback if needed +- [ ] **Monitoring:** Metrics and logs are accessible for verification +- [ ] **Communication:** Stakeholders are notified (for planned failover) +- [ ] **Application readiness:** Applications can handle brief connection interruption +- [ ] **kubectl-documentdb plugin:** Install the plugin for streamlined failover operations [kubectl-plugin.md](../kubectl-plugin.md) + +### Check Current Replication Status + +Identify the current primary and verify replica health: + +```bash +# View current primary setting +kubectl --context hub get documentdb documentdb-preview \ + -n documentdb-preview-ns -o jsonpath='{.spec.clusterReplication.primary}' + +# Check replication status on primary +kubectl --context primary exec -it -n documentdb-preview-ns \ + documentdb-preview-1 -- psql -U postgres -c "SELECT * FROM pg_stat_replication;" +``` + +Expected output shows active replication to all replicas: + +``` + pid | usename | application_name | client_addr | state | sent_lsn | write_lsn | flush_lsn | replay_lsn | sync_state +-----+----------+------------------+-------------+-----------+------------+------------+------------+------------+------------ + 123 | postgres | replica1 | 10.2.1.5 | streaming | 0/30000A8 | 0/30000A8 | 0/30000A8 | 0/30000A8 | async + 124 | postgres | replica2 | 10.3.1.5 | streaming | 0/30000A8 | 0/30000A8 | 0/30000A8 | 0/30000A8 | async +``` + +**Key indicators:** + +- **state:** Should be `streaming` +- **LSN values:** `replay_lsn` should be close to `sent_lsn` (low replication lag) + +## Planned Failover Procedure + +Use this procedure when the primary cluster is healthy and you want to switch primary regions in a controlled manner. + +### Step 1: Pre-Failover Verification + +Verify system health before starting: + +```bash +# 1. Check all DocumentDB clusters are ready +kubectl --context hub get documentdb -A + +# 2. Verify replication lag is low (< 1 second) +kubectl --context current-primary exec -it -n documentdb-preview-ns \ + documentdb-preview-1 -- psql -U postgres -c \ + "SELECT client_addr, state, replay_lag FROM pg_stat_replication;" + +# 3. Check target replica is healthy +kubectl --context new-primary get pods -n documentdb-preview-ns +``` + +All checks should show healthy status before proceeding. + +### Step 2: Perform Failover + +Promote the replica to become the new primary cluster. + +=== "Using kubectl-documentdb Plugin (KubeFleet deployment required)" + + The plugin handles the CRD change and automatically waits for convergence: + + ```bash + kubectl documentdb promote \ + --documentdb documentdb-preview \ + --namespace documentdb-preview-ns \ + --target-cluster new-primary-cluster-name \ + --hub-context hub + ``` + +=== "Using kubectl patch with Fleet" + + Update the DocumentDB resource on the hub cluster: + + ```bash + kubectl --context hub patch documentdb documentdb-preview \ + -n documentdb-preview-ns \ + --type='merge' \ + -p '{"spec":{"clusterReplication":{"primary":"new-primary-cluster-name"}}}' + ``` + + The fleet controller propagates the change to all member clusters automatically. + +=== "Using kubectl patch without KubeFleet" + + Update the DocumentDB resource on **all** clusters: + + ```bash + # Update on all clusters (use loop or run individually) + for context in cluster1 cluster2 cluster3; do + kubectl --context $context patch documentdb documentdb-preview \ + -n documentdb-preview-ns \ + --type='merge' \ + -p '{"spec":{"clusterReplication":{"primary":"new-primary-cluster-name"}}}' + done + ``` + +**What happens:** + +1. The operator detects the primary change +2. The old primary becomes a replica after flushing writes +3. The new primary cluster scales up (if HA) and starts to accept writes +4. Replication direction reverses (new primary → replicas including old primary) + +### Step 3: Monitor Failover Progress + +Watch operator logs and DocumentDB status: + +```bash +# Watch DocumentDB status on new primary +watch kubectl --context new-primary get documentdb -n documentdb-preview-ns + +# Monitor operator logs on new primary +kubectl --context new-primary logs -n documentdb-operator \ + -l app.kubernetes.io/name=documentdb-operator -f + +# Check pod status +kubectl --context new-primary get pods -n documentdb-preview-ns -w +``` + +### Step 4: Verify New Primary + +Confirm the new primary accepts writes: + +```bash +# Port forward to new primary +kubectl --context new-primary port-forward \ + -n documentdb-preview-ns svc/documentdb-preview 10260:10260 & + +# Connect with mongosh +mongosh "mongodb://admin:password@localhost:10260/?tls=true&tlsAllowInvalidCertificates=true" + +# Test write operation +db.testCollection.insertOne({ + message: "Write test after failover", + timestamp: new Date() +}) + +# Should succeed without errors +``` + +### Step 5: Verify Old Primary as Replica + +Check that the old primary is now replicating from the new primary: + +```bash +# Verify replication status ON NEW PRIMARY +kubectl --context new-primary exec -it -n documentdb-preview-ns \ + documentdb-preview-1 -- psql -U postgres -c "SELECT * FROM pg_stat_replication;" +``` + +You should see the old primary listed as a replica receiving replication stream. + +### Step 6: Post-Failover Validation + +Run comprehensive checks: + +```bash +# 1. Verify all clusters are in sync +for context in cluster1 cluster2 cluster3; do + echo "=== $context ===" + kubectl --context $context get documentdb -n documentdb-preview-ns +done + +# 2. Check application health +kubectl --context new-primary get pods -n app-namespace + +# 3. Review metrics and logs for errors +# (use your monitoring system: Prometheus, Grafana, CloudWatch, etc.) + +# 4. Verify data consistency (read from all replicas) +``` + +## Unplanned Failover Procedure (Disaster Recovery) + +Use this procedure when the primary cluster is unavailable and you need to immediately promote a replica. + +!!! danger "Data Loss Risk" + Unplanned failover may result in data loss if the primary cluster failed before replicating recent writes. Assess replication lag before deciding which replica to promote. + +### Step 1: Assess the Situation + +Determine the scope of the outage: + +```bash +# 1. Check primary cluster accessibility +kubectl --context primary get nodes +# If this fails, primary cluster is unreachable + +# 2. Check replica cluster health +kubectl --context replica1 get documentdb -n documentdb-preview-ns +kubectl --context replica2 get documentdb -n documentdb-preview-ns + +# 3. Check cloud provider status pages for regional outages +``` + +### Step 2: Select Target Replica + +Choose which replica to promote based on: + +- **Replication lag:** Prefer the replica with the lowest lag (most recent data) +- **Geographic location:** Consider application proximity +- **Cluster health:** Ensure target cluster is fully operational + +If you cannot query the primary, check the last known replication status from monitoring dashboards or logs. + +### Step 3: Promote Replica to Primary + +Immediately promote the selected replica to become the new primary. + +=== "Using kubectl-documentdb Plugin (Recommended)" + + ```bash + kubectl documentdb promote \ + --documentdb documentdb-preview \ + --namespace documentdb-preview-ns \ + --target-cluster replica-cluster-name \ + --hub-context hub \ + --failover \ + --wait-timeout 15m + ``` + + The plugin handles the change to clusterList and primary and monitors for successful convergence. Use `--skip-wait` if you need to return immediately and verify manually. + +=== "Using kubectl patch with KubeFleet" + + ```bash + # Remove failed primary from cluster list and set new primary in one command + kubectl --context hub patch documentdb documentdb-preview \ + -n documentdb-preview-ns \ + --type='merge' \ + -p '{"spec":{"clusterReplication":{"primary":"replica-cluster-name","clusterList":[{"name":"replica-cluster-name"},{"name":"other-replica-cluster-name"}]}}}' + ``` + + Replace the `clusterList` entries with your actual list of healthy clusters, excluding the failed primary. + +=== "Using kubectl without KubeFleet" + + ```bash + # Update on all accessible clusters + # Remove failed primary from cluster list and set new primary in one command + for context in replica1 replica2; do + kubectl --context $context patch documentdb documentdb-preview \ + -n documentdb-preview-ns \ + --type='merge' \ + -p '{"spec":{"clusterReplication":{"primary":"replica-cluster-name","clusterList":[{"name":"replica-cluster-name"},{"name":"other-replica-cluster-name"}]}}}' + done + ``` + + Replace the `clusterList` entries with your actual list of healthy clusters, excluding the failed primary. + +**What happens:** + +1. The operator detects the primary and cluster list changes +2. The new primary cluster scales up (if HA) and starts to accept writes +3. The old primary is removed from replication + +### Step 4: Verify New Primary + +Confirm the promoted replica is accepting writes: + +```bash +# Check status +kubectl --context new-primary get documentdb documentdb-preview \ + -n documentdb-preview-ns + +# Test write access +kubectl --context new-primary port-forward \ + -n documentdb-preview-ns svc/documentdb-preview 10260:10260 & + +mongosh "mongodb://admin:password@localhost:10260/?tls=true&tlsAllowInvalidCertificates=true" +db.testCollection.insertOne({message: "DR failover test"}) +``` + +### Step 5: Monitor Recovery + +```bash +# Application pod logs +kubectl --context app-cluster logs -l app=your-app --tail=100 -f + +# DocumentDB operator logs +kubectl --context new-primary logs -n documentdb-operator \ + -l app.kubernetes.io/name=documentdb-operator -f +``` + +### Step 6: Handle Failed Primary Recovery + +When the failed primary Kubernetes cluster recovers, you need to re-add it to the DocumentDB cluster +as a replica. For detailed guidance on adding a region back to your DocumentDB cluster, +see the [add region playground guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/fleet-add-region/README.md). diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/overview.md b/docs/operator-public-documentation/preview/multi-region-deployment/overview.md new file mode 100644 index 00000000..7b0a4f48 --- /dev/null +++ b/docs/operator-public-documentation/preview/multi-region-deployment/overview.md @@ -0,0 +1,181 @@ +--- +title: Multi-Region Deployment Overview +description: Understand multi-region DocumentDB deployments for disaster recovery, low-latency access, and compliance with geographic data residency requirements. +tags: + - multi-region + - disaster-recovery + - high-availability + - architecture +--- + +# Multi-Region Deployment Overview + +Multi-region deployment distributes DocumentDB clusters across multiple geographic regions, enabling disaster recovery, low-latency access for global users, and compliance with data residency regulations. + +## Use Cases + +### Disaster Recovery (DR) + +Protect against regional outages by maintaining database replicas in separate geographic regions. If the primary region fails, promote a replica in another region to maintain service availability. + +### Low-Latency Global Access + +Reduce application response times and distribute load by deploying read replicas closer to end users. + +### Compliance and Data Residency + +Meet regulatory requirements for data storage location by deploying replicas in specific regions. Ensure that data resides within required geographic boundaries while maintaining availability. + +## Architecture + +### Primary-Replica Model + +DocumentDB uses a primary-replica architecture where: + +- **Primary cluster:** Accepts both read and write operations +- **Replica clusters:** Accept read-only operations and replicate changes from primary +- **Replication:** PostgreSQL streaming replication propagates changes from primary to replicas + +### Cluster Components + +Each regional DocumentDB cluster includes: + +- **Gateway containers:** Provide MongoDB-compatible API and connection management +- **PostgreSQL containers:** Store data and handle replication (managed by CloudNative-PG) +- **Persistent storage:** Regional block storage for data persistence +- **Service endpoints:** LoadBalancer or ClusterIP for client connections +- **Self-name ConfigMap:** A config map giving the name of the cluster (must match clusterList[].name) + +### Replication Configuration + +Multi-region replication is configured in the `DocumentDB` resource: + +```yaml +apiVersion: documentdb.io/preview +kind: DocumentDB +metadata: + name: documentdb-preview + namespace: documentdb-preview-ns +spec: + clusterReplication: + primary: member-eastus2-cluster + clusterList: + - name: member-westus3-cluster + - name: member-uksouth-cluster + - name: member-eastus2-cluster +``` + +The operator handles: + +- Creating replica clusters in specified regions +- Establishing streaming replication from primary to replicas +- Monitoring replication lag and health +- Coordinating failover operations + +## Network Requirements + +### Inter-Region Connectivity + +Use cloud-native VNet/VPC peering for direct cluster-to-cluster communication: + +- **Azure:** VNet peering between AKS clusters +- **AWS:** VPC peering between EKS clusters +- **GCP:** VPC peering between GKE clusters + +### Port Requirements + +DocumentDB replication requires these ports between Kubernetes clusters: + +| Port | Protocol | Purpose | +|------|----------|---------| +| 5432 | TCP | PostgreSQL streaming replication | +| 443 | TCP | Kubernetes API (for KubeFleet, optional) | + +Ensure firewall rules and network security groups allow traffic on these ports between regional clusters. + +### DNS and Service Discovery + +The operator will by default use the DocumentDB cluster name and its corresponding CNPG cluster's +generated service to connect the multi-regional clusters, and it will be up to the user to coordinate +that connection between the clusters. Alternatively, there are two integrations that can be used. + +#### Istio Networking + +If Istio is installed on the cluster, Istio networking is enabled, and an east-west gateway is present +connecting each Kubernetes cluster then the operator will generate services that automatically route the +default service names across regions. + +#### Fleet Networking + +If Fleet networking is installed on each of your clusters, then instead of populating the connections +with default service names, the Operator will create ServiceExports and MultiClusterServices on each +Kubernetes cluster then, use those generated cross-regional services to connect the CNPG instances to one +another. + +## Deployment Models + +### Managed Fleet Orchestration + +Use a multi-cluster orchestration system such as KubeFleet to manage deployments of resources across clusters and centrally manage changes, ensuring +your topology stays synchronized between regions. + +**Example:** [AKS Fleet Deployment](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) + +### Manual Multi-Cluster Management + +Deploy DocumentDB resources individually to each Kubernetes cluster, manually ensuring that each DocumentDB CRD +is in sync. + +## Performance Considerations + +### Replication Lag + +Distance between regions affects replication lag. Monitor replication lag with PostgreSQL metrics and adjust application read patterns accordingly. + +### Storage Performance + +Each region requires independent storage resources, and each replica must have an equal or greater volume of +available storage compared to the primary. + +## Security Considerations + +### TLS Encryption + +Enable TLS for all connections: + +- **Client-to-gateway:** Encrypt application connections (see [TLS Configuration](../configuration/tls.md)) +- **Replication traffic:** PostgreSQL SSL for inter-cluster replication +- **Service mesh:** mTLS for cross-cluster service communication + +### Authentication and Authorization + +Credentials must be synchronized across regions: + +- **Kubernetes Secrets:** Replicate secrets to all clusters (KubeFleet handles this automatically) +- **RBAC policies:** Apply consistent access controls across regions +- **Credential rotation:** Coordinate credential changes across all clusters + +### Network Security + +Restrict network access between regions: + +- **Private connectivity:** Use VNet/VPC peering instead of public internet +- **Network policies:** Kubernetes NetworkPolicy to limit pod-to-pod communication +- **Firewall rules:** Allow only required ports between regional clusters + +## Monitoring and Observability + +Track multi-region health and performance: + +- **Replication lag:** Monitor `pg_stat_replication` metrics +- **Cluster health:** Pod status, resource usage, connection counts +- **Network metrics:** Bandwidth, latency, packet loss between regions +- **Application performance:** Request latency, error rates per region + +See [Telemetry Examples](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/telemetry/README.md) for OpenTelemetry, Prometheus, and Grafana setup. + +## Next Steps + +- [Multi-Region Setup Guide](setup.md) - Deploy your first multi-region DocumentDB cluster +- [Failover Procedures](failover-procedures.md) - Learn how to handle planned and unplanned failovers +- [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - Complete Azure multi-region automation diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/setup.md b/docs/operator-public-documentation/preview/multi-region-deployment/setup.md new file mode 100644 index 00000000..a9fd0042 --- /dev/null +++ b/docs/operator-public-documentation/preview/multi-region-deployment/setup.md @@ -0,0 +1,605 @@ +--- +title: Multi-Region Setup Guide +description: Step-by-step instructions for deploying DocumentDB across multiple Kubernetes clusters with replication, prerequisites, and configuration examples. +tags: + - multi-region + - setup + - deployment + - replication +--- + +# Multi-Region Setup Guide + +This guide walks through deploying DocumentDB across multiple Kubernetes clusters with regional replication. + +## Prerequisites + +### Infrastructure Requirements + +Before deploying DocumentDB in multi-region mode, ensure you have: + +- **Multiple Kubernetes clusters:** 2+ clusters in different regions +- **Network connectivity:** Clusters can communicate over private network or internet +- **Storage:** CSI-compatible storage class in each cluster with snapshot support +- **Load balancing:** LoadBalancer or Ingress capability for external access (optional) + +### Required Components + +Install these components on **all** Kubernetes clusters: + +#### 1. cert-manager + +Required for TLS certificate management between clusters. + +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --set installCRDs=true +``` + +Verify installation: + +```bash +kubectl get pods -n cert-manager +``` + +See [Get Started](../index.md#install-cert-manager) for detailed cert-manager setup. + +#### 2. DocumentDB Operator + +Install the operator on each Kubernetes cluster. If using KubeFleet (recommended), install once on the hub cluster and let it propagate to all member clusters. See [Fleet Setup](#with-fleet-management-recommended) below. + +```bash +helm repo add documentdb https://documentdb.github.io/documentdb-kubernetes-operator +helm repo update +helm install documentdb-operator documentdb/documentdb-operator \ + --namespace documentdb-operator \ + --create-namespace +``` + +Verify on each cluster: + +```bash +kubectl get pods -n documentdb-operator +``` + +#### Self Label + +Each Kubernetes cluster participating in multi-region deployment must identify itself with a unique cluster name. Create a ConfigMap on each cluster: + +```bash +# Run on each cluster - replace with actual cluster name +CLUSTER_NAME="member-eastus2-cluster" # e.g., member-eastus2-cluster, member-westus3-cluster + +kubectl create configmap cluster-identity \ + --namespace kube-system \ + --from-literal=cluster-name="${CLUSTER_NAME}" +``` + +Important: The cluster name in this ConfigMap must exactly match one of the member cluster names in the DocumentDB resource's spec.clusterReplication.clusterList[].name. See Cluster Identification for details. + +This is needed since the DocumentDB CRD will be the same across primaries and replicas for ease of use, but the clusters need +to be able to identify where in the topology they lie. + +### Network Configuration + +#### VNet/VPC Peering (Single Cloud Provider) + +For clusters within the same cloud provider, configure VNet or VPC peering: + +=== "Azure (AKS)" + + Create VNet peering between all AKS cluster VNets: + + ```bash + az network vnet peering create \ + --name peer-to-cluster2 \ + --resource-group cluster1-rg \ + --vnet-name cluster1-vnet \ + --remote-vnet /subscriptions/.../cluster2-vnet \ + --allow-vnet-access + ``` + + Repeat for all cluster pairs in a full mesh topology. + + See [AKS Fleet Deployment](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) for automated Azure multi-region setup with VNet peering. + +=== "AWS (EKS)" + + Create VPC peering connections between EKS cluster VPCs: + + ```bash + aws ec2 create-vpc-peering-connection \ + --vpc-id vpc-cluster1 \ + --peer-vpc-id vpc-cluster2 \ + --peer-region us-west-2 + ``` + + Update route tables to allow traffic between VPCs. + +=== "GCP (GKE)" + + Enable VPC peering between GKE cluster networks: + + ```bash + gcloud compute networks peerings create peer-to-cluster2 \ + --network=cluster1-network \ + --peer-network=cluster2-network + ``` + +#### Networking management + +Configure inter-cluster networking using `spec.clusterReplication.crossCloudNetworkingStrategy`: + +**Valid options:** + +- **None** (default): Direct service-to-service connections using standard Kubernetes service names for the PostgreSQL backend server +- **Istio**: Use Istio service mesh for cross-cluster connectivity +- **AzureFleet**: Use Azure Fleet Networking for cross-cluster communication (separate from KubeFleet) + +**Example:** + +```yaml +spec: + clusterReplication: + primary: member-eastus2-cluster + crossCloudNetworkingStrategy: Istio # or AzureFleet, None + clusterList: + - name: member-eastus2-cluster + - name: member-westus3-cluster +``` + +## Deployment Options + +Choose a deployment approach based on your infrastructure and operational preferences. + +### With KubeFleet (Recommended) + +KubeFleet systems simplify multi-region operations by: + +- **Centralized control:** Define resources once, deploy everywhere +- **Automatic propagation:** Resources sync to member clusters automatically +- **Coordinated updates:** Roll out changes across regions consistently + +#### Step 1: Deploy Fleet Infrastructure + +Install KubeFleet or another fleet management system: + +```bash +# Example: KubeFleet on hub cluster +kubectl apply -f https://github.com/kubefleet-dev/kubefleet/releases/latest/download/install.yaml +``` + +Configure member clusters to join the fleet. See the [AKS Fleet Deployment guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) for a complete automated setup example. + +#### Step 2: Install DocumentDB Operator via KubeFleet + +Create a `ClusterResourcePlacement` to deploy the operator to all member clusters: + +```yaml title="documentdb-operator-crp.yaml" +apiVersion: placement.kubernetes-fleet.io/v1beta1 +kind: ClusterResourcePlacement +metadata: + name: documentdb-operator-crp + namespace: fleet-system +spec: + resourceSelectors: + - group: "" + kind: Namespace + name: documentdb-operator + version: v1 + placement: + strategy: + type: PickAll # Deploy to all member clusters +``` + +Apply to hub cluster: + +```bash +kubectl --context hub apply -f documentdb-operator-crp.yaml +``` + +The fleet controller will install the operator on all member clusters automatically. + +#### Step 3: Deploy Multi-Region DocumentDB + +Create a DocumentDB resource with replication configuration: + +```yaml title="multi-region-documentdb.yaml" +apiVersion: documentdb.io/preview +kind: DocumentDB +metadata: + name: documentdb-preview + namespace: documentdb-preview-ns +spec: + nodeCount: 2 + instancesPerNode: 1 + resource: + storage: + pvcSize: 100Gi + storageClass: managed-csi # Use appropriate storage class per cloud + exposeViaService: + serviceType: LoadBalancer + tls: + gateway: + mode: SelfSigned + clusterReplication: # (1)! + primary: member-eastus2-cluster # (2)! + clusterList: # (3)! + - name: member-westus3-cluster + - name: member-uksouth-cluster + - name: member-eastus2-cluster + documentDbCredentialSecret: documentdb-secret +``` + +1. The `clusterReplication` section enables multi-region deployment +2. `primary` specifies which cluster accepts write operations +3. `clusterList` lists all member clusters that will host DocumentDB instances (including the primary) + +Create the credentials secret: + +```bash +kubectl create secret generic documentdb-secret \ + --namespace documentdb-preview-ns \ + --from-literal=password='YourSecurePassword123!' +``` + +Apply via KubeFleet to propagate to all clusters: + +```yaml title="documentdb-crp.yaml" +apiVersion: placement.kubernetes-fleet.io/v1beta1 +kind: ClusterResourcePlacement +metadata: + name: documentdb-crp + namespace: fleet-system +spec: + resourceSelectors: + - group: documentdb.io + kind: DocumentDB + name: documentdb-preview + version: preview + - group: "" + version: v1 + kind: Secret + name: documentdb-secret + placement: + strategy: + type: PickAll +``` + +Apply both resources: + +```bash +kubectl --context hub apply -f multi-region-documentdb.yaml +kubectl --context hub apply -f documentdb-crp.yaml +``` + +### Without KubeFleet + +If not using KubeFleet, deploy DocumentDB resources to each cluster individually. + +#### Step 1: Identify Cluster Names + +Determine the name for each Kubernetes cluster. These names are used in the replication configuration: + +```bash +# List your clusters +kubectl config get-contexts + +# Or for cloud-managed clusters: +az aks list --query "[].name" -o table # Azure +aws eks list-clusters --query "clusters" --output table # AWS +gcloud container clusters list --format="table(name)" # GCP +``` + +#### Step 2: Create Cluster Identification + +On each cluster, create a ConfigMap to identify the cluster name: + +```bash +# Run on each cluster +CLUSTER_NAME="cluster-region-name" # e.g., member-eastus2-cluster + +kubectl create configmap cluster-identity \ + --namespace kube-system \ + --from-literal=cluster-name="${CLUSTER_NAME}" +``` + +#### Step 3: Deploy DocumentDB to Primary Cluster + +On the primary cluster: + +```yaml title="primary-documentdb.yaml" +apiVersion: documentdb.io/preview +kind: DocumentDB +metadata: + name: documentdb-preview + namespace: documentdb-preview-ns +spec: + nodeCount: 2 + instancesPerNode: 1 + resource: + storage: + pvcSize: 100Gi + exposeViaService: + serviceType: LoadBalancer + tls: + gateway: + mode: SelfSigned + clusterReplication: + primary: cluster-primary-name # This cluster's name + clusterList: + - name: cluster-primary-name + - name: cluster-replica1-name + - name: cluster-replica2-name + documentDbCredentialSecret: documentdb-secret +``` + +Apply to primary: + +```bash +kubectl --context primary apply -f primary-documentdb.yaml +``` + +#### Step 4: Deploy DocumentDB to Replica Clusters + +Use the **same YAML** on each replica cluster. The operator detects whether the cluster is primary or replica based on the `clusterReplication.primary` field. + +```bash +kubectl --context replica1 apply -f primary-documentdb.yaml +kubectl --context replica2 apply -f primary-documentdb.yaml +``` + +Each cluster will: + +- **Primary:** Run a read-write DocumentDB cluster +- **Replicas:** Run read-only DocumentDB clusters replicating from primary + +## Configuration Details + +### Replication Configuration + +The `clusterReplication` section controls multi-region behavior: + +```yaml +spec: + clusterReplication: + primary: member-eastus2-cluster # Write cluster + clusterList: # All participating member clusters + - name: member-westus3-cluster + - name: member-uksouth-cluster + - name: member-eastus2-cluster +``` + +**Key points:** + +- **Primary cluster:** The cluster name specified in `primary` accepts read and write operations +- **Replica clusters:** All other clusters in `clusterList` are read-only replicas +- **Include primary in list:** The primary cluster name must appear in the `clusterList` +- **Cluster name uniqueness:** Each cluster must have a unique name across all regions + +### Credential Synchronization + +The DocumentDB operator expects the same credentials secret on all clusters. With KubeFleet, create the secret on the hub (or in fleet-managed namespace) and it propagates automatically. Without fleet management, create the secret manually on each cluster: + +```bash +# Same secret on all clusters +kubectl --context cluster1 create secret generic documentdb-secret \ + --namespace documentdb-preview-ns \ + --from-literal=password='YourSecurePassword123!' + +kubectl --context cluster2 create secret generic documentdb-secret \ + --namespace documentdb-preview-ns \ + --from-literal=password='YourSecurePassword123!' + +# And so on... +``` + +!!! warning "Credential Management" + Synchronize password changes across all clusters manually if not using fleet management. Mismatched credentials will break replication. + +### Storage Configuration + +Each cluster in a multi-region deployment can use different storage classes. Configure storage at the global level or override per member cluster: + +**Global storage configuration:** + +```yaml +spec: + resource: + storage: + pvcSize: 100Gi + storageClass: default-storage-class # Used by all clusters +``` + +**Per-cluster storage override:** + +```yaml +spec: + resource: + storage: + pvcSize: 100Gi + storageClass: default-storage-class # Fallback + clusterReplication: + primary: member-eastus2-cluster + clusterList: + - name: member-westus3-cluster + storageClass: managed-csi-premium # Override for this cluster + - name: member-uksouth-cluster + storageClass: azuredisk-standard-ssd # Override for this cluster + - name: member-eastus2-cluster + # Uses global storageClass +``` + +**Cloud-specific storage classes:** + +=== "Azure (AKS)" + + ```yaml + - name: member-eastus2-cluster + storageClass: managed-csi # Azure Disk managed CSI driver + environment: aks + ``` + +=== "AWS (EKS)" + + ```yaml + - name: member-us-east-1-cluster + storageClass: gp3 # AWS EBS GP3 volumes + environment: eks + ``` + +=== "GCP (GKE)" + + ```yaml + - name: member-us-central1-cluster + storageClass: standard-rwo # GCP Persistent Disk + environment: gke + ``` + +### Service Exposure + +Configure how DocumentDB is exposed in each region: + +=== "LoadBalancer" + + **Best for:** Production deployments with external access + + ```yaml + spec: + exposeViaService: + serviceType: LoadBalancer + ``` + + Each cluster gets a public IP for client connections. When using the `environment` configuration at either + the DocumentDB cluster or Kubernetes cluster level, the tags for the LoadBalancer will change. See the + cloud-specific setup docs for more details. + +=== "ClusterIP" + + **Best for:** In-cluster access only or Ingress-based routing + + ```yaml + spec: + exposeViaService: + serviceType: ClusterIP + ``` + + Clients must access DocumentDB through Ingress or service mesh. + +## Verification + +### Check Operator Status + +Verify the operator is running on all clusters: + +```bash +# With KubeFleet +kubectl --context hub get clusterresourceplacement + +# Without fleet management (run on each cluster) +kubectl --context cluster1 get pods -n documentdb-operator +kubectl --context cluster2 get pods -n documentdb-operator +``` + +### Check DocumentDB Status + +View DocumentDB status on each cluster: + +```bash +kubectl --context primary get documentdb -n documentdb-preview-ns +kubectl --context replica1 get documentdb -n documentdb-preview-ns +kubectl --context replica2 get documentdb -n documentdb-preview-ns +``` + +Expected output shows `Ready` status and role (primary or replica): + +``` +NAME STATUS ROLE AGE +documentdb-preview Ready primary 10m +``` + +## Troubleshooting + +### Replication Not Established + +If replicas don't receive data from the primary: + +1. **Verify network connectivity:** + + ```bash + # From replica cluster, test connectivity to primary + kubectl --context replica1 run test-pod --rm -it --image=nicolaka/netshoot -- \ + nc -zv primary-service-endpoint 5432 + ``` + +2. **Check PostgreSQL replication status on primary:** + + ```bash + kubectl --context primary exec -it -n documentdb-preview-ns \ + documentdb-preview-1 -- psql -U postgres -c "SELECT * FROM pg_stat_replication;" + ``` + +3. **Review operator logs:** + + ```bash + kubectl --context primary logs -n documentdb-operator \ + -l app.kubernetes.io/name=documentdb-operator --tail=100 + ``` + +### Cluster Name Mismatch + +If a cluster doesn't recognize itself as primary or replica: + +1. **Check cluster-identity ConfigMap:** + + ```bash + kubectl --context cluster1 get configmap cluster-identity \ + -n kube-system -o jsonpath='{.data.cluster-name}' + ``` + +2. **Verify name matches DocumentDB spec:** + + The returned name must exactly match one of the cluster names in `spec.clusterReplication.clusterList[*].name`. + +3. **Update ConfigMap if incorrect:** + + ```bash + kubectl --context cluster1 create configmap cluster-identity \ + --namespace kube-system \ + --from-literal=cluster-name="correct-cluster-name" \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + +### Storage Issues + +If PVCs aren't provisioning: + +1. **Verify storage class exists:** + + ```bash + kubectl --context cluster1 get storageclass + ``` + +2. **Check for VolumeSnapshotClass (required for backups):** + + ```bash + kubectl --context cluster1 get volumesnapshotclass + ``` + +3. **Review PVC events:** + + ```bash + kubectl --context cluster1 get events -n documentdb-preview-ns \ + --field-selector involvedObject.kind=PersistentVolumeClaim + ``` + +## Next Steps + +- [Failover Procedures](failover-procedures.md) - Learn how to perform planned and unplanned fail overs +- [Backup and Restore](../backup-and-restore.md) - Configure multi-region backup strategies +- [TLS Configuration](../configuration/tls.md) - Secure connections with proper TLS certificates +- [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - Automated Azure multi-region setup diff --git a/documentdb-playground/aks-fleet-deployment/README.md b/documentdb-playground/aks-fleet-deployment/README.md index 853b1c7f..4b107a8d 100644 --- a/documentdb-playground/aks-fleet-deployment/README.md +++ b/documentdb-playground/aks-fleet-deployment/README.md @@ -158,7 +158,7 @@ Load aliases: source ~/.bashrc ``` -## Fleet Management +## KubeFleet ```bash # List member clusters diff --git a/mkdocs.yml b/mkdocs.yml index 1bc4e919..f642b5c6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,10 @@ nav: - Networking: preview/configuration/networking.md - TLS: preview/configuration/tls.md - Storage: preview/configuration/storage.md + - Multi-Region Deployment: + - Overview: preview/multi-region-deployment/overview.md + - Setup Guide: preview/multi-region-deployment/setup.md + - Failover Procedures: preview/multi-region-deployment/failover-procedures.md - Advanced Configuration: preview/advanced-configuration/README.md - Backup and Restore: preview/backup-and-restore.md - API Reference: preview/api-reference.md From 277aa1c4db35174b20eddf6359e150cdb71c5266 Mon Sep 17 00:00:00 2001 From: Alexander Laye Date: Mon, 16 Mar 2026 16:49:46 -0400 Subject: [PATCH 2/3] temp Signed-off-by: Alexander Laye --- .../failover-procedures.md | 45 ++++----- .../multi-region-deployment/overview.md | 93 +++++++++++-------- .../preview/multi-region-deployment/setup.md | 18 ++-- 3 files changed, 88 insertions(+), 68 deletions(-) diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md b/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md index e1139348..331a454e 100644 --- a/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md +++ b/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md @@ -8,13 +8,11 @@ tags: - operations --- -# Multi-Region Failover Procedures - -This guide provides runbooks for switching the primary DocumentDB cluster between regions during planned maintenance or unplanned outages. - ## Overview -A **failover** promotes a replica cluster to become the new primary, making it accept write operations. The previous primary (if still available) becomes a replica replicating from the new primary. +A **failover** promotes a replica cluster to become the new primary, making it +accept write operations. The previous primary (if still available) becomes a replica +replicating from the new primary. **When to perform failover:** @@ -27,31 +25,34 @@ A **failover** promotes a replica cluster to become the new primary, making it a ### Planned Failover -This is a failover where the primary is safely demoted, writes are all flushed, and then the new primary is -promoted. This kind of failover has no data loss, a set window where writes aren't accepted, and the same number -of replicas before and after. +This is a failover where the primary is safely demoted, writes are all flushed, +and then the new primary is promoted. This kind of failover has no data loss, a +set window where writes aren't accepted, and the same number of replicas before +and after. ### Unplanned Failover (Disaster Recovery) -This is a failover where the primary becomes unavailable and has to be forced out of the DocumentDB cluster -entirely. Downtime is relative to how long it takes to detect the degradation of the primary, as well as the -time it takes to spin up HA on the new primary (if HA is enabled). There is a potential for lost writes to -the primary, but with High-Availability the client should know when writes weren't committed to remotes. -By the end of this type of failover, there will be one fewer region in the DocumentDB cluster and likely steps -will be needed to add the new region. See the [add region playground guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/fleet-add-region/README.md) +This is a failover where the primary becomes unavailable and has to be forced out +of the DocumentDB cluster entirely. Downtime depends on how quickly primary degradation +is detected and, if HA is enabled, how long it takes to scale up the new primary. +Some writes to the failed primary might be lost, but with high availability enabled, +clients can determine when writes were not committed to replicas. After an unplanned +failover, the DocumentDB cluster has one fewer region, and you will need to add +the region back when the Kubernetes cluster is back online, or add a replacement +region. See the [add region playground guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/fleet-add-region/README.md) for an example. ## Prerequisites Before performing any failover: -- [ ] **Replica health:** Target replica cluster is running and replication is current -- [ ] **Network access:** You have kubectl access to all clusters involved -- [ ] **Backup available:** Recent backup exists for rollback if needed -- [ ] **Monitoring:** Metrics and logs are accessible for verification -- [ ] **Communication:** Stakeholders are notified (for planned failover) -- [ ] **Application readiness:** Applications can handle brief connection interruption -- [ ] **kubectl-documentdb plugin:** Install the plugin for streamlined failover operations [kubectl-plugin.md](../kubectl-plugin.md) +- **Replica health:** Target replica cluster is running and replication is current +- **Network access:** You have kubectl access to all clusters involved +- **Backup available:** Recent backup exists for rollback if needed +- **Monitoring:** Metrics and logs are accessible for verification +- **Communication:** Stakeholders are notified (for planned failover) +- **Application readiness:** Applications can handle brief connection interruption +- **kubectl-documentdb plugin:** Install the plugin for streamlined failover operations [kubectl-plugin](../kubectl-plugin.md) ### Check Current Replication Status @@ -69,7 +70,7 @@ kubectl --context primary exec -it -n documentdb-preview-ns \ Expected output shows active replication to all replicas: -``` +```text pid | usename | application_name | client_addr | state | sent_lsn | write_lsn | flush_lsn | replay_lsn | sync_state -----+----------+------------------+-------------+-----------+------------+------------+------------+------------+------------ 123 | postgres | replica1 | 10.2.1.5 | streaming | 0/30000A8 | 0/30000A8 | 0/30000A8 | 0/30000A8 | async diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/overview.md b/docs/operator-public-documentation/preview/multi-region-deployment/overview.md index 7b0a4f48..af0a94a1 100644 --- a/docs/operator-public-documentation/preview/multi-region-deployment/overview.md +++ b/docs/operator-public-documentation/preview/multi-region-deployment/overview.md @@ -1,6 +1,8 @@ --- title: Multi-Region Deployment Overview -description: Understand multi-region DocumentDB deployments for disaster recovery, low-latency access, and compliance with geographic data residency requirements. +description: Understand multi-region DocumentDB deployments for disaster + recovery, low-latency access, and compliance with geographic data residency + requirements. tags: - multi-region - disaster-recovery @@ -8,23 +10,24 @@ tags: - architecture --- -# Multi-Region Deployment Overview - -Multi-region deployment distributes DocumentDB clusters across multiple geographic regions, enabling disaster recovery, low-latency access for global users, and compliance with data residency regulations. - ## Use Cases ### Disaster Recovery (DR) -Protect against regional outages by maintaining database replicas in separate geographic regions. If the primary region fails, promote a replica in another region to maintain service availability. +Protect against regional outages by maintaining database replicas in separate +geographic regions. If the primary region fails, promote a replica in another +region to maintain service availability. ### Low-Latency Global Access -Reduce application response times and distribute load by deploying read replicas closer to end users. +Reduce application response times and distribute load by deploying read replicas +closer to end users. ### Compliance and Data Residency -Meet regulatory requirements for data storage location by deploying replicas in specific regions. Ensure that data resides within required geographic boundaries while maintaining availability. +Meet regulatory requirements for data storage location by deploying replicas in +specific regions. Ensure that data resides within required geographic +boundaries while maintaining availability. ## Architecture @@ -33,18 +36,22 @@ Meet regulatory requirements for data storage location by deploying replicas in DocumentDB uses a primary-replica architecture where: - **Primary cluster:** Accepts both read and write operations -- **Replica clusters:** Accept read-only operations and replicate changes from primary -- **Replication:** PostgreSQL streaming replication propagates changes from primary to replicas +- **Replica clusters:** Accept read-only operations and replicate changes from + primary +- **Replication:** PostgreSQL streaming replication propagates changes from + primary to replicas ### Cluster Components Each regional DocumentDB cluster includes: - **Gateway containers:** Provide MongoDB-compatible API and connection management -- **PostgreSQL containers:** Store data and handle replication (managed by CloudNative-PG) +- **PostgreSQL containers:** Store data and handle replication (managed by + CloudNative-PG) - **Persistent storage:** Regional block storage for data persistence - **Service endpoints:** LoadBalancer or ClusterIP for client connections -- **Self-name ConfigMap:** A config map giving the name of the cluster (must match clusterList[].name) +- **Self-name ConfigMap:** A ConfigMap giving the name of the cluster (must + match clusterList[].name) ### Replication Configuration @@ -79,7 +86,7 @@ The operator handles: Use cloud-native VNet/VPC peering for direct cluster-to-cluster communication: - **Azure:** VNet peering between AKS clusters -- **AWS:** VPC peering between EKS clusters +- **AWS:** VPC peering between EKS clusters - **GCP:** VPC peering between GKE clusters ### Port Requirements @@ -91,51 +98,57 @@ DocumentDB replication requires these ports between Kubernetes clusters: | 5432 | TCP | PostgreSQL streaming replication | | 443 | TCP | Kubernetes API (for KubeFleet, optional) | -Ensure firewall rules and network security groups allow traffic on these ports between regional clusters. +Ensure firewall rules and network security groups allow traffic on these ports +between regional clusters. ### DNS and Service Discovery -The operator will by default use the DocumentDB cluster name and its corresponding CNPG cluster's -generated service to connect the multi-regional clusters, and it will be up to the user to coordinate -that connection between the clusters. Alternatively, there are two integrations that can be used. +The operator uses the DocumentDB cluster name and its corresponding CNPG +cluster's generated service to connect the multi-regional clusters, and you +must coordinate that connection between the clusters. Alternatively, there are +two integrations with existing services. #### Istio Networking -If Istio is installed on the cluster, Istio networking is enabled, and an east-west gateway is present -connecting each Kubernetes cluster then the operator will generate services that automatically route the -default service names across regions. +If Istio is installed on the Kubernetes cluster, Istio networking is enabled, +and an east-west gateway is present connecting each Kubernetes cluster, then +the operator generates services that automatically route the default service +names across regions. #### Fleet Networking -If Fleet networking is installed on each of your clusters, then instead of populating the connections -with default service names, the Operator will create ServiceExports and MultiClusterServices on each -Kubernetes cluster then, use those generated cross-regional services to connect the CNPG instances to one -another. +If Fleet networking is installed on each of your Kubernetes clusters, then +instead of populating the connections with default service names, the operator +creates ServiceExports and MultiClusterServices on each Kubernetes cluster, +then use those generated cross-regional services to connect the CNPG instances +to one another. ## Deployment Models ### Managed Fleet Orchestration -Use a multi-cluster orchestration system such as KubeFleet to manage deployments of resources across clusters and centrally manage changes, ensuring -your topology stays synchronized between regions. +Use a multi-cluster orchestration system such as KubeFleet to manage +deployments of resources across Kubernetes clusters and centrally manage +changes, ensuring your topology stays synchronized between regions. **Example:** [AKS Fleet Deployment](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) ### Manual Multi-Cluster Management -Deploy DocumentDB resources individually to each Kubernetes cluster, manually ensuring that each DocumentDB CRD -is in sync. +Deploy DocumentDB resources individually to each Kubernetes cluster, manually +ensuring that each DocumentDB CRD is in sync. ## Performance Considerations ### Replication Lag -Distance between regions affects replication lag. Monitor replication lag with PostgreSQL metrics and adjust application read patterns accordingly. +Distance between regions affects replication lag. Monitor replication lag with +PostgreSQL metrics and adjust application read patterns accordingly. ### Storage Performance -Each region requires independent storage resources, and each replica must have an equal or greater volume of -available storage compared to the primary. +Each region requires independent storage resources, and each replica must have +an equal or greater volume of available storage compared to the primary. ## Security Considerations @@ -151,7 +164,8 @@ Enable TLS for all connections: Credentials must be synchronized across regions: -- **Kubernetes Secrets:** Replicate secrets to all clusters (KubeFleet handles this automatically) +- **Kubernetes Secrets:** Replicate secrets to all Kubernetes clusters + (KubeFleet handles this automatically) - **RBAC policies:** Apply consistent access controls across regions - **Credential rotation:** Coordinate credential changes across all clusters @@ -160,7 +174,8 @@ Credentials must be synchronized across regions: Restrict network access between regions: - **Private connectivity:** Use VNet/VPC peering instead of public internet -- **Network policies:** Kubernetes NetworkPolicy to limit pod-to-pod communication +- **Network policies:** Kubernetes NetworkPolicy to limit pod-to-pod + communication - **Firewall rules:** Allow only required ports between regional clusters ## Monitoring and Observability @@ -172,10 +187,14 @@ Track multi-region health and performance: - **Network metrics:** Bandwidth, latency, packet loss between regions - **Application performance:** Request latency, error rates per region -See [Telemetry Examples](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/telemetry/README.md) for OpenTelemetry, Prometheus, and Grafana setup. +See [Telemetry Examples](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/telemetry/README.md) +for OpenTelemetry, Prometheus, and Grafana setup. ## Next Steps -- [Multi-Region Setup Guide](setup.md) - Deploy your first multi-region DocumentDB cluster -- [Failover Procedures](failover-procedures.md) - Learn how to handle planned and unplanned failovers -- [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - Complete Azure multi-region automation +- [Multi-Region Setup Guide](setup.md) - Deploy your first multi-region + DocumentDB cluster +- [Failover Procedures](failover-procedures.md) - Learn how to handle planned + and unplanned failovers +- [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) + - Complete Azure multi-region automation diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/setup.md b/docs/operator-public-documentation/preview/multi-region-deployment/setup.md index a9fd0042..79253ca4 100644 --- a/docs/operator-public-documentation/preview/multi-region-deployment/setup.md +++ b/docs/operator-public-documentation/preview/multi-region-deployment/setup.md @@ -8,10 +8,6 @@ tags: - replication --- -# Multi-Region Setup Guide - -This guide walks through deploying DocumentDB across multiple Kubernetes clusters with regional replication. - ## Prerequisites ### Infrastructure Requirements @@ -50,7 +46,9 @@ See [Get Started](../index.md#install-cert-manager) for detailed cert-manager se #### 2. DocumentDB Operator -Install the operator on each Kubernetes cluster. If using KubeFleet (recommended), install once on the hub cluster and let it propagate to all member clusters. See [Fleet Setup](#with-fleet-management-recommended) below. +Install the operator on each Kubernetes cluster. If using KubeFleet (recommended), +install once on the hub cluster and let it propagate to all member clusters. See +[KubeFleet Setup](#with-kubefleet-recommended) below. ```bash helm repo add documentdb https://documentdb.github.io/documentdb-kubernetes-operator @@ -66,9 +64,10 @@ Verify on each cluster: kubectl get pods -n documentdb-operator ``` -#### Self Label +#### Create the cluster identity ConfigMap -Each Kubernetes cluster participating in multi-region deployment must identify itself with a unique cluster name. Create a ConfigMap on each cluster: +Each Kubernetes cluster participating in multi-region deployment must identify +itself with a unique cluster name. Create a ConfigMap on each cluster: ```bash # Run on each cluster - replace with actual cluster name @@ -79,7 +78,8 @@ kubectl create configmap cluster-identity \ --from-literal=cluster-name="${CLUSTER_NAME}" ``` -Important: The cluster name in this ConfigMap must exactly match one of the member cluster names in the DocumentDB resource's spec.clusterReplication.clusterList[].name. See Cluster Identification for details. +Important: The cluster name in this ConfigMap must exactly match one of the member +cluster names in the DocumentDB resource's spec.clusterReplication.clusterList[].name. This is needed since the DocumentDB CRD will be the same across primaries and replicas for ease of use, but the clusters need to be able to identify where in the topology they lie. @@ -599,7 +599,7 @@ If PVCs aren't provisioning: ## Next Steps -- [Failover Procedures](failover-procedures.md) - Learn how to perform planned and unplanned fail overs +- [Failover Procedures](failover-procedures.md) - Learn how to perform planned and unplanned failovers - [Backup and Restore](../backup-and-restore.md) - Configure multi-region backup strategies - [TLS Configuration](../configuration/tls.md) - Secure connections with proper TLS certificates - [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - Automated Azure multi-region setup From f47ba0ce2749967a0edb1465bb9d747093a7599f Mon Sep 17 00:00:00 2001 From: Alexander Laye Date: Tue, 17 Mar 2026 11:44:59 -0400 Subject: [PATCH 3/3] AI review and streamline setup.md Signed-off-by: Alexander Laye --- .../failover-procedures.md | 115 +++--- .../multi-region-deployment/overview.md | 114 +++--- .../preview/multi-region-deployment/setup.md | 360 ++++-------------- .../deploy-fleet-bicep.sh | 4 + 4 files changed, 202 insertions(+), 391 deletions(-) diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md b/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md index 331a454e..24dc4d0a 100644 --- a/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md +++ b/docs/operator-public-documentation/preview/multi-region-deployment/failover-procedures.md @@ -1,5 +1,5 @@ --- -title: Multi-Region Failover Procedures +title: Multi-region failover procedures description: Step-by-step runbooks for planned and unplanned DocumentDB failovers across regions, including verification and rollback procedures. tags: - multi-region @@ -10,7 +10,7 @@ tags: ## Overview -A **failover** promotes a replica cluster to become the new primary, making it +A **failover** promotes a replica Kubernetes cluster to become the new primary, making it accept write operations. The previous primary (if still available) becomes a replica replicating from the new primary. @@ -21,16 +21,16 @@ replicating from the new primary. - **Performance optimization:** Move primary closer to write-heavy workload - **Testing:** Validate disaster recovery procedures -## Failover Types +## Failover types -### Planned Failover +### Planned failover This is a failover where the primary is safely demoted, writes are all flushed, and then the new primary is promoted. This kind of failover has no data loss, a set window where writes aren't accepted, and the same number of replicas before and after. -### Unplanned Failover (Disaster Recovery) +### Unplanned failover (disaster recovery) This is a failover where the primary becomes unavailable and has to be forced out of the DocumentDB cluster entirely. Downtime depends on how quickly primary degradation @@ -38,7 +38,7 @@ is detected and, if HA is enabled, how long it takes to scale up the new primary Some writes to the failed primary might be lost, but with high availability enabled, clients can determine when writes were not committed to replicas. After an unplanned failover, the DocumentDB cluster has one fewer region, and you will need to add -the region back when the Kubernetes cluster is back online, or add a replacement +the region back when the failed Kubernetes cluster is back online, or add a replacement region. See the [add region playground guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/fleet-add-region/README.md) for an example. @@ -46,15 +46,15 @@ for an example. Before performing any failover: -- **Replica health:** Target replica cluster is running and replication is current -- **Network access:** You have kubectl access to all clusters involved +- **Replica health:** The target replica Kubernetes cluster is running and replication is current +- **Network access:** You have `kubectl` access to all Kubernetes clusters involved - **Backup available:** Recent backup exists for rollback if needed - **Monitoring:** Metrics and logs are accessible for verification - **Communication:** Stakeholders are notified (for planned failover) - **Application readiness:** Applications can handle brief connection interruption -- **kubectl-documentdb plugin:** Install the plugin for streamlined failover operations [kubectl-plugin](../kubectl-plugin.md) +- **kubectl-documentdb plugin:** Install the plugin for streamlined failover operations in [kubectl-plugin](../kubectl-plugin.md) -### Check Current Replication Status +### Check current replication status Identify the current primary and verify replica health: @@ -82,11 +82,11 @@ Expected output shows active replication to all replicas: - **state:** Should be `streaming` - **LSN values:** `replay_lsn` should be close to `sent_lsn` (low replication lag) -## Planned Failover Procedure +## Planned failover procedure -Use this procedure when the primary cluster is healthy and you want to switch primary regions in a controlled manner. +Use this procedure when the primary Kubernetes cluster is healthy and you want to switch primary regions in a controlled manner. -### Step 1: Pre-Failover Verification +### Step 1: Pre-failover verification Verify system health before starting: @@ -105,12 +105,14 @@ kubectl --context new-primary get pods -n documentdb-preview-ns All checks should show healthy status before proceeding. -### Step 2: Perform Failover +### Step 2: Perform failover -Promote the replica to become the new primary cluster. +Promote the replica to become the new primary Kubernetes cluster. -=== "Using kubectl-documentdb Plugin (KubeFleet deployment required)" +=== "Plugin" + !!! note + KubeFleet deployment required The plugin handles the CRD change and automatically waits for convergence: ```bash @@ -121,9 +123,9 @@ Promote the replica to become the new primary cluster. --hub-context hub ``` -=== "Using kubectl patch with Fleet" +=== "kubectl patch (KubeFleet)" - Update the DocumentDB resource on the hub cluster: + Update the DocumentDB resource on the hub Kubernetes cluster: ```bash kubectl --context hub patch documentdb documentdb-preview \ @@ -132,16 +134,16 @@ Promote the replica to become the new primary cluster. -p '{"spec":{"clusterReplication":{"primary":"new-primary-cluster-name"}}}' ``` - The fleet controller propagates the change to all member clusters automatically. + The fleet controller propagates the change to all member Kubernetes clusters automatically. -=== "Using kubectl patch without KubeFleet" +=== "kubectl patch (manual)" - Update the DocumentDB resource on **all** clusters: + Update the DocumentDB resource on **all** Kubernetes clusters: ```bash - # Update on all clusters (use loop or run individually) + # Update on all Kubernetes clusters (use a loop or run individually) for context in cluster1 cluster2 cluster3; do - kubectl --context $context patch documentdb documentdb-preview \ + kubectl --context "$context" patch documentdb documentdb-preview \ -n documentdb-preview-ns \ --type='merge' \ -p '{"spec":{"clusterReplication":{"primary":"new-primary-cluster-name"}}}' @@ -152,10 +154,10 @@ Promote the replica to become the new primary cluster. 1. The operator detects the primary change 2. The old primary becomes a replica after flushing writes -3. The new primary cluster scales up (if HA) and starts to accept writes +3. The new primary Kubernetes cluster scales up (if HA) and starts to accept writes 4. Replication direction reverses (new primary → replicas including old primary) -### Step 3: Monitor Failover Progress +### Step 3: Monitor failover progress Watch operator logs and DocumentDB status: @@ -171,7 +173,7 @@ kubectl --context new-primary logs -n documentdb-operator \ kubectl --context new-primary get pods -n documentdb-preview-ns -w ``` -### Step 4: Verify New Primary +### Step 4: Verify promoted primary Confirm the new primary accepts writes: @@ -192,7 +194,7 @@ db.testCollection.insertOne({ # Should succeed without errors ``` -### Step 5: Verify Old Primary as Replica +### Step 5: Verify old primary as replica Check that the old primary is now replicating from the new primary: @@ -204,64 +206,67 @@ kubectl --context new-primary exec -it -n documentdb-preview-ns \ You should see the old primary listed as a replica receiving replication stream. -### Step 6: Post-Failover Validation +### Step 6: Post-failover validation Run comprehensive checks: ```bash -# 1. Verify all clusters are in sync +# 1. Verify all Kubernetes clusters are in sync for context in cluster1 cluster2 cluster3; do echo "=== $context ===" - kubectl --context $context get documentdb -n documentdb-preview-ns + kubectl --context "$context" get documentdb -n documentdb-preview-ns done # 2. Check application health kubectl --context new-primary get pods -n app-namespace # 3. Review metrics and logs for errors -# (use your monitoring system: Prometheus, Grafana, CloudWatch, etc.) +# (use your monitoring system, such as Prometheus, Grafana, or CloudWatch) # 4. Verify data consistency (read from all replicas) ``` -## Unplanned Failover Procedure (Disaster Recovery) +## Unplanned failover procedure (disaster recovery) -Use this procedure when the primary cluster is unavailable and you need to immediately promote a replica. +Use this procedure when the primary Kubernetes cluster is unavailable and you need to immediately promote a replica. -!!! danger "Data Loss Risk" - Unplanned failover may result in data loss if the primary cluster failed before replicating recent writes. Assess replication lag before deciding which replica to promote. +!!! danger "Data loss risk" + Unplanned failover may result in data loss if the primary DocumentDB cluster failed before replicating recent writes. Assess replication lag before deciding which replica to promote. -### Step 1: Assess the Situation +### Step 1: Assess the situation Determine the scope of the outage: ```bash -# 1. Check primary cluster accessibility +# 1. Check primary Kubernetes cluster accessibility kubectl --context primary get nodes -# If this fails, primary cluster is unreachable +# If this fails, the primary Kubernetes cluster is unreachable -# 2. Check replica cluster health +# 2. Check replica Kubernetes cluster health kubectl --context replica1 get documentdb -n documentdb-preview-ns kubectl --context replica2 get documentdb -n documentdb-preview-ns # 3. Check cloud provider status pages for regional outages ``` -### Step 2: Select Target Replica +### Step 2: Select target replica Choose which replica to promote based on: - **Replication lag:** Prefer the replica with the lowest lag (most recent data) - **Geographic location:** Consider application proximity -- **Cluster health:** Ensure target cluster is fully operational +- **Kubernetes cluster health:** Ensure the target Kubernetes cluster is fully operational If you cannot query the primary, check the last known replication status from monitoring dashboards or logs. -### Step 3: Promote Replica to Primary +### Step 3: Promote replica to primary Immediately promote the selected replica to become the new primary. -=== "Using kubectl-documentdb Plugin (Recommended)" +=== "Plugin" + + !!! note + KubeFleet deployment required ```bash kubectl documentdb promote \ @@ -273,9 +278,11 @@ Immediately promote the selected replica to become the new primary. --wait-timeout 15m ``` - The plugin handles the change to clusterList and primary and monitors for successful convergence. Use `--skip-wait` if you need to return immediately and verify manually. + The plugin handles the change to `clusterList` and `primary` and monitors for + successful convergence. Use `--skip-wait` if you need to return immediately + and verify manually. -=== "Using kubectl patch with KubeFleet" +=== "kubectl patch (KubeFleet)" ```bash # Remove failed primary from cluster list and set new primary in one command @@ -285,30 +292,30 @@ Immediately promote the selected replica to become the new primary. -p '{"spec":{"clusterReplication":{"primary":"replica-cluster-name","clusterList":[{"name":"replica-cluster-name"},{"name":"other-replica-cluster-name"}]}}}' ``` - Replace the `clusterList` entries with your actual list of healthy clusters, excluding the failed primary. + Replace the `clusterList` entries with your actual list of healthy Kubernetes clusters, excluding the failed primary. -=== "Using kubectl without KubeFleet" +=== "kubectl patch (manual)" ```bash - # Update on all accessible clusters + # Update on all accessible Kubernetes clusters # Remove failed primary from cluster list and set new primary in one command for context in replica1 replica2; do - kubectl --context $context patch documentdb documentdb-preview \ + kubectl --context "$context" patch documentdb documentdb-preview \ -n documentdb-preview-ns \ --type='merge' \ -p '{"spec":{"clusterReplication":{"primary":"replica-cluster-name","clusterList":[{"name":"replica-cluster-name"},{"name":"other-replica-cluster-name"}]}}}' done ``` - Replace the `clusterList` entries with your actual list of healthy clusters, excluding the failed primary. + Replace the `clusterList` entries with your actual list of healthy Kubernetes clusters, excluding the failed primary. **What happens:** 1. The operator detects the primary and cluster list changes -2. The new primary cluster scales up (if HA) and starts to accept writes +2. The new primary Kubernetes cluster scales up (if HA) and starts to accept writes 3. The old primary is removed from replication -### Step 4: Verify New Primary +### Step 4: Verify new primary Confirm the promoted replica is accepting writes: @@ -325,7 +332,7 @@ mongosh "mongodb://admin:password@localhost:10260/?tls=true&tlsAllowInvalidCerti db.testCollection.insertOne({message: "DR failover test"}) ``` -### Step 5: Monitor Recovery +### Step 5: Monitor recovery ```bash # Application pod logs @@ -336,7 +343,7 @@ kubectl --context new-primary logs -n documentdb-operator \ -l app.kubernetes.io/name=documentdb-operator -f ``` -### Step 6: Handle Failed Primary Recovery +### Step 6: Handle failed primary recovery When the failed primary Kubernetes cluster recovers, you need to re-add it to the DocumentDB cluster as a replica. For detailed guidance on adding a region back to your DocumentDB cluster, diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/overview.md b/docs/operator-public-documentation/preview/multi-region-deployment/overview.md index af0a94a1..341b3e7d 100644 --- a/docs/operator-public-documentation/preview/multi-region-deployment/overview.md +++ b/docs/operator-public-documentation/preview/multi-region-deployment/overview.md @@ -1,5 +1,5 @@ --- -title: Multi-Region Deployment Overview +title: Multi-region deployment overview description: Understand multi-region DocumentDB deployments for disaster recovery, low-latency access, and compliance with geographic data residency requirements. @@ -10,20 +10,20 @@ tags: - architecture --- -## Use Cases +## Use cases -### Disaster Recovery (DR) +### Disaster recovery (DR) Protect against regional outages by maintaining database replicas in separate geographic regions. If the primary region fails, promote a replica in another region to maintain service availability. -### Low-Latency Global Access +### Low-latency global access Reduce application response times and distribute load by deploying read replicas closer to end users. -### Compliance and Data Residency +### Compliance and data residency Meet regulatory requirements for data storage location by deploying replicas in specific regions. Ensure that data resides within required geographic @@ -31,29 +31,29 @@ boundaries while maintaining availability. ## Architecture -### Primary-Replica Model +### Primary-replica model DocumentDB uses a primary-replica architecture where: - **Primary cluster:** Accepts both read and write operations - **Replica clusters:** Accept read-only operations and replicate changes from - primary + the primary Kubernetes cluster - **Replication:** PostgreSQL streaming replication propagates changes from - primary to replicas + the primary Kubernetes cluster to replica Kubernetes clusters -### Cluster Components +### DocumentDB cluster components -Each regional DocumentDB cluster includes: +Each regional Kubernetes cluster includes: - **Gateway containers:** Provide MongoDB-compatible API and connection management - **PostgreSQL containers:** Store data and handle replication (managed by CloudNative-PG) - **Persistent storage:** Regional block storage for data persistence - **Service endpoints:** LoadBalancer or ClusterIP for client connections -- **Self-name ConfigMap:** A ConfigMap giving the name of the cluster (must - match clusterList[].name) +- **Self-name ConfigMap:** A ConfigMap that stores the Kubernetes cluster name + (must match `clusterList[].name`) -### Replication Configuration +### Replication configuration Multi-region replication is configured in the `DocumentDB` resource: @@ -74,58 +74,57 @@ spec: The operator handles: -- Creating replica clusters in specified regions -- Establishing streaming replication from primary to replicas +- Creating replica Kubernetes clusters in specified regions +- Establishing streaming replication from the primary to replicas - Monitoring replication lag and health - Coordinating failover operations -## Network Requirements +## Network requirements -### Inter-Region Connectivity +### Inter-region connectivity -Use cloud-native VNet/VPC peering for direct cluster-to-cluster communication: +Use cloud-native VNet/VPC peering for direct Kubernetes cluster-to-cluster communication: - **Azure:** VNet peering between AKS clusters - **AWS:** VPC peering between EKS clusters - **GCP:** VPC peering between GKE clusters -### Port Requirements +### Port requirements DocumentDB replication requires these ports between Kubernetes clusters: -| Port | Protocol | Purpose | -|------|----------|---------| -| 5432 | TCP | PostgreSQL streaming replication | -| 443 | TCP | Kubernetes API (for KubeFleet, optional) | +| Port | Protocol | Purpose | +| | | | +| 5432 | TCP | PostgreSQL streaming replication | +| 443 | TCP | Kubernetes API (for KubeFleet, optional) | Ensure firewall rules and network security groups allow traffic on these ports -between regional clusters. +between regional Kubernetes clusters. -### DNS and Service Discovery +### DNS and service discovery -The operator uses the DocumentDB cluster name and its corresponding CNPG -cluster's generated service to connect the multi-regional clusters, and you -must coordinate that connection between the clusters. Alternatively, there are -two integrations with existing services. +The operator uses the DocumentDB cluster name and the generated service for the +corresponding CNPG cluster to connect regional deployments. You must make sure +those connections can resolve and route correctly between Kubernetes clusters. +You can also use either of the built-in networking integrations. -#### Istio Networking +#### Istio networking If Istio is installed on the Kubernetes cluster, Istio networking is enabled, and an east-west gateway is present connecting each Kubernetes cluster, then the operator generates services that automatically route the default service names across regions. -#### Fleet Networking +#### Fleet networking -If Fleet networking is installed on each of your Kubernetes clusters, then -instead of populating the connections with default service names, the operator -creates ServiceExports and MultiClusterServices on each Kubernetes cluster, -then use those generated cross-regional services to connect the CNPG instances -to one another. +If Fleet networking is installed on each Kubernetes cluster, instead of using +default service names, the operator creates ServiceExports and +MultiClusterServices on each Kubernetes cluster. It then uses those generated +cross-regional services to connect CNPG instances to one another. -## Deployment Models +## Deployment models -### Managed Fleet Orchestration +### Managed fleet orchestration Use a multi-cluster orchestration system such as KubeFleet to manage deployments of resources across Kubernetes clusters and centrally manage @@ -133,68 +132,69 @@ changes, ensuring your topology stays synchronized between regions. **Example:** [AKS Fleet Deployment](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) -### Manual Multi-Cluster Management +### Manual multi-cluster management Deploy DocumentDB resources individually to each Kubernetes cluster, manually ensuring that each DocumentDB CRD is in sync. -## Performance Considerations +## Performance considerations -### Replication Lag +### Replication lag Distance between regions affects replication lag. Monitor replication lag with PostgreSQL metrics and adjust application read patterns accordingly. -### Storage Performance +### Storage performance Each region requires independent storage resources, and each replica must have an equal or greater volume of available storage compared to the primary. -## Security Considerations +## Security considerations -### TLS Encryption +### TLS encryption Enable TLS for all connections: -- **Client-to-gateway:** Encrypt application connections (see [TLS Configuration](../configuration/tls.md)) +- **Client-to-gateway:** Encrypt application connections (see [TLS configuration](../configuration/tls.md)) - **Replication traffic:** PostgreSQL SSL for inter-cluster replication - **Service mesh:** mTLS for cross-cluster service communication -### Authentication and Authorization +### Authentication and authorization Credentials must be synchronized across regions: - **Kubernetes Secrets:** Replicate secrets to all Kubernetes clusters (KubeFleet handles this automatically) - **RBAC policies:** Apply consistent access controls across regions -- **Credential rotation:** Coordinate credential changes across all clusters +- **Credential rotation:** Coordinate credential changes across all Kubernetes + clusters -### Network Security +### Network security Restrict network access between regions: - **Private connectivity:** Use VNet/VPC peering instead of public internet - **Network policies:** Kubernetes NetworkPolicy to limit pod-to-pod communication -- **Firewall rules:** Allow only required ports between regional clusters +- **Firewall rules:** Allow only required ports between regional Kubernetes + clusters -## Monitoring and Observability +## Monitoring and observability Track multi-region health and performance: - **Replication lag:** Monitor `pg_stat_replication` metrics -- **Cluster health:** Pod status, resource usage, connection counts +- **Kubernetes cluster health:** Pod status, resource usage, and connection counts - **Network metrics:** Bandwidth, latency, packet loss between regions - **Application performance:** Request latency, error rates per region -See [Telemetry Examples](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/telemetry/README.md) +See [Telemetry examples](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/telemetry/README.md) for OpenTelemetry, Prometheus, and Grafana setup. -## Next Steps +## Next steps -- [Multi-Region Setup Guide](setup.md) - Deploy your first multi-region +- [Multi-region setup guide](setup.md) - Deploy your first multi-region DocumentDB cluster -- [Failover Procedures](failover-procedures.md) - Learn how to handle planned +- [Failover procedures](failover-procedures.md) - Learn how to handle planned and unplanned failovers -- [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - - Complete Azure multi-region automation +- [AKS Fleet deployment example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) diff --git a/docs/operator-public-documentation/preview/multi-region-deployment/setup.md b/docs/operator-public-documentation/preview/multi-region-deployment/setup.md index 79253ca4..1b7890d7 100644 --- a/docs/operator-public-documentation/preview/multi-region-deployment/setup.md +++ b/docs/operator-public-documentation/preview/multi-region-deployment/setup.md @@ -1,5 +1,5 @@ --- -title: Multi-Region Setup Guide +title: Multi-region setup guide description: Step-by-step instructions for deploying DocumentDB across multiple Kubernetes clusters with replication, prerequisites, and configuration examples. tags: - multi-region @@ -10,22 +10,22 @@ tags: ## Prerequisites -### Infrastructure Requirements +### Infrastructure requirements Before deploying DocumentDB in multi-region mode, ensure you have: -- **Multiple Kubernetes clusters:** 2+ clusters in different regions -- **Network connectivity:** Clusters can communicate over private network or internet -- **Storage:** CSI-compatible storage class in each cluster with snapshot support +- **Multiple Kubernetes clusters:** 2 or more Kubernetes clusters in different regions +- **Network connectivity:** Kubernetes clusters can communicate over private networking or the internet +- **Storage:** CSI-compatible storage class in each Kubernetes cluster with snapshot support - **Load balancing:** LoadBalancer or Ingress capability for external access (optional) -### Required Components +### Required components Install these components on **all** Kubernetes clusters: #### 1. cert-manager -Required for TLS certificate management between clusters. +Required for TLS certificate management between Kubernetes clusters. ```bash helm repo add jetstack https://charts.jetstack.io @@ -44,11 +44,9 @@ kubectl get pods -n cert-manager See [Get Started](../index.md#install-cert-manager) for detailed cert-manager setup. -#### 2. DocumentDB Operator +#### 2. DocumentDB operator -Install the operator on each Kubernetes cluster. If using KubeFleet (recommended), -install once on the hub cluster and let it propagate to all member clusters. See -[KubeFleet Setup](#with-kubefleet-recommended) below. +Install the operator on each Kubernetes cluster. ```bash helm repo add documentdb https://documentdb.github.io/documentdb-kubernetes-operator @@ -58,37 +56,38 @@ helm install documentdb-operator documentdb/documentdb-operator \ --create-namespace ``` -Verify on each cluster: +Verify installation: ```bash kubectl get pods -n documentdb-operator ``` -#### Create the cluster identity ConfigMap +#### 3. Kubernetes cluster identity ConfigMap -Each Kubernetes cluster participating in multi-region deployment must identify -itself with a unique cluster name. Create a ConfigMap on each cluster: +Each Kubernetes cluster in a multi-region deployment must identify itself with +a unique Kubernetes cluster name. Create a ConfigMap on each Kubernetes cluster: ```bash -# Run on each cluster - replace with actual cluster name -CLUSTER_NAME="member-eastus2-cluster" # e.g., member-eastus2-cluster, member-westus3-cluster +# Run on each Kubernetes cluster and replace with your actual cluster name. +CLUSTER_NAME="member-eastus2-cluster" # for example: member-eastus2-cluster, member-westus3-cluster kubectl create configmap cluster-identity \ --namespace kube-system \ --from-literal=cluster-name="${CLUSTER_NAME}" ``` -Important: The cluster name in this ConfigMap must exactly match one of the member -cluster names in the DocumentDB resource's spec.clusterReplication.clusterList[].name. +!!! note + The Kubernetes cluster name in this ConfigMap must exactly match one + of the member Kubernetes cluster names in `spec.clusterReplication.clusterList[].name`. -This is needed since the DocumentDB CRD will be the same across primaries and replicas for ease of use, but the clusters need -to be able to identify where in the topology they lie. +This is required because the DocumentDB CRD is the same across primaries and +replicas, and each Kubernetes cluster must identify its own role in the topology. -### Network Configuration +### Network configuration -#### VNet/VPC Peering (Single Cloud Provider) +#### VNet/VPC peering (single cloud provider) -For clusters within the same cloud provider, configure VNet or VPC peering: +For Kubernetes clusters in the same cloud provider, configure VNet or VPC peering: === "Azure (AKS)" @@ -103,7 +102,7 @@ For clusters within the same cloud provider, configure VNet or VPC peering: --allow-vnet-access ``` - Repeat for all cluster pairs in a full mesh topology. + Repeat for all Kubernetes cluster pairs in a full mesh topology. See [AKS Fleet Deployment](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) for automated Azure multi-region setup with VNet peering. @@ -152,136 +151,52 @@ spec: - name: member-westus3-cluster ``` -## Deployment Options +## Deployment options Choose a deployment approach based on your infrastructure and operational preferences. -### With KubeFleet (Recommended) +### With KubeFleet (recommended) KubeFleet systems simplify multi-region operations by: - **Centralized control:** Define resources once, deploy everywhere -- **Automatic propagation:** Resources sync to member clusters automatically +- **Automatic propagation:** Resources sync to member Kubernetes clusters automatically - **Coordinated updates:** Roll out changes across regions consistently -#### Step 1: Deploy Fleet Infrastructure +#### Step 1: Deploy fleet infrastructure Install KubeFleet or another fleet management system: -```bash -# Example: KubeFleet on hub cluster -kubectl apply -f https://github.com/kubefleet-dev/kubefleet/releases/latest/download/install.yaml -``` - -Configure member clusters to join the fleet. See the [AKS Fleet Deployment guide](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) for a complete automated setup example. - -#### Step 2: Install DocumentDB Operator via KubeFleet - -Create a `ClusterResourcePlacement` to deploy the operator to all member clusters: - -```yaml title="documentdb-operator-crp.yaml" -apiVersion: placement.kubernetes-fleet.io/v1beta1 -kind: ClusterResourcePlacement -metadata: - name: documentdb-operator-crp - namespace: fleet-system -spec: - resourceSelectors: - - group: "" - kind: Namespace - name: documentdb-operator - version: v1 - placement: - strategy: - type: PickAll # Deploy to all member clusters -``` - -Apply to hub cluster: - -```bash -kubectl --context hub apply -f documentdb-operator-crp.yaml -``` - -The fleet controller will install the operator on all member clusters automatically. - -#### Step 3: Deploy Multi-Region DocumentDB - -Create a DocumentDB resource with replication configuration: - -```yaml title="multi-region-documentdb.yaml" -apiVersion: documentdb.io/preview -kind: DocumentDB -metadata: - name: documentdb-preview - namespace: documentdb-preview-ns -spec: - nodeCount: 2 - instancesPerNode: 1 - resource: - storage: - pvcSize: 100Gi - storageClass: managed-csi # Use appropriate storage class per cloud - exposeViaService: - serviceType: LoadBalancer - tls: - gateway: - mode: SelfSigned - clusterReplication: # (1)! - primary: member-eastus2-cluster # (2)! - clusterList: # (3)! - - name: member-westus3-cluster - - name: member-uksouth-cluster - - name: member-eastus2-cluster - documentDbCredentialSecret: documentdb-secret -``` +Configure member Kubernetes clusters to join the fleet. See +[deploy-fleet-bicep.sh](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh) +"KUBEFLEET SETUP" for a complete automated setup example. -1. The `clusterReplication` section enables multi-region deployment -2. `primary` specifies which cluster accepts write operations -3. `clusterList` lists all member clusters that will host DocumentDB instances (including the primary) +#### Step 2: Install cert-manager and DocumentDB operator -Create the credentials secret: +Install the cert manager and DocumentDB operator to the hub per the +[Required Components](#required-components) section, then create `ClusterResourcePlacements` +to deploy them both to all member Kubernetes clusters. -```bash -kubectl create secret generic documentdb-secret \ - --namespace documentdb-preview-ns \ - --from-literal=password='YourSecurePassword123!' -``` +- [cert-manager CRP](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/cert-manager-crp.yaml) +- [documentdb-operator CRP](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/documentdb-operator-crp.yaml) -Apply via KubeFleet to propagate to all clusters: +#### Step 3: Deploy multi-region DocumentDB -```yaml title="documentdb-crp.yaml" -apiVersion: placement.kubernetes-fleet.io/v1beta1 -kind: ClusterResourcePlacement -metadata: - name: documentdb-crp - namespace: fleet-system -spec: - resourceSelectors: - - group: documentdb.io - kind: DocumentDB - name: documentdb-preview - version: preview - - group: "" - version: v1 - kind: Secret - name: documentdb-secret - placement: - strategy: - type: PickAll -``` +Create a DocumentDB resource with replication configuration. The example uses substitutions +with a script, so you will need to replace all the {{PLACEHOLDERS}}. -Apply both resources: +- [DocumentDB CRD, secret, and CRP](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/documentdb-resource-crp.yaml) -```bash -kubectl --context hub apply -f multi-region-documentdb.yaml -kubectl --context hub apply -f documentdb-crp.yaml -``` +Within the CRD The `clusterReplication` section enables multi-region deployment, +`primary` specifies which Kubernetes cluster accepts write operations, and `clusterList` +lists all member Kubernetes clusters that host DocumentDB instances (including the +primary) and accepts a more granular `environment` and `storageClass` variable. ### Without KubeFleet -If not using KubeFleet, deploy DocumentDB resources to each cluster individually. +If you are not using KubeFleet, deploy DocumentDB resources to each Kubernetes cluster individually. -#### Step 1: Identify Cluster Names +#### Step 1: Identify Kubernetes cluster names Determine the name for each Kubernetes cluster. These names are used in the replication configuration: @@ -295,115 +210,31 @@ aws eks list-clusters --query "clusters" --output table # AWS gcloud container clusters list --format="table(name)" # GCP ``` -#### Step 2: Create Cluster Identification +#### Step 2: Create Kubernetes cluster identification -On each cluster, create a ConfigMap to identify the cluster name: +On each Kubernetes cluster, create a ConfigMap to identify the Kubernetes cluster name: ```bash -# Run on each cluster -CLUSTER_NAME="cluster-region-name" # e.g., member-eastus2-cluster +# Run on each Kubernetes cluster +CLUSTER_NAME="cluster-region-name" # for example: member-eastus2-cluster kubectl create configmap cluster-identity \ --namespace kube-system \ --from-literal=cluster-name="${CLUSTER_NAME}" ``` -#### Step 3: Deploy DocumentDB to Primary Cluster - -On the primary cluster: - -```yaml title="primary-documentdb.yaml" -apiVersion: documentdb.io/preview -kind: DocumentDB -metadata: - name: documentdb-preview - namespace: documentdb-preview-ns -spec: - nodeCount: 2 - instancesPerNode: 1 - resource: - storage: - pvcSize: 100Gi - exposeViaService: - serviceType: LoadBalancer - tls: - gateway: - mode: SelfSigned - clusterReplication: - primary: cluster-primary-name # This cluster's name - clusterList: - - name: cluster-primary-name - - name: cluster-replica1-name - - name: cluster-replica2-name - documentDbCredentialSecret: documentdb-secret -``` - -Apply to primary: +#### Step 3: Deploy cert-manager and DocumentDB operator to each cluster -```bash -kubectl --context primary apply -f primary-documentdb.yaml -``` +Install the cert manager and DocumentDB operator to the hub per the +[Required Components](#required-components) section on each Kubernetes cluster. +When making changes to any resource, you must make that same change across +each Kubernetes cluster so they are all in sync, as the operator works under +the assumption that all members have the same resources. -#### Step 4: Deploy DocumentDB to Replica Clusters +### Storage configuration -Use the **same YAML** on each replica cluster. The operator detects whether the cluster is primary or replica based on the `clusterReplication.primary` field. - -```bash -kubectl --context replica1 apply -f primary-documentdb.yaml -kubectl --context replica2 apply -f primary-documentdb.yaml -``` - -Each cluster will: - -- **Primary:** Run a read-write DocumentDB cluster -- **Replicas:** Run read-only DocumentDB clusters replicating from primary - -## Configuration Details - -### Replication Configuration - -The `clusterReplication` section controls multi-region behavior: - -```yaml -spec: - clusterReplication: - primary: member-eastus2-cluster # Write cluster - clusterList: # All participating member clusters - - name: member-westus3-cluster - - name: member-uksouth-cluster - - name: member-eastus2-cluster -``` - -**Key points:** - -- **Primary cluster:** The cluster name specified in `primary` accepts read and write operations -- **Replica clusters:** All other clusters in `clusterList` are read-only replicas -- **Include primary in list:** The primary cluster name must appear in the `clusterList` -- **Cluster name uniqueness:** Each cluster must have a unique name across all regions - -### Credential Synchronization - -The DocumentDB operator expects the same credentials secret on all clusters. With KubeFleet, create the secret on the hub (or in fleet-managed namespace) and it propagates automatically. Without fleet management, create the secret manually on each cluster: - -```bash -# Same secret on all clusters -kubectl --context cluster1 create secret generic documentdb-secret \ - --namespace documentdb-preview-ns \ - --from-literal=password='YourSecurePassword123!' - -kubectl --context cluster2 create secret generic documentdb-secret \ - --namespace documentdb-preview-ns \ - --from-literal=password='YourSecurePassword123!' - -# And so on... -``` - -!!! warning "Credential Management" - Synchronize password changes across all clusters manually if not using fleet management. Mismatched credentials will break replication. - -### Storage Configuration - -Each cluster in a multi-region deployment can use different storage classes. Configure storage at the global level or override per member cluster: +Each Kubernetes cluster in a multi-region deployment can use different storage classes. +Configure storage at the global level or override per member Kubernetes cluster: **Global storage configuration:** @@ -412,10 +243,10 @@ spec: resource: storage: pvcSize: 100Gi - storageClass: default-storage-class # Used by all clusters + storageClass: default-storage-class # Used by all Kubernetes clusters ``` -**Per-cluster storage override:** +**Per-Kubernetes-cluster storage override:** ```yaml spec: @@ -427,9 +258,9 @@ spec: primary: member-eastus2-cluster clusterList: - name: member-westus3-cluster - storageClass: managed-csi-premium # Override for this cluster + storageClass: managed-csi-premium # Override for this Kubernetes cluster - name: member-uksouth-cluster - storageClass: azuredisk-standard-ssd # Override for this cluster + storageClass: azuredisk-standard-ssd # Override for this Kubernetes cluster - name: member-eastus2-cluster # Uses global storageClass ``` @@ -460,7 +291,7 @@ spec: environment: gke ``` -### Service Exposure +### Service exposure Configure how DocumentDB is exposed in each region: @@ -474,8 +305,9 @@ Configure how DocumentDB is exposed in each region: serviceType: LoadBalancer ``` - Each cluster gets a public IP for client connections. When using the `environment` configuration at either - the DocumentDB cluster or Kubernetes cluster level, the tags for the LoadBalancer will change. See the + Each Kubernetes cluster gets a public IP for client connections. When you use the `environment` + configuration at either the DocumentDB cluster or Kubernetes cluster level, the tags for the + LoadBalancer change. See the cloud-specific setup docs for more details. === "ClusterIP" @@ -490,48 +322,16 @@ Configure how DocumentDB is exposed in each region: Clients must access DocumentDB through Ingress or service mesh. -## Verification - -### Check Operator Status - -Verify the operator is running on all clusters: - -```bash -# With KubeFleet -kubectl --context hub get clusterresourceplacement - -# Without fleet management (run on each cluster) -kubectl --context cluster1 get pods -n documentdb-operator -kubectl --context cluster2 get pods -n documentdb-operator -``` - -### Check DocumentDB Status - -View DocumentDB status on each cluster: - -```bash -kubectl --context primary get documentdb -n documentdb-preview-ns -kubectl --context replica1 get documentdb -n documentdb-preview-ns -kubectl --context replica2 get documentdb -n documentdb-preview-ns -``` - -Expected output shows `Ready` status and role (primary or replica): - -``` -NAME STATUS ROLE AGE -documentdb-preview Ready primary 10m -``` - ## Troubleshooting -### Replication Not Established +### Replication not established If replicas don't receive data from the primary: 1. **Verify network connectivity:** ```bash - # From replica cluster, test connectivity to primary + # From a replica Kubernetes cluster, test connectivity to primary kubectl --context replica1 run test-pod --rm -it --image=nicolaka/netshoot -- \ nc -zv primary-service-endpoint 5432 ``` @@ -550,9 +350,9 @@ If replicas don't receive data from the primary: -l app.kubernetes.io/name=documentdb-operator --tail=100 ``` -### Cluster Name Mismatch +### Kubernetes cluster name mismatch -If a cluster doesn't recognize itself as primary or replica: +If a Kubernetes cluster doesn't recognize itself as primary or replica: 1. **Check cluster-identity ConfigMap:** @@ -561,9 +361,9 @@ If a cluster doesn't recognize itself as primary or replica: -n kube-system -o jsonpath='{.data.cluster-name}' ``` -2. **Verify name matches DocumentDB spec:** +2. **Verify the name matches the DocumentDB spec:** - The returned name must exactly match one of the cluster names in `spec.clusterReplication.clusterList[*].name`. + The returned name must exactly match one of the Kubernetes cluster names in `spec.clusterReplication.clusterList[*].name`. 3. **Update ConfigMap if incorrect:** @@ -574,7 +374,7 @@ If a cluster doesn't recognize itself as primary or replica: --dry-run=client -o yaml | kubectl apply -f - ``` -### Storage Issues +### Storage issues If PVCs aren't provisioning: @@ -597,9 +397,9 @@ If PVCs aren't provisioning: --field-selector involvedObject.kind=PersistentVolumeClaim ``` -## Next Steps +## Next steps -- [Failover Procedures](failover-procedures.md) - Learn how to perform planned and unplanned failovers -- [Backup and Restore](../backup-and-restore.md) - Configure multi-region backup strategies -- [TLS Configuration](../configuration/tls.md) - Secure connections with proper TLS certificates -- [AKS Fleet Deployment Example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - Automated Azure multi-region setup +- [Failover procedures](failover-procedures.md) - Learn how to perform planned and unplanned failovers +- [Backup and restore](../backup-and-restore.md) - Configure multi-region backup strategies +- [TLS configuration](../configuration/tls.md) - Secure connections with proper TLS certificates +- [AKS Fleet deployment example](https://github.com/documentdb/documentdb-kubernetes-operator/blob/main/documentdb-playground/aks-fleet-deployment/README.md) - Automated Azure multi-region setup diff --git a/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh b/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh index eab2d86f..dae119e0 100755 --- a/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh +++ b/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh @@ -109,6 +109,8 @@ while read -r cluster; do if [[ "$cluster" == *"$HUB_REGION"* ]]; then HUB_CLUSTER="$cluster"; fi done <<< "$MEMBER_CLUSTER_NAMES" +######### KUBEFLEET SETUP ######### + kubeDir=$(mktemp -d) git clone https://github.com/kubefleet-dev/kubefleet.git $kubeDir pushd $kubeDir @@ -200,6 +202,8 @@ done <<< "$MEMBER_CLUSTER_NAMES" popd +####### END KUBEFLEET SETUP ######## + # Create kubectl aliases and export FLEET_ID (k-hub and k-) persisted in ~/.bashrc ALIASES_BLOCK_START="# BEGIN aks aliases" ALIASES_BLOCK_END="# END aks aliases"