chore: bundle govcloud publish script

.gitlab/datasources/environments.yaml

@@ -1,9 +1,9 @@
 environments:
-  - name: sandbox
+  sandbox:
     external_id: sandbox-publish-externalid
     role_to_assume: sandbox-layer-deployer
     account: 425362996713
-  - name: prod
+  prod:
     external_id: prod-publish-externalid
     role_to_assume: dd-serverless-layer-deployer-role
     account: 464622532012

.gitlab/scripts/publish_layer.sh

@@ -29,10 +29,9 @@ LAYERS=(
     "Datadog-Ruby3-3"
     "Datadog-Ruby3-3-ARM"
 )
-STAGES=('prod', 'sandbox', 'staging')
+STAGES=('prod', 'sandbox', 'staging', 'gov-staging', 'gov-prod')
 
 printf "Starting script...\n\n"
-printf "Installing dependencies\n"
 
 publish_layer() {
     region=$1
@@ -108,8 +107,14 @@ if [[ ! ${STAGES[@]} =~ $STAGE ]]; then
 fi
 
 layer="${LAYERS[$index]}"
+if [ -z "$LAYER_NAME_SUFFIX" ]; then
+    echo "No layer name suffix"
+else
+    layer="${layer}-${LAYER_NAME_SUFFIX}"
+fi
+echo "layer name: $layer"
 
-if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then
+if [[ "$STAGE" =~ ^(staging|sandbox|gov-staging)$ ]]; then
     # Deploy latest version
     latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $layer --query 'LayerVersions[0].Version || `0`')
     VERSION=$(($latest_version + 1))

.gitlab/template.yaml.tpl

@@ -74,9 +74,6 @@ integration test ({{ $runtime.ruby_version }}, {{ $runtime.arch }}):
   script:
     - RUNTIME_PARAM={{ $runtime.ruby_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
 
-{{ range $environment := (ds "environments").environments }}
-
-{{ if or (eq $environment.name "prod") }}
 sign layer ({{ $runtime.ruby_version }}, {{ $runtime.arch }}):
   stage: sign
   tags: ["arch:amd64"]
@@ -97,22 +94,25 @@ sign layer ({{ $runtime.ruby_version }}, {{ $runtime.arch }}):
     paths:
       - .layers/datadog-lambda_ruby-{{ $runtime.arch }}-{{ $runtime.ruby_version }}.zip
   before_script:
+    {{ with $environment := (ds "environments").environments.prod }}
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
+    {{ end }}
   script:
-    - LAYER_FILE=datadog-lambda_ruby-{{ $runtime.arch}}-{{ $runtime.ruby_version }}.zip ./scripts/sign_layers.sh {{ $environment.name }}
-{{ end }}
+    - LAYER_FILE=datadog-lambda_ruby-{{ $runtime.arch}}-{{ $runtime.ruby_version }}.zip ./scripts/sign_layers.sh prod
+
+{{ range $environment_name, $environment := (ds "environments").environments }}
 
-publish layer {{ $environment.name }} ({{ $runtime.ruby_version }}, {{ $runtime.arch }}):
+publish layer {{ $environment_name }} ({{ $runtime.ruby_version }}, {{ $runtime.arch }}):
   stage: publish
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
-    - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/'
+    - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
   needs:
-{{ if or (eq $environment.name "prod") }}
+{{ if eq $environment_name "prod" }}
       - sign layer ({{ $runtime.ruby_version }}, {{ $runtime.arch }})
 {{ else }}
       - build layer ({{ $runtime.ruby_version }}, {{ $runtime.arch }})
@@ -122,7 +122,7 @@ publish layer {{ $environment.name }} ({{ $runtime.ruby_version }}, {{ $runtime.
       - integration test ({{ $runtime.ruby_version }}, {{ $runtime.arch }})
 {{ end }}
   dependencies:
-{{ if or (eq $environment.name "prod") }}
+{{ if eq $environment_name "prod" }}
       - sign layer ({{ $runtime.ruby_version }}, {{ $runtime.arch }})
 {{ else }}
       - build layer ({{ $runtime.ruby_version }}, {{ $runtime.arch }})
@@ -135,7 +135,7 @@ publish layer {{ $environment.name }} ({{ $runtime.ruby_version }}, {{ $runtime.
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   script:
-    - STAGE={{ $environment.name }} RUBY_VERSION={{ $runtime.ruby_version }} ARCH={{ $runtime.arch }} .gitlab/scripts/publish_layer.sh
+    - STAGE={{ $environment_name }} RUBY_VERSION={{ $runtime.ruby_version }} ARCH={{ $runtime.arch }} .gitlab/scripts/publish_layer.sh
 
 {{- end }}
 
@@ -154,3 +154,49 @@ publish rubygems:
   {{- end }}
   script:
     - .gitlab/scripts/publish_rubygems.sh
+
+layer bundle:
+  stage: build
+  tags: ["arch:amd64"]
+  image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
+  needs:
+    {{ range (ds "runtimes").runtimes }}
+    - build layer ({{ .ruby_version }}, {{ .arch }})
+    {{ end }}
+  dependencies:
+    {{ range (ds "runtimes").runtimes }}
+    - build layer ({{ .ruby_version }}, {{ .arch }})
+    {{ end }}
+  artifacts:
+    expire_in: 1 hr
+    paths:
+      - datadog-lambda_ruby-bundle-${CI_JOB_ID}/
+    name: datadog-lambda_ruby-bundle-${CI_JOB_ID}
+  script:
+    - rm -rf datadog-lambda_ruby-bundle-${CI_JOB_ID}
+    - mkdir -p datadog-lambda_ruby-bundle-${CI_JOB_ID}
+    - cp .layers/datadog-lambda_ruby-*.zip datadog-lambda_ruby-bundle-${CI_JOB_ID}
+
+signed layer bundle:
+  stage: sign
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  tags: ["arch:amd64"]
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+  needs:
+    {{ range (ds "runtimes").runtimes }}
+    - build layer ({{ .ruby_version }}, {{ .arch }})
+    {{ end }}
+  dependencies:
+    {{ range (ds "runtimes").runtimes }}
+    - build layer ({{ .ruby_version }}, {{ .arch }})
+    {{ end }}
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - datadog-lambda_ruby-signed-bundle-${CI_JOB_ID}/
+    name: datadog-lambda_ruby-signed-bundle-${CI_JOB_ID}
+  script:
+    - rm -rf datadog-lambda_ruby-signed-bundle-${CI_JOB_ID}
+    - mkdir -p datadog-lambda_ruby-signed-bundle-${CI_JOB_ID}
+    - cp .layers/datadog-lambda_ruby-*.zip datadog-lambda_ruby-signed-bundle-${CI_JOB_ID}

scripts/publish_govcloud.sh

@@ -0,0 +1,105 @@
+#! /usr/bin/env bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2025 Datadog, Inc.
+#
+# USAGE: download the layer bundle from the build pipeline in gitlab. Use the
+# Download button on the `layer bundle` job. This will be a zip file containing
+# all of the required layers. Run this script as follows:
+#
+# ENVIRONMENT=[us1-staging-fed or us1-fed] [LAYER_NAME_SUFFIX=optional-layer-suffix] [REGIONS=us-gov-west-1] ./scripts/publish_govcloud.sh <layer-bundle.zip>
+#
+# protip: you can drag the zip file from finder into your terminal to insert
+# its path.
+
+set -e
+
+LAYER_PACKAGE=$1
+
+if [ -z "$LAYER_PACKAGE" ]; then
+    printf "[ERROR]: layer package not provided\n"
+    exit 1
+fi
+
+PACKAGE_NAME=$(basename "$LAYER_PACKAGE" .zip)
+
+if [ -z "$ENVIRONMENT" ]; then
+    printf "[ERROR]: ENVIRONMENT not specified\n"
+    exit 1
+fi
+
+if [ "$ENVIRONMENT" = "us1-staging-fed" ]; then
+    AWS_VAULT_ROLE=sso-govcloud-us1-staging-fed-power-user
+
+    export STAGE=gov-staging
+
+    if [[ ! "$PACKAGE_NAME" =~ ^datadog-lambda_ruby-(signed-)?bundle-[0-9]+$ ]]; then
+        echo "[ERROR]: Unexpected package name: $PACKAGE_NAME"
+        exit 1
+    fi
+
+elif [ $ENVIRONMENT = "us1-fed" ]; then
+    AWS_VAULT_ROLE=sso-govcloud-us1-fed-engineering
+
+    export STAGE=gov-prod
+
+    if [[ ! "$PACKAGE_NAME" =~ ^datadog-lambda_ruby-signed-bundle-[0-9]+$ ]]; then
+        echo "[ERROR]: Unexpected package name: $PACKAGE_NAME"
+        exit 1
+    fi
+
+else
+    printf "[ERROR]: ENVIRONMENT not supported, must be us1-staging-fed or us1-fed.\n"
+    exit 1
+fi
+
+TEMP_DIR=$(mktemp -d)
+unzip $LAYER_PACKAGE -d $TEMP_DIR
+mkdir -p .layers
+cp -v $TEMP_DIR/$PACKAGE_NAME/*.zip .layers/
+
+
+AWS_VAULT_PREFIX="aws-vault exec $AWS_VAULT_ROLE --"
+
+echo "Checking that you have access to the GovCloud AWS account"
+$AWS_VAULT_PREFIX aws sts get-caller-identity
+
+
+AVAILABLE_REGIONS=$($AWS_VAULT_PREFIX aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
+
+# Determine the target regions
+if [ -z "$REGIONS" ]; then
+    echo "Region not specified, running for all available regions."
+    REGIONS=$AVAILABLE_REGIONS
+else
+    echo "Region specified: $REGIONS"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then
+        echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS"
+        echo ""
+        echo "EXITING SCRIPT."
+        exit 1
+    fi
+fi
+
+for region in $REGIONS
+do
+    echo "Starting publishing layers for region $region..."
+
+    export REGION=$region
+
+    for ruby_version in "3.2" "3.3"; do
+        for arch in "amd64" "arm64"; do
+            export RUBY_VERSION=$ruby_version
+            export ARCH=$arch
+
+            echo "Publishing layer for $RUBY_VERSION and $ARCH"
+
+            $AWS_VAULT_PREFIX .gitlab/scripts/publish_layer.sh
+        done
+    done
+done
+
+echo "Done !"
+

scripts/publish_prod.sh

@@ -43,11 +43,7 @@ echo "Ensure you have access to gem account"
 ruby scripts/check_credentials.rb
 gem signin
 
-echo "Ensure you have access to the AWS GovCloud account"
-ddsaml2aws login -a govcloud-us1-fed-human-engineering
-aws-vault exec sso-govcloud-us1-fed-engineering -- aws sts get-caller-identity
-
-echo "Ensure you have access to the commercial AWS GovCloud account"
+echo "Ensure you have access to the commercial AWS account"
 aws-vault exec sso-prod-engineering -- aws sts get-caller-identity
 
 CURRENT_VERSION=$(gem build datadog-lambda | grep Version | sed -n -e 's/^.*Version: //p')
@@ -81,10 +77,6 @@ echo
 echo "Publishing layers to commercial AWS regions"
 VERSION=$LAYER_VERSION aws-vault exec sso-prod-engineering --no-session -- ./scripts/publish_layers.sh
 
-echo "Publishing layers to GovCloud AWS regions"
-ddsaml2aws login -a govcloud-us1-fed-human-engineering
-VERSION=$LAYER_VERSION aws-vault exec sso-govcloud-us1-fed-engineering -- ./scripts/publish_layers.sh
-
 read -p "Ready to publish gem $NEW_VERSION (y/n)?" CONT
 if [ "$CONT" != "y" ]; then
     echo "Exiting"