Skip to content

Commit 53306e3

Browse files
apiarian-datadogapiarian-datadog
authored
chore: new govcloud release script (#567)
also refactoring the environments template datasource and adding layer bundling jobs.
1 parent aec4910 commit 53306e3

File tree

5 files changed

+255
-100
lines changed

5 files changed

+255
-100
lines changed

‎ci/datasources/environments.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
environments:
2-
- name: sandbox
2+
sandbox:
33
external_id: sandbox-publish-externalid
44
role_to_assume: sandbox-layer-deployer
55
account: 425362996713
6-
- name: prod
6+
prod:
77
external_id: prod-publish-externalid
88
role_to_assume: dd-serverless-layer-deployer-role
99
account: 464622532012

‎ci/input_files/build.yaml.tpl‎

Lines changed: 56 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,6 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
103103
script:
104104
- RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
105105

106-
{{ range $environment := (ds "environments").environments }}
107-
108-
{{ if or (eq $environment.name "prod") }}
109106
sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
110107
stage: sign
111108
tags: ["arch:amd64"]
@@ -128,22 +125,25 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
128125
before_script:
129126
- apt-get update
130127
- apt-get install -y uuid-runtime
128+
{{ with $environment := (ds "environments").environments.prod }}
131129
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
130+
{{ end }}
132131
script:
133-
- LAYER_FILE=datadog_lambda_py-{{ $runtime.arch}}-{{ $runtime.python_version }}.zip ./scripts/sign_layers.sh {{ $environment.name }}
134-
{{ end }}
132+
- LAYER_FILE=datadog_lambda_py-{{ $runtime.arch}}-{{ $runtime.python_version }}.zip ./scripts/sign_layers.sh prod
133+
134+
{{ range $environment_name, $environment := (ds "environments").environments }}
135135

136-
publish-layer-{{ $environment.name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
136+
publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
137137
stage: publish
138138
tags: ["arch:amd64"]
139139
image: registry.ddbuild.io/images/docker:20.10-py3
140140
rules:
141-
- if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/'
141+
- if: '"{{ $environment_name }}" == "sandbox"'
142142
when: manual
143143
allow_failure: true
144144
- if: '$CI_COMMIT_TAG =~ /^v.*/'
145145
needs:
146-
{{ if or (eq $environment.name "prod") }}
146+
{{ if or (eq $environment_name "prod") }}
147147
- sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
148148
{{ else }}
149149
- build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
@@ -153,7 +153,7 @@ publish-layer-{{ $environment.name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
153153
- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
154154
{{ end }}
155155
dependencies:
156-
{{ if or (eq $environment.name "prod") }}
156+
{{ if or (eq $environment_name "prod") }}
157157
- sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
158158
{{ else }}
159159
- build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
@@ -166,7 +166,7 @@ publish-layer-{{ $environment.name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
166166
before_script:
167167
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
168168
script:
169-
- STAGE={{ $environment.name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh
169+
- STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh
170170

171171
{{- end }}
172172

@@ -186,3 +186,49 @@ publish-pypi-package:
186186
{{- end }}
187187
script:
188188
- ./ci/publish_pypi.sh
189+
190+
layer bundle:
191+
stage: build
192+
tags: ["arch:amd64"]
193+
image: registry.ddbuild.io/images/docker:20.10
194+
needs:
195+
{{ range (ds "runtimes").runtimes }}
196+
- build-layer ({{ .name }}-{{ .arch }})
197+
{{ end }}
198+
dependencies:
199+
{{ range (ds "runtimes").runtimes }}
200+
- build-layer ({{ .name }}-{{ .arch }})
201+
{{ end }}
202+
artifacts:
203+
expire_in: 1 hr
204+
paths:
205+
- datadog_lambda_py-bundle-${CI_JOB_ID}/
206+
name: datadog_lambda_py-bundle-${CI_JOB_ID}
207+
script:
208+
- rm -rf datadog_lambda_py-bundle-${CI_JOB_ID}
209+
- mkdir -p datadog_lambda_py-bundle-${CI_JOB_ID}
210+
- cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-bundle-${CI_JOB_ID}
211+
212+
signed layer bundle:
213+
stage: sign
214+
image: registry.ddbuild.io/images/docker:20.10-py3
215+
tags: ["arch:amd64"]
216+
rules:
217+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
218+
needs:
219+
{{ range (ds "runtimes").runtimes }}
220+
- sign-layer ({{ .name }}-{{ .arch }})
221+
{{ end }}
222+
dependencies:
223+
{{ range (ds "runtimes").runtimes }}
224+
- sign-layer ({{ .name }}-{{ .arch }})
225+
{{ end }}
226+
artifacts:
227+
expire_in: 1 day
228+
paths:
229+
- datadog_lambda_py-signed-bundle-${CI_JOB_ID}/
230+
name: datadog_lambda_py-signed-bundle-${CI_JOB_ID}
231+
script:
232+
- rm -rf datadog_lambda_py-signed-bundle-${CI_JOB_ID}
233+
- mkdir -p datadog_lambda_py-signed-bundle-${CI_JOB_ID}
234+
- cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-signed-bundle-${CI_JOB_ID}

‎ci/publish_layers.sh‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,20 @@ AWS_CLI_PYTHON_VERSIONS=(
2424
"python3.13"
2525
"python3.13"
2626
)
27-
PYTHON_VERSIONS=("3.8-amd64" "3.8-arm64" "3.9-amd64" "3.9-arm64" "3.10-amd64" "3.10-arm64" "3.11-amd64" "3.11-arm64" "3.12-amd64" "3.12-arm64" "3.13-amd64" "3.13-arm64")
27+
PYTHON_VERSIONS=(
28+
"3.8-amd64"
29+
"3.8-arm64"
30+
"3.9-amd64"
31+
"3.9-arm64"
32+
"3.10-amd64"
33+
"3.10-arm64"
34+
"3.11-amd64"
35+
"3.11-arm64"
36+
"3.12-amd64"
37+
"3.12-arm64"
38+
"3.13-amd64"
39+
"3.13-arm64"
40+
)
2841
LAYER_PATHS=(
2942
".layers/datadog_lambda_py-amd64-3.8.zip"
3043
".layers/datadog_lambda_py-arm64-3.8.zip"
@@ -53,11 +66,16 @@ LAYERS=(
5366
"Datadog-Python313"
5467
"Datadog-Python313-ARM"
5568
)
56-
STAGES=('prod', 'sandbox', 'staging')
69+
STAGES=('prod', 'sandbox', 'staging', 'gov-staging', 'gov-prod')
5770

5871
printf "Starting script...\n\n"
59-
printf "Installing dependencies\n"
60-
pip install awscli
72+
73+
if [ -z "$SKIP_PIP_INSTALL" ]; then
74+
echo "Installing dependencies"
75+
pip install awscli
76+
else
77+
echo "Skipping pip install"
78+
fi
6179

6280
publish_layer() {
6381
region=$1
@@ -89,7 +107,7 @@ fi
89107

90108
printf "Python version specified: $PYTHON_VERSION\n"
91109
if [[ ! ${PYTHON_VERSIONS[@]} =~ $PYTHON_VERSION ]]; then
92-
printf "[Error] Unsupported PYTHON_VERSION found.\n"
110+
printf "[Error] Unsupported PYTHON_VERSION found: $PYTHON_VERSION.\n"
93111
exit 1
94112
fi
95113

@@ -133,8 +151,14 @@ if [[ ! ${STAGES[@]} =~ $STAGE ]]; then
133151
fi
134152

135153
layer="${LAYERS[$index]}"
154+
if [ -z "$LAYER_NAME_SUFFIX" ]; then
155+
echo "No layer name suffix"
156+
else
157+
layer="${layer}-${LAYER_NAME_SUFFIX}"
158+
fi
159+
echo "layer name: $layer"
136160

137-
if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then
161+
if [[ "$STAGE" =~ ^(staging|sandbox|gov-staging)$ ]]; then
138162
# Deploy latest version
139163
latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $layer --query 'LayerVersions[0].Version || `0`')
140164
VERSION=$(($latest_version + 1))

‎scripts/publish_govcloud.sh‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#! /usr/bin/env bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2025 Datadog, Inc.
7+
#
8+
# USAGE: download the layer bundle from the build pipeline in gitlab. Use the
9+
# Download button on the `layer bundle` job. This will be a zip file containing
10+
# all of the required layers. Run this script as follows:
11+
#
12+
# ENVIRONMENT=[us1-staging-fed or us1-fed] [LAYER_NAME_SUFFIX=optional-layer-suffix] [REGIONS=us-gov-west-1] ./scripts/publish_govcloud.sh <layer-bundle.zip>
13+
#
14+
# protip: you can drag the zip file from finder into your terminal to insert
15+
# its path.
16+
17+
set -e
18+
19+
LAYER_PACKAGE=$1
20+
21+
if [ -z "$LAYER_PACKAGE" ]; then
22+
printf "[ERROR]: layer package not provided\n"
23+
exit 1
24+
fi
25+
26+
PACKAGE_NAME=$(basename "$LAYER_PACKAGE" .zip)
27+
28+
if [ -z "$ENVIRONMENT" ]; then
29+
printf "[ERROR]: ENVIRONMENT not specified\n"
30+
exit 1
31+
fi
32+
33+
if [ "$ENVIRONMENT" = "us1-staging-fed" ]; then
34+
AWS_VAULT_ROLE=sso-govcloud-us1-staging-fed-power-user
35+
36+
export STAGE=gov-staging
37+
38+
if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_py-(signed-)?bundle-[0-9]+$ ]]; then
39+
echo "[ERROR]: Unexpected package name: $PACKAGE_NAME"
40+
exit 1
41+
fi
42+
43+
elif [ $ENVIRONMENT = "us1-fed" ]; then
44+
AWS_VAULT_ROLE=sso-govcloud-us1-fed-engineering
45+
46+
export STAGE=gov-prod
47+
48+
if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_py-signed-bundle-[0-9]+$ ]]; then
49+
echo "[ERROR]: Unexpected package name: $PACKAGE_NAME"
50+
exit 1
51+
fi
52+
53+
else
54+
printf "[ERROR]: ENVIRONMENT not supported, must be us1-staging-fed or us1-fed.\n"
55+
exit 1
56+
fi
57+
58+
TEMP_DIR=$(mktemp -d)
59+
unzip $LAYER_PACKAGE -d $TEMP_DIR
60+
cp -v $TEMP_DIR/$PACKAGE_NAME/*.zip .layers/
61+
62+
63+
AWS_VAULT_PREFIX="aws-vault exec $AWS_VAULT_ROLE --"
64+
65+
echo "Checking that you have access to the GovCloud AWS account"
66+
$AWS_VAULT_PREFIX aws sts get-caller-identity
67+
68+
69+
AVAILABLE_REGIONS=$($AWS_VAULT_PREFIX aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
70+
71+
# Determine the target regions
72+
if [ -z "$REGIONS" ]; then
73+
echo "Region not specified, running for all available regions."
74+
REGIONS=$AVAILABLE_REGIONS
75+
else
76+
echo "Region specified: $REGIONS"
77+
if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then
78+
echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS"
79+
echo ""
80+
echo "EXITING SCRIPT."
81+
exit 1
82+
fi
83+
fi
84+
85+
for region in $REGIONS
86+
do
87+
echo "Starting publishing layers for region $region..."
88+
89+
export REGION=$region
90+
91+
for python_version in "3.8" "3.9" "3.10" "3.11" "3.12" "3.13"; do
92+
for arch in "amd64" "arm64"; do
93+
export PYTHON_VERSION=$python_version
94+
export ARCH=$arch
95+
96+
export SKIP_PIP_INSTALL=true
97+
98+
echo "Publishing layer for $PYTHON_VERSION and $ARCH"
99+
100+
$AWS_VAULT_PREFIX ./ci/publish_layers.sh
101+
done
102+
done
103+
done
104+
105+
echo "Done !"

0 commit comments

Comments
 (0)