Run e2e tests on each PR. (#584)

Author: purple4reina

ci/input_files/build.yaml.tpl

@@ -1,8 +1,11 @@
+{{- $e2e_region := "us-west-2" -}}
+
 stages:
  - build
  - test
  - sign
  - publish
+ - e2e
 
 .python-before-script: &python-before-script
   - pip install virtualenv
@@ -56,11 +59,11 @@ check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: test
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10
-  needs: 
+  needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  script: 
+  script:
     - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
 
 lint python:
@@ -69,7 +72,7 @@ lint python:
   image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
   cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
   before_script: *python-before-script
-  script: 
+  script:
     - source venv/bin/activate
     - ./scripts/check_format.sh
 
@@ -79,15 +82,15 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
   image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
   cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
   before_script: *python-before-script
-  script: 
+  script:
     - source venv/bin/activate
     - pytest -vv
 
 integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: test
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
-  needs: 
+  needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
@@ -132,16 +135,22 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
     - LAYER_FILE=datadog_lambda_py-{{ $runtime.arch}}-{{ $runtime.python_version }}.zip ./scripts/sign_layers.sh prod
 
 {{ range $environment_name, $environment := (ds "environments").environments }}
+{{ $dotenv := print $runtime.name "_" $runtime.arch "_" $environment_name ".env" }}
 
 publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: publish
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
+    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}" && "{{ $runtime.arch }}" == "amd64"'
+      when: on_success
     - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
+  artifacts:
+    reports:
+      dotenv: {{ $dotenv }}
   needs:
 {{ if or (eq $environment_name "prod") }}
       - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
@@ -166,7 +175,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
-    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh
+    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
 
 {{- end }}
 
@@ -232,3 +241,56 @@ signed layer bundle:
     - rm -rf datadog_lambda_py-signed-bundle-${CI_JOB_ID}
     - mkdir -p datadog_lambda_py-signed-bundle-${CI_JOB_ID}
     - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-signed-bundle-${CI_JOB_ID}
+
+e2e-test:
+  stage: e2e
+  trigger:
+    project: DataDog/serverless-e2e-tests
+    strategy: depend
+  variables:
+      LANGUAGES_SUBSET: python
+      # These env vars are inherited from the dotenv reports of the publish-layer jobs
+    {{- range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+    {{- $version := print (.name | strings.Trim "python") }}
+      PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION
+    {{- end }}
+    {{- end }}
+  needs: {{ range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+      - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
+    {{- end }}
+    {{- end }}
+
+e2e-test-status:
+  stage: e2e
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  tags: ["arch:amd64"]
+  timeout: 3h
+  script: |
+      GITLAB_API_TOKEN=$(aws ssm get-parameter --region us-east-1 --name "ci.${CI_PROJECT_NAME}.serverless-e2e-gitlab-token" --with-decryption --query "Parameter.Value" --out text)
+      URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
+      echo "Fetching E2E job status from: $URL"
+      while true; do
+        RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" "$URL")
+        E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .downstream_pipeline.status')
+        echo -n "E2E job status: $E2E_JOB_STATUS, "
+        if [ "$E2E_JOB_STATUS" == "success" ]; then
+          echo "✅ E2E tests completed successfully"
+          exit 0
+        elif [ "$E2E_JOB_STATUS" == "failed" ]; then
+          echo "❌ E2E tests failed"
+          exit 1
+        elif [ "$E2E_JOB_STATUS" == "running" ]; then
+          echo "⏳ E2E tests are still running, retrying in 1 minute..."
+        elif [ "$E2E_JOB_STATUS" == "canceled" ]; then
+          echo "🚫 E2E tests were canceled"
+          exit 1
+        elif [ "$E2E_JOB_STATUS" == "skipped" ]; then
+          echo "⏭️ E2E tests were skipped"
+          exit 0
+        else
+          echo "❓ Unknown E2E test status: $E2E_JOB_STATUS, retrying in 1 minute..."
+        fi
+        sleep 60
+      done

ci/publish_layers.sh

@@ -199,6 +199,8 @@ fi
 while [ $latest_version -lt $VERSION ]; do
     latest_version=$(publish_layer $REGION $layer $aws_cli_python_version_key $layer_path)
     printf "[$REGION] Published version $latest_version for layer $layer in region $REGION\n"
+    latest_arn=$(aws lambda get-layer-version --layer-name $layer --version-number $latest_version --region $REGION --query 'LayerVersionArn' --output text)
+    printf "[$REGION] Published arn $latest_arn\n"
 
     # This shouldn't happen unless someone manually deleted the latest version, say 28, and
     # then tries to republish 28 again. The published version would actually be 29, because
@@ -209,4 +211,10 @@ while [ $latest_version -lt $VERSION ]; do
     fi
 done
 
+if [ -n "$DOTENV" ]; then
+    printf "[$REGION] Exporting layer version to $DOTENV file...\n"
+    echo "PYTHON_${PYTHON_VERSION/./}_VERSION=$latest_arn" >> "$DOTENV"
+    cat "$DOTENV"
+fi
+
 printf "[$REGION] Finished publishing layers...\n\n"