[APPSEC-61865] Add AppSec setup for AWS Lambda

.gitlab/scripts/build_layer.sh

@@ -40,9 +40,12 @@ function docker_build_zip {
 
     # Install datadog ruby in a docker container to avoid the mess from switching
     # between different ruby runtimes.
+    #
+    # NOTE: using the Lambda base image so native extensions (FFI, libddwaf)
+    #       compile against the same libffi available at runtime on Lambda.
     temp_dir=$(mktemp -d)
     docker buildx build -t datadog-lambda-ruby-${arch}:$1 . --no-cache \
-        --build-arg "image=ruby:${1}" \
+        --build-arg "image=public.ecr.aws/lambda/ruby:${1}" \
         --build-arg "runtime=${1}.0" \
         --platform linux/${arch} \
         --progress=plain \

.rubocop.yml

@@ -1,5 +1,8 @@
 AllCops:
   TargetRubyVersion: 3.2
+  Exclude:
+    - 'test/**/*'
+    - 'vendor/**/*'
 
 Metrics/MethodLength:
   Max: 20

Dockerfile

@@ -4,15 +4,28 @@ ARG runtime
 # Install dev dependencies
 COPY . /var/task/datadog-lambda-rb
 WORKDIR /var/task/datadog-lambda-rb
-RUN apt-get update
-RUN apt-get install -y gcc zip binutils
+
+# NOTE: AL2 (Ruby 3.2) uses yum, AL2023 (Ruby 3.3+) uses dnf
+RUN PKG=$(command -v dnf || command -v yum) && \
+    $PKG install -y gcc gcc-c++ make zip binutils libffi-devel
 
 # Install this gem
 RUN gem build datadog-lambda
 
 # Install ddtrace gem
-RUN gem install datadog-lambda --install-dir "/opt/ruby/gems/$runtime"
-RUN gem install datadog -v 2.12 --install-dir "/opt/ruby/gems/$runtime"
+RUN MAKEFLAGS="-j$(nproc)" \
+    gem install datadog-lambda --install-dir "/opt/ruby/gems/$runtime" --no-document
+RUN MAKEFLAGS="-j$(nproc)" \
+    gem install datadog -v 2.33 --install-dir "/opt/ruby/gems/$runtime" --no-document
+
+# Recompile FFI from source — precompiled binaries have glibc mismatch with Lambda AL2
+#
+# NOTE: runs after datadog gem as a defensive measure — force-replaces whatever
+#       transitive FFI variant was pulled, regardless of version resolution.
+RUN MAKEFLAGS="-j$(nproc)" \
+    gem install ffi -v 1.17.4 --platform ruby --force --install-dir "/opt/ruby/gems/$runtime" --no-document
+RUN rm -rf /opt/ruby/gems/$runtime/gems/ffi-*-*-linux-* \
+           /opt/ruby/gems/$runtime/specifications/ffi-*-*-linux-*.gemspec
 
 WORKDIR /opt
 # Remove native extension debase-ruby_core_source (25MB) runtimes below Ruby 2.6
@@ -22,7 +35,7 @@ RUN rm -rf ./ruby/gems/$runtime/gems/aws*/
 # Remove binaries not needed in AWS Lambda
 RUN find . -name '*linux-musl*' -prune -exec rm -rf {} +
 
-# Cache files zipped gem files, that aren't used by during runtime, only during 
+# Cache files zipped gem files, that aren't used by during runtime, only during
 # installation, so they are safe to delete
 RUN rm -rf "/opt/ruby/gems/${runtime}/cache"
 RUN cd /opt

integration_tests/appsec_request.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'datadog/lambda'
+
+Datadog::Lambda.configure_apm do |c|
+  c.appsec.enabled = true
+end
+
+def handle(event:, context:)
+  Datadog::Lambda.wrap(event, context) do
+    Datadog::Lambda.metric('serverless.integration_test.execution', 1, function: 'appsec-request')
+
+    {
+      'statusCode' => 200,
+      'message' => 'hello, dog!',
+      'eventType' => 'APIGateway'
+    }
+  end
+end

integration_tests/input_events/api-gateway-appsec-blocking.json

@@ -0,0 +1,31 @@
+{
+  "path": "/test/hello",
+  "headers": {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com",
+    "X-Forwarded-For": "192.168.100.1, 192.168.1.1",
+    "X-Forwarded-Port": "443",
+    "X-Forwarded-Proto": "https"
+  },
+  "pathParameters": {
+    "proxy": "hello"
+  },
+  "requestContext": {
+    "accountId": "123456789012",
+    "resourceId": "us4z18",
+    "stage": "test",
+    "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9",
+    "identity": {
+      "sourceIp": "192.168.100.1"
+    },
+    "resourcePath": "/{proxy+}",
+    "httpMethod": "GET",
+    "apiId": "wt6mne2s9k"
+  },
+  "resource": "/{proxy+}",
+  "httpMethod": "GET",
+  "queryStringParameters": {
+    "q": "1' OR '1'='1"
+  },
+  "stageVariables": null
+}

integration_tests/input_events/api-gateway-appsec.json

@@ -0,0 +1,31 @@
+{
+  "path": "/test/hello",
+  "headers": {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com",
+    "User-Agent": "Arachni/v1.0",
+    "X-Forwarded-For": "192.168.100.1, 192.168.1.1",
+    "X-Forwarded-Port": "443",
+    "X-Forwarded-Proto": "https"
+  },
+  "pathParameters": {
+    "proxy": "hello"
+  },
+  "requestContext": {
+    "accountId": "123456789012",
+    "resourceId": "us4z18",
+    "stage": "test",
+    "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9",
+    "identity": {
+      "sourceIp": "192.168.100.1",
+      "userAgent": "Arachni/v1.0"
+    },
+    "resourcePath": "/{proxy+}",
+    "httpMethod": "GET",
+    "apiId": "wt6mne2s9k"
+  },
+  "resource": "/{proxy+}",
+  "httpMethod": "GET",
+  "queryStringParameters": null,
+  "stageVariables": null
+}

integration_tests/serverless.yml

@@ -60,3 +60,15 @@ functions:
       - { Ref: RubyLambdaLayer }
     environment:
       DD_FLUSH_TO_LOG: true
+
+  # appsec-request
+  appsec-request_ruby:
+    name: integration-tests-rb-${sls:stage}-appsec-request_${env:RUNTIME}
+    handler: appsec_request.handle
+    runtime: ${env:SERVERLESS_RUNTIME}
+    memorySize: 1024
+    layers:
+      - { Ref: RubyLambdaLayer }
+    environment:
+      DD_FLUSH_TO_LOG: true
+      DD_APPSEC_ENABLED: true

integration_tests/snapshots/logs/appsec-request_ruby32.log

@@ -0,0 +1,29 @@
+
+END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - CORE - {"date":"XXXX","os_name":"XXXX","version":"2.32.0","lang":"ruby","lang_version":"3.2.X","env":null,"service":"index","dd_version":null,"debug":false,"tags":"_dd.origin:lambda","runtime_metrics_enabled":false,"vm":"ruby-3.2.X","health_metrics_enabled":false,"profiling_enabled":false,"dynamic_instrumentation_enabled":false}
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - TRACING - {"enabled":true,"agent_url":null,"analytics_enabled":false,"sample_rate":null,"sampling_rules":null,"integrations_loaded":"aws@","partial_flushing_enabled":false}
+START
+START
+START
+START
+START
+W, [XXXX]  WARN XXXX[datadog] Unable to patch Datadog::Tracing::Contrib::Aws::Integration (Available?: false, Loaded? false, Compatible? false, Patchable? false)
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:true","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"is_sqli\",\"operator_value\":\"\",\"parameters\":[{\"address\":\"server.request.query\",\"key_path\":[\"q\"],\"value\":\"1' OR '1'='1\",\"highlight\":[\"s&sos\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"match_regex\",\"operator_value\":\"^Arachni\\\\/v\",\"parameters\":[{\"address\":\"server.request.headers.no_cookies\",\"key_path\":[\"user-agent\"],\"value\":\"Arachni/v1.0\",\"highlight\":[\"Arachni/v\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}

integration_tests/snapshots/logs/appsec-request_ruby33.log

@@ -0,0 +1,29 @@
+
+END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - CORE - {"date":"XXXX","os_name":"XXXX","version":"2.32.0","lang":"ruby","lang_version":"3.3.X","env":null,"service":"index","dd_version":null,"debug":false,"tags":"_dd.origin:lambda","runtime_metrics_enabled":false,"vm":"ruby-3.3.X","health_metrics_enabled":false,"profiling_enabled":false,"dynamic_instrumentation_enabled":false}
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - TRACING - {"enabled":true,"agent_url":null,"analytics_enabled":false,"sample_rate":null,"sampling_rules":null,"integrations_loaded":"aws@","partial_flushing_enabled":false}
+START
+START
+START
+START
+START
+W, [XXXX]  WARN XXXX[datadog] Unable to patch Datadog::Tracing::Contrib::Aws::Integration (Available?: false, Loaded? false, Compatible? false, Patchable? false)
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:true","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"is_sqli\",\"operator_value\":\"\",\"parameters\":[{\"address\":\"server.request.query\",\"key_path\":[\"q\"],\"value\":\"1' OR '1'='1\",\"highlight\":[\"s&sos\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"match_regex\",\"operator_value\":\"^Arachni\\\\/v\",\"parameters\":[{\"address\":\"server.request.headers.no_cookies\",\"key_path\":[\"user-agent\"],\"value\":\"Arachni/v1.0\",\"highlight\":[\"Arachni/v\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}