[EXPERIMENTAL] AppSec snapshot generation with unreleased tracer

.github/workflows/update-snapshots.yml

@@ -0,0 +1,60 @@
+name: update-snapshots
+
+on:
+  workflow_dispatch:
+
+jobs:
+  update-snapshots:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      matrix:
+        runtime-param: [3.2, 3.3, 3.4]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Node 20
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version: 20
+
+      - name: Cache Node modules
+        id: cache-node-modules
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: "**/node_modules"
+          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install Serverless Framework
+        run: sudo yarn global add serverless@3 --prefix /usr/local
+
+      - name: Install dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        working-directory: integration_tests
+        run: yarn install
+
+      - name: Install Crossbuild dependencies
+        run: |
+          sudo apt-get update --allow-releaseinfo-change --fix-missing
+          sudo apt install -y qemu-user-static binfmt-support
+
+      - name: Update Snapshots (amd64)
+        env:
+          ARCH: amd64
+          UPDATE_SNAPSHOTS: true
+          BUILD_LAYERS: true
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          RUNTIME_PARAM: ${{ matrix.runtime-param }}
+        run: ./scripts/run_integration_tests.sh
+
+      - name: Upload snapshots
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: snapshots-${{ matrix.runtime-param }}
+          path: |
+            integration_tests/snapshots/

.gitlab/scripts/build_layer.sh

@@ -9,7 +9,7 @@ set -e
 
 LAYER_DIR=".layers"
 LAYER_FILES_PREFIX="datadog-lambda_ruby"
-AVAILABLE_RUBY_VERSIONS=("3.2" "3.3" "3.4")
+AVAILABLE_RUBY_VERSIONS=("3.2" "3.3" "3.4" "4.0")
 
 if [ -z "$ARCH" ]; then
     echo "[ERROR]: ARCH not specified"
@@ -37,13 +37,18 @@ function docker_build_zip {
 
     destination=$(make_path_absolute $2)
     arch=$3
+    ref=$4
 
     # Install datadog ruby in a docker container to avoid the mess from switching
     # between different ruby runtimes.
+    #
+    # NOTE: using the Lambda base image so native extensions (FFI, libddwaf)
+    #       compile against the same libffi available at runtime on Lambda.
     temp_dir=$(mktemp -d)
     docker buildx build -t datadog-lambda-ruby-${arch}:$1 . --no-cache \
-        --build-arg "image=ruby:${1}" \
+        --build-arg "image=public.ecr.aws/lambda/ruby:${1}" \
         --build-arg "runtime=${1}.0" \
+        --build-arg "git_ref=${ref}" \
         --platform linux/${arch} \
         --progress=plain \
         -o $temp_dir
@@ -59,7 +64,7 @@ rm -rf $LAYER_DIR
 mkdir $LAYER_DIR
 
 echo "Building layer for Ruby $RUBY_VERSION with architecture $ARCH"
-docker_build_zip $RUBY_VERSION $LAYER_DIR/${LAYER_FILES_PREFIX}-${ARCH}-${RUBY_VERSION}.zip $ARCH
+docker_build_zip $RUBY_VERSION $LAYER_DIR/${LAYER_FILES_PREFIX}-${ARCH}-${RUBY_VERSION}.zip $ARCH $GIT_REF
 
 echo "Done creating layers:"
 ls $LAYER_DIR | xargs -I _ echo "$LAYER_DIR/_"

.rubocop.yml

@@ -1,5 +1,8 @@
 AllCops:
   TargetRubyVersion: 3.2
+  Exclude:
+    - 'test/**/*'
+    - 'vendor/**/*'
 
 Metrics/MethodLength:
   Max: 20

Dockerfile

@@ -1,18 +1,45 @@
 ARG image
 FROM $image AS builder
+ARG git_ref
 ARG runtime
+RUN echo "git_ref: $git_ref"
 # Install dev dependencies
 COPY . /var/task/datadog-lambda-rb
 WORKDIR /var/task/datadog-lambda-rb
-RUN apt-get update
-RUN apt-get install -y gcc zip binutils
+# NOTE: AL2 (Ruby 3.2) uses yum, AL2023 (Ruby 3.3+) uses dnf
+RUN PKG=$(command -v dnf || command -v yum) && \
+    $PKG install -y gcc gcc-c++ make zip binutils libffi-devel
 
 # Install this gem
 RUN gem build datadog-lambda
 
 # Install ddtrace gem
-RUN gem install datadog-lambda --install-dir "/opt/ruby/gems/$runtime"
-RUN gem install datadog -v 2.12 --install-dir "/opt/ruby/gems/$runtime"
+RUN MAKEFLAGS="-j$(nproc)" \
+    gem install datadog-lambda --install-dir "/opt/ruby/gems/$runtime" --no-document
+RUN set -eux; \
+    if [ -z "${git_ref:-}" ]; then \
+        # NOTE: datadog gem must be >= 2.24 to install on Ruby 4.0.x.
+        MAKEFLAGS="-j$(nproc)" \
+        gem install datadog -v 2.30 --install-dir "/opt/ruby/gems/$runtime" --no-document; \
+    else \
+        echo "building tracer from ref: $git_ref\n"; \
+        git clone https://github.com/DataDog/dd-trace-rb.git --depth 1 --single-branch -b $git_ref /tmp/dd-trace-rb; \
+        cd /tmp/dd-trace-rb; \
+        gem build datadog.gemspec; \
+        MAKEFLAGS="-j$(nproc)" \
+        gem install ./datadog-*.gem --install-dir "/opt/ruby/gems/$runtime" --no-document; \
+    fi
+
+# Recompile FFI from source — precompiled binaries ship ABI-specific ffi_c.so
+# for Ruby 3.3/3.4 only. Ruby 3.2 ABI is missing, causing LoadError at boot
+# when AppSec loads libddwaf → ffi → ffi_c.
+#
+# NOTE: runs after datadog gem as a defensive measure — force-replaces whatever
+#       transitive FFI variant was pulled, regardless of version resolution.
+RUN gem uninstall ffi --all --ignore-dependencies --executables --force \
+        --install-dir "/opt/ruby/gems/$runtime" || true
+RUN MAKEFLAGS="-j$(nproc)" \
+    gem install ffi --platform ruby --install-dir "/opt/ruby/gems/$runtime" --no-document
 
 WORKDIR /opt
 # Remove native extension debase-ruby_core_source (25MB) runtimes below Ruby 2.6

integration_tests/appsec_request.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'datadog/lambda'
+
+Datadog::Lambda.configure_apm do |c|
+  c.appsec.enabled = true
+end
+
+def handle(event:, context:)
+  Datadog::Lambda.wrap(event, context) do
+    Datadog::Lambda.metric('serverless.integration_test.execution', 1, function: 'appsec-request')
+
+    {
+      'statusCode' => 200,
+      'message' => 'hello, dog!',
+      'eventType' => 'APIGateway'
+    }
+  end
+end

integration_tests/input_events/api-gateway-appsec-blocking.json

@@ -0,0 +1,31 @@
+{
+  "path": "/test/hello",
+  "headers": {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com",
+    "X-Forwarded-For": "192.168.100.1, 192.168.1.1",
+    "X-Forwarded-Port": "443",
+    "X-Forwarded-Proto": "https"
+  },
+  "pathParameters": {
+    "proxy": "hello"
+  },
+  "requestContext": {
+    "accountId": "123456789012",
+    "resourceId": "us4z18",
+    "stage": "test",
+    "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9",
+    "identity": {
+      "sourceIp": "192.168.100.1"
+    },
+    "resourcePath": "/{proxy+}",
+    "httpMethod": "GET",
+    "apiId": "wt6mne2s9k"
+  },
+  "resource": "/{proxy+}",
+  "httpMethod": "GET",
+  "queryStringParameters": {
+    "q": "1' OR '1'='1"
+  },
+  "stageVariables": null
+}

integration_tests/input_events/api-gateway-appsec.json

@@ -0,0 +1,31 @@
+{
+  "path": "/test/hello",
+  "headers": {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com",
+    "User-Agent": "Arachni/v1.0",
+    "X-Forwarded-For": "192.168.100.1, 192.168.1.1",
+    "X-Forwarded-Port": "443",
+    "X-Forwarded-Proto": "https"
+  },
+  "pathParameters": {
+    "proxy": "hello"
+  },
+  "requestContext": {
+    "accountId": "123456789012",
+    "resourceId": "us4z18",
+    "stage": "test",
+    "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9",
+    "identity": {
+      "sourceIp": "192.168.100.1",
+      "userAgent": "Arachni/v1.0"
+    },
+    "resourcePath": "/{proxy+}",
+    "httpMethod": "GET",
+    "apiId": "wt6mne2s9k"
+  },
+  "resource": "/{proxy+}",
+  "httpMethod": "GET",
+  "queryStringParameters": null,
+  "stageVariables": null
+}

integration_tests/serverless.yml

@@ -60,3 +60,15 @@ functions:
       - { Ref: RubyLambdaLayer }
     environment:
       DD_FLUSH_TO_LOG: true
+
+  # appsec-request
+  appsec-request_ruby:
+    name: integration-tests-rb-${sls:stage}-appsec-request_${env:RUNTIME}
+    handler: appsec_request.handle
+    runtime: ${env:SERVERLESS_RUNTIME}
+    memorySize: 1024
+    layers:
+      - { Ref: RubyLambdaLayer }
+    environment:
+      DD_FLUSH_TO_LOG: true
+      DD_APPSEC_ENABLED: true

integration_tests/snapshots/logs/appsec-request_ruby32.log

@@ -0,0 +1,29 @@
+
+END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - CORE - {"date":"XXXX","os_name":"XXXX","version":"2.33.0","lang":"ruby","lang_version":"3.2.X","env":null,"service":"index","dd_version":null,"debug":false,"tags":"_dd.origin:lambda","runtime_metrics_enabled":false,"vm":"ruby-3.2.X","health_metrics_enabled":false,"profiling_enabled":false,"dynamic_instrumentation_enabled":false}
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - TRACING - {"enabled":true,"agent_url":null,"analytics_enabled":false,"sample_rate":null,"sampling_rules":null,"integrations_loaded":"aws@","partial_flushing_enabled":false}
+START
+START
+START
+START
+START
+W, [XXXX]  WARN XXXX[datadog] Unable to patch Datadog::Tracing::Contrib::Aws::Integration (Available?: false, Loaded? false, Compatible? false, Patchable? false)
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby32","functionname:integration-tests-rb-XXXX-appsec-request_ruby32","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:true","runtime:Ruby 3.2.X","resource:integration-tests-rb-XXXX-appsec-request_ruby32","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby32","function:appsec-request"],"v":1}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"is_sqli\",\"operator_value\":\"\",\"parameters\":[{\"address\":\"server.request.query\",\"key_path\":[\"q\"],\"value\":\"1' OR '1'='1\",\"highlight\":[\"s&sos\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"match_regex\",\"operator_value\":\"^Arachni\\\\/v\",\"parameters\":[{\"address\":\"server.request.headers.no_cookies\",\"key_path\":[\"user-agent\"],\"value\":\"Arachni/v1.0\",\"highlight\":[\"Arachni/v\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}

integration_tests/snapshots/logs/appsec-request_ruby33.log

@@ -0,0 +1,29 @@
+
+END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+END Duration: XXXX ms (init: XXXX) Memory Used: XXXX MB
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - CORE - {"date":"XXXX","os_name":"XXXX","version":"2.33.0","lang":"ruby","lang_version":"3.3.X","env":null,"service":"index","dd_version":null,"debug":false,"tags":"_dd.origin:lambda","runtime_metrics_enabled":false,"vm":"ruby-3.3.X","health_metrics_enabled":false,"profiling_enabled":false,"dynamic_instrumentation_enabled":false}
+I, [XXXX]  INFO XXXX[datadog] DATADOG CONFIGURATION - TRACING - {"enabled":true,"agent_url":null,"analytics_enabled":false,"sample_rate":null,"sampling_rules":null,"integrations_loaded":"aws@","partial_flushing_enabled":false}
+START
+START
+START
+START
+START
+W, [XXXX]  WARN XXXX[datadog] Unable to patch Datadog::Tracing::Contrib::Aws::Integration (Available?: false, Loaded? false, Compatible? false, Patchable? false)
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:false","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"aws.lambda.enhanced.invocations","t":["dd_lambda_layer:datadog-ruby33","functionname:integration-tests-rb-XXXX-appsec-request_ruby33","region:eu-west-1","account_id:XXXX","memorysize:1024","cold_start:true","runtime:Ruby 3.3.X","resource:integration-tests-rb-XXXX-appsec-request_ruby33","datadog_lambda:X.X.X","dd_trace:2.XX.X"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"e":XXXX,"m":"serverless.integration_test.execution","t":["dd_lambda_layer:datadog-ruby33","function:appsec-request"],"v":1}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"is_sqli\",\"operator_value\":\"\",\"parameters\":[{\"address\":\"server.request.query\",\"key_path\":[\"q\"],\"value\":\"1' OR '1'='1\",\"highlight\":[\"s&sos\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}
+{"traces":[[{"error":0,"meta":{"XXXX": "XXXX"},\"on_match\":[]},\"rule_matches\":[{\"operator\":\"match_regex\",\"operator_value\":\"^Arachni\\\\/v\",\"parameters\":[{\"address\":\"server.request.headers.no_cookies\",\"key_path\":[\"user-agent\"],\"value\":\"Arachni/v1.0\",\"highlight\":[\"Arachni/v\"]}]}]}]}"},"metrics":{"XXXX": "XXXX"},"meta_struct":{},"name":"aws.lambda","parent_id":"XXXX","resource":"dd-tracer-serverless-span","service":"aws.lambda","span_id":"XXXX","trace_id":"XXXX","type":"serverless","span_links":[],"start":XXXX,"duration":XXXX}]]}