diff --git a/.github/workflows/vault.yml b/.github/workflows/vault.yml new file mode 100644 index 00000000..8deca0ff --- /dev/null +++ b/.github/workflows/vault.yml @@ -0,0 +1,73 @@ +name: LocalStack Vault Extension Tests + +on: + push: + paths: + - vault/** + branches: + - main + pull_request: + paths: + - .github/workflows/vault.yml + - vault/** + workflow_dispatch: + +env: + LOCALSTACK_DISABLE_EVENTS: "1" + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + +jobs: + integration-tests: + name: Run Integration Tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Vault + uses: eLco/setup-vault@v1 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Setup LocalStack and extension + run: | + cd vault + + # Pull Docker images in parallel + docker pull localstack/localstack-pro & + docker pull hashicorp/vault:latest & + docker pull public.ecr.aws/lambda/python:3.11 & + pip install localstack awscli-local[ver1] + + # Install and build extension + make install + make lint + make dist + localstack extensions -v install file://$(ls ./dist/localstack_vault-*.tar.gz) + + # Wait for Docker pulls to complete + wait + + # Start LocalStack with extension + DEBUG=1 EXTENSION_DEV_MODE=1 localstack start -d + localstack wait + + - name: Run pytest tests + run: | + cd vault + make test + + - name: Run sample app (Lambda + Vault Extension) + run: | + cd vault + make sample-app + + - name: Print logs + if: always() + run: | + localstack logs + localstack stop diff --git a/vault/.gitignore b/vault/.gitignore new file mode 100644 index 00000000..559f213d --- /dev/null +++ b/vault/.gitignore @@ -0,0 +1,24 @@ +# Python +.venv/ +build/ +dist/ +.eggs/ +*egg-info/ +__pycache__/ +.pytest_cache/ + +# Terraform +.terraform/ +.terraform.lock.hcl +terraform.tfstate* +*.tfplan + +# OS +.DS_Store + +# Temp files +*.zip +/tmp/ + +# Ignored Directories +sample-app-terraform/ diff --git a/vault/Makefile b/vault/Makefile new file mode 100644 index 00000000..cfb83e0c --- /dev/null +++ b/vault/Makefile @@ -0,0 +1,55 @@ +VENV_BIN = python3 -m venv +VENV_DIR ?= .venv +VENV_ACTIVATE = $(VENV_DIR)/bin/activate +VENV_RUN = . $(VENV_ACTIVATE) + +usage: ## Shows usage for this Makefile + @cat Makefile | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' + +venv: $(VENV_ACTIVATE) + +$(VENV_ACTIVATE): pyproject.toml + test -d .venv || $(VENV_BIN) .venv + $(VENV_RUN); pip install --upgrade pip setuptools plux + $(VENV_RUN); pip install -e .[dev] + touch $(VENV_DIR)/bin/activate + +clean: + rm -rf .venv/ + rm -rf build/ + rm -rf .eggs/ + rm -rf *.egg-info/ + +install: venv ## Install dependencies + $(VENV_RUN); python -m plux entrypoints + +dist: venv ## Create distribution + $(VENV_RUN); python -m build + +publish: clean-dist venv dist ## Publish extension to pypi + $(VENV_RUN); pip install --upgrade twine; twine upload dist/* + +entrypoints: venv # Generate plugin entrypoints for Python package + $(VENV_RUN); python -m plux entrypoints + +format: ## Run ruff to format the whole codebase + $(VENV_RUN); python -m ruff format .; python -m ruff check --output-format=full --fix . + +lint: ## Run ruff to lint the codebase + $(VENV_RUN); python -m ruff check --output-format=full . + +test: ## Run pytest tests (requires LocalStack running) + $(VENV_RUN); python -m pytest tests/ -v + +sample-app: ## Deploy sample app with Lambda + Vault + @echo "Setting up Vault secrets..." + sample-app-extension/bin/setup-vault.sh + @echo "Deploying Lambda function with Vault Lambda Extension..." + sample-app-extension/bin/deploy-lambda.sh + @echo "Testing Lambda invocation..." + sample-app-extension/bin/test-lambda.sh + +clean-dist: clean + rm -rf dist/ + +.PHONY: clean clean-dist dist install publish usage venv format lint test sample-app diff --git a/vault/README.md b/vault/README.md new file mode 100644 index 00000000..cd6be626 --- /dev/null +++ b/vault/README.md @@ -0,0 +1,104 @@ +# HashiCorp Vault on LocalStack + +This [LocalStack Extension](https://github.com/localstack/localstack-extensions) runs [HashiCorp Vault](https://www.vaultproject.io/) alongside LocalStack for secrets management testing. + +## Prerequisites + +- Docker +- LocalStack Pro (free trial available) +- `localstack` CLI + +## Install from GitHub repository + +```bash +localstack extensions install "git+https://github.com/localstack/localstack-extensions.git#egg=localstack-extension-vault&subdirectory=vault" +``` + +## Install local development version + +```bash +make install +localstack extensions dev enable . +``` + +Start LocalStack with `EXTENSION_DEV_MODE=1`: + +```bash +EXTENSION_DEV_MODE=1 localstack start +``` + +## Usage + +Vault is available at `http://vault.localhost.localstack.cloud:4566`. + +### Add Secrets + +```bash +export VAULT_ADDR=http://vault.localhost.localstack.cloud:4566 +export VAULT_TOKEN=root + +# Add a secret +vault kv put secret/my-app/config api_key=secret123 db_password=hunter2 + +# Verify +vault kv get secret/my-app/config +``` + +### Use with Lambda + +Deploy a Lambda with the [Vault Lambda Extension](https://github.com/hashicorp/vault-lambda-extension) layer and these environment variables: + +| Variable | Value | +|----------|-------| +| `VAULT_ADDR` | `http://vault.localhost.localstack.cloud:4566` | +| `VAULT_AUTH_PROVIDER` | `aws` | +| `VAULT_AUTH_ROLE` | `default-lambda-role` | +| `VAULT_SECRET_PATH_*` | Path to your secret (e.g., `secret/data/my-app/config`) | +| `VAULT_SECRET_FILE_*` | Where extension writes secrets (e.g., `/tmp/secrets/myapp`) | + +See `sample-app-extension/` for a complete working example. + +## Configuration + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `VAULT_ROOT_TOKEN` | `root` | Dev mode root token | +| `VAULT_PORT` | `8200` | Vault API port (internal) | + +## Pre-configured Resources + +### Secrets Engines + +| Path | Type | Description | +|------|------|-------------| +| `secret/` | KV v2 | Key-value secrets storage | +| `transit/` | Transit | Encryption as a service | + +### Auth Methods + +| Path | Type | Description | +|------|------|-------------| +| `aws/` | AWS IAM | Pre-configured for Lambda IAM auth | + +### Policies + +| Name | Permissions | +|------|-------------| +| `default-lambda-policy` | Full access to `secret/*` and `transit/*` | + +## Sample App + +See `sample-app-extension/` for a complete working example using the official Vault Lambda Extension layer. + +```bash +make sample-app +``` + +## Limitations + +- **Ephemeral**: All secrets are lost when LocalStack restarts +- **Dev mode only**: No production Vault features (seal/unseal, HA, etc.) + +## Disclaimer + +This extension is not affiliated with HashiCorp. Vault is a trademark of HashiCorp, Inc. diff --git a/vault/localstack_vault/__init__.py b/vault/localstack_vault/__init__.py new file mode 100644 index 00000000..552b3439 --- /dev/null +++ b/vault/localstack_vault/__init__.py @@ -0,0 +1 @@ +# HashiCorp Vault Extension for LocalStack diff --git a/vault/localstack_vault/extension.py b/vault/localstack_vault/extension.py new file mode 100644 index 00000000..124050fe --- /dev/null +++ b/vault/localstack_vault/extension.py @@ -0,0 +1,230 @@ +import logging +import os +import time + +import hvac +import requests + +from localstack import config, constants +from localstack.utils.container_networking import get_main_container_ip +from localstack.utils.net import get_addressable_container_host +from localstack_extensions.utils.docker import ProxiedDockerContainerExtension + +LOG = logging.getLogger(__name__) + +# Environment variables +ENV_VAULT_ROOT_TOKEN = "VAULT_ROOT_TOKEN" +ENV_VAULT_PORT = "VAULT_PORT" + +# Defaults +DEFAULT_ROOT_TOKEN = "root" +DEFAULT_PORT = 8200 + + +class VaultExtension(ProxiedDockerContainerExtension): + """ + HashiCorp Vault Extension for LocalStack. + + Runs Vault in dev mode with: + - KV v2 secrets engine at secret/ + - Transit secrets engine at transit/ + - IAM auth method pre-configured to accept any Lambda role + """ + + name = "localstack-vault" + + HOST = "vault." + DOCKER_IMAGE = "hashicorp/vault:latest" + + def __init__(self): + self.root_token = os.getenv(ENV_VAULT_ROOT_TOKEN, DEFAULT_ROOT_TOKEN) + self.vault_port = int(os.getenv(ENV_VAULT_PORT, DEFAULT_PORT)) + + env_vars = { + "VAULT_DEV_ROOT_TOKEN_ID": self.root_token, + "VAULT_DEV_LISTEN_ADDRESS": f"0.0.0.0:{self.vault_port}", + "VAULT_LOG_LEVEL": "info", + } + + def _health_check(): + """Check if Vault is initialized and unsealed.""" + container_host = get_addressable_container_host() + health_url = f"http://{container_host}:{self.vault_port}/v1/sys/health" + LOG.debug("Vault health check: %s", health_url) + response = requests.get(health_url, timeout=5) + # Vault returns 200 when initialized, unsealed, and active + # In dev mode, it should always be ready + assert response.status_code == 200, f"Vault not ready: {response.status_code}" + + super().__init__( + image_name=self.DOCKER_IMAGE, + container_ports=[self.vault_port], + host=self.HOST, + env_vars=env_vars, + health_check_fn=_health_check, + health_check_retries=60, + health_check_sleep=1.0, + ) + + def on_platform_ready(self): + """Configure Vault after it's running and LocalStack is ready.""" + try: + self._configure_vault() + except Exception as e: + LOG.error("Failed to configure Vault: %s", e) + raise + + url = f"http://vault.{constants.LOCALHOST_HOSTNAME}:{config.get_edge_port_http()}" + LOG.info("Vault extension ready: %s", url) + LOG.info("Root token: %s", self.root_token) + + def _configure_vault(self): + """Set up Vault with KV v2, Transit, and IAM auth.""" + container_host = get_addressable_container_host() + vault_addr = f"http://{container_host}:{self.vault_port}" + + # Wait a moment for Vault to be fully ready for API calls + time.sleep(1) + + client = hvac.Client(url=vault_addr, token=self.root_token) + + if not client.is_authenticated(): + raise RuntimeError("Failed to authenticate with Vault") + + LOG.info("Configuring Vault secrets engines and auth methods...") + + # KV v2 is enabled by default at secret/ in dev mode + # Just verify it's there + try: + secrets_engines = client.sys.list_mounted_secrets_engines() + if "secret/" in secrets_engines: + LOG.debug("KV v2 secrets engine already mounted at secret/") + except Exception as e: + LOG.warning("Could not verify secrets engines: %s", e) + + # Enable Transit secrets engine + self._enable_transit_engine(client) + + # Configure IAM auth method + self._configure_iam_auth(client) + + # Create default Lambda policy + self._create_default_policy(client) + + LOG.info("Vault configuration complete") + + def _enable_transit_engine(self, client: hvac.Client): + """Enable the Transit secrets engine for encryption-as-a-service.""" + try: + secrets_engines = client.sys.list_mounted_secrets_engines() + if "transit/" not in secrets_engines: + client.sys.enable_secrets_engine( + backend_type="transit", + path="transit", + ) + LOG.info("Enabled Transit secrets engine at transit/") + else: + LOG.debug("Transit secrets engine already mounted") + except Exception as e: + LOG.warning("Could not enable Transit engine: %s", e) + + def _configure_iam_auth(self, client: hvac.Client): + """Configure AWS IAM auth method to work with LocalStack.""" + try: + # Enable AWS auth method + auth_methods = client.sys.list_auth_methods() + if "aws/" not in auth_methods: + client.sys.enable_auth_method( + method_type="aws", + path="aws", + ) + LOG.info("Enabled AWS auth method at aws/") + + # Configure the AWS auth to use LocalStack's STS endpoint + # Use get_main_container_ip() to get LocalStack's actual container IP + # on the Docker network (e.g., 172.17.0.2), which is reachable from + # the Vault container. get_addressable_container_host() returns the + # Docker gateway IP (172.17.0.1), which may not be accessible. + localstack_ip = get_main_container_ip() + localstack_endpoint = f"http://{localstack_ip}:{config.get_edge_port_http()}" + + client.auth.aws.configure( + sts_endpoint=localstack_endpoint, + sts_region=os.getenv("AWS_DEFAULT_REGION", "us-east-1"), + iam_server_id_header_value="", + ) + LOG.info("Configured AWS auth to use LocalStack STS: %s", localstack_endpoint) + + # Create a wildcard IAM role that accepts any Lambda + # This role maps any IAM principal to the default-lambda-policy + self._create_wildcard_iam_role(client) + + except Exception as e: + LOG.warning("Could not configure IAM auth: %s", e) + + def _create_wildcard_iam_role(self, client: hvac.Client): + """Create an IAM role that accepts any AWS principal from LocalStack.""" + role_name = "default-lambda-role" + + try: + # Create a role that accepts any IAM role from LocalStack's account + # Note: bound_iam_principal_arn="*" doesn't work in Vault - we need a + # specific ARN pattern. LocalStack uses account 000000000000. + # We also MUST set resolve_aws_unique_ids=false since Vault can't + # resolve LocalStack IAM principals via AWS APIs. + client.auth.aws.create_role( + role=role_name, + auth_type="iam", + bound_iam_principal_arn=["arn:aws:iam::000000000000:role/*"], + token_policies=["default-lambda-policy"], + token_ttl="24h", + token_max_ttl="24h", + resolve_aws_unique_ids=False, # Critical for LocalStack + ) + LOG.info("Created IAM auth role: %s", role_name) + except hvac.exceptions.InvalidRequest as e: + if "already exists" in str(e).lower(): + LOG.debug("IAM role %s already exists", role_name) + else: + LOG.warning("Could not create IAM role %s: %s", role_name, e) + raise + + def _create_default_policy(self, client: hvac.Client): + """Create a default policy for Lambda functions.""" + policy_name = "default-lambda-policy" + policy_hcl = """ +# Default policy for Lambda functions using Vault +# Allows full access to secret/ and transit/ paths + +path "secret/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "secret/data/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "secret/metadata/*" { + capabilities = ["list", "read", "delete"] +} + +path "transit/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "transit/encrypt/*" { + capabilities = ["create", "update"] +} + +path "transit/decrypt/*" { + capabilities = ["create", "update"] +} +""" + try: + client.sys.create_or_update_policy( + name=policy_name, + policy=policy_hcl, + ) + LOG.info("Created policy: %s", policy_name) + except Exception as e: + LOG.warning("Could not create policy %s: %s", policy_name, e) diff --git a/vault/pyproject.toml b/vault/pyproject.toml new file mode 100644 index 00000000..2accbb4b --- /dev/null +++ b/vault/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools", "wheel", "plux>=1.3.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "localstack-vault" +version = "0.1.0" +description = "HashiCorp Vault Extension for LocalStack" +readme = {file = "README.md", content-type = "text/markdown; charset=UTF-8"} +requires-python = ">=3.9" +authors = [ + { name = "LocalStack Team"} +] +keywords = ["LocalStack", "Vault", "HashiCorp", "Secrets"] +classifiers = [] +dependencies = [ + "priority", + "localstack-extensions-utils", + "hvac>=2.0.0", + "jsonpatch", +] + +[project.urls] +Homepage = "https://github.com/localstack/localstack-extensions/tree/main/vault" + +[project.optional-dependencies] +dev = [ + "boto3", + "build", + "jsonpatch", + "localstack", + "pytest", + "requests", + "rolo", + "ruff", + "twisted", +] + +[project.entry-points."localstack.extensions"] +localstack-vault = "localstack_vault.extension:VaultExtension" diff --git a/vault/sample-app-extension/.gitignore b/vault/sample-app-extension/.gitignore new file mode 100644 index 00000000..a87311fd --- /dev/null +++ b/vault/sample-app-extension/.gitignore @@ -0,0 +1,2 @@ +# Build artifacts +*.zip diff --git a/vault/sample-app-extension/Makefile b/vault/sample-app-extension/Makefile new file mode 100644 index 00000000..b0e39d35 --- /dev/null +++ b/vault/sample-app-extension/Makefile @@ -0,0 +1,23 @@ +.PHONY: run setup-vault deploy test clean help + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' + +run: setup-vault deploy test ## Run the full sample (setup, deploy, test) + +setup-vault: ## Create test secrets in Vault + @echo "=== Setting up Vault secrets ===" + ./bin/setup-vault.sh + +deploy: ## Deploy Lambda with Vault extension layer + @echo "=== Deploying Lambda with Vault Lambda Extension ===" + ./bin/deploy-lambda.sh + +test: ## Test the Lambda function + @echo "=== Testing Lambda invocation ===" + ./bin/test-lambda.sh + +clean: ## Delete the Lambda function + @echo "=== Cleaning up ===" + awslocal lambda delete-function --function-name vault-test-function 2>/dev/null || true + @echo "Done" diff --git a/vault/sample-app-extension/README.md b/vault/sample-app-extension/README.md new file mode 100644 index 00000000..882c0773 --- /dev/null +++ b/vault/sample-app-extension/README.md @@ -0,0 +1,83 @@ +# Vault Lambda Extension Sample App + +A sample Lambda application demonstrating the [HashiCorp Vault Lambda Extension](https://github.com/hashicorp/vault-lambda-extension) with LocalStack. + +## Overview + +This sample app deploys a Lambda function that: + +- Uses the official Vault Lambda Extension layer +- Authenticates with Vault via IAM auth +- Reads secrets written by the extension to `/tmp/secrets/` + +## Prerequisites + +- [LocalStack](https://localstack.cloud/) with the Vault extension installed +- `awslocal` CLI +- Vault CLI + +## Setup + +### 1. Start LocalStack with Vault Extension + +```bash +EXTENSION_DEV_MODE=1 localstack start +``` + +### 2. Create Secrets in Vault + +```bash +make setup-vault +``` + +Or manually: + +```bash +export VAULT_ADDR=http://vault.localhost.localstack.cloud:4566 +export VAULT_TOKEN=root + +vault kv put secret/myapp/config api_key=secret123 db_password=hunter2 +``` + +### 3. Deploy the Lambda + +```bash +make deploy +``` + +### 4. Test the Function + +```bash +make test +``` + +## Environment Variables + +The Lambda function requires these environment variables for the extension: + +| Variable | Description | +|----------|-------------| +| `VAULT_ADDR` | Vault server address | +| `VAULT_AUTH_PROVIDER` | Auth method (`aws` for IAM auth) | +| `VAULT_AUTH_ROLE` | Vault role name for IAM auth | +| `VAULT_SECRET_PATH_*` | Secret path to fetch | +| `VAULT_SECRET_FILE_*` | File path where extension writes secrets | + +## Usage + +```bash +# Run everything at once +make run + +# Or step by step +make setup-vault +make deploy +make test +``` + +## How It Works + +1. Lambda is deployed with the Vault Lambda Extension layer +2. Extension authenticates to Vault via IAM auth +3. Extension fetches secrets and writes to `/tmp/secrets/` +4. Lambda handler reads secrets from the file diff --git a/vault/sample-app-extension/bin/deploy-lambda.sh b/vault/sample-app-extension/bin/deploy-lambda.sh new file mode 100755 index 00000000..fc28b97b --- /dev/null +++ b/vault/sample-app-extension/bin/deploy-lambda.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Deploy the sample Lambda function with Vault Lambda Extension layer +# +# NOTE: This approach currently has issues with LocalStack - the extension +# cannot read environment variables. See README.md for details. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SAMPLE_APP_DIR="$(dirname "$SCRIPT_DIR")" +LAMBDA_DIR="$SAMPLE_APP_DIR/lambda" + +FUNCTION_NAME="vault-test-function" +VAULT_ADDR="${VAULT_ADDR:-http://vault.localhost.localstack.cloud:4566}" +AWS_REGION="${AWS_DEFAULT_REGION:-us-east-1}" + +# Public Vault Lambda Extension layer ARN from HashiCorp +# See: https://developer.hashicorp.com/vault/docs/deploy/aws/lambda-extension +# Detect architecture and use appropriate layer +ARCH=$(uname -m) +if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then + VAULT_LAYER_VERSION="${VAULT_LAYER_VERSION:-12}" + VAULT_LAYER_ARN="arn:aws:lambda:${AWS_REGION}:634166935893:layer:vault-lambda-extension-arm64:${VAULT_LAYER_VERSION}" + LAMBDA_ARCH="arm64" +else + VAULT_LAYER_VERSION="${VAULT_LAYER_VERSION:-24}" + VAULT_LAYER_ARN="arn:aws:lambda:${AWS_REGION}:634166935893:layer:vault-lambda-extension:${VAULT_LAYER_VERSION}" + LAMBDA_ARCH="x86_64" +fi + +echo "=== Deploying Lambda function with Vault Lambda Extension ===" +echo "Detected architecture: $ARCH -> Lambda: $LAMBDA_ARCH" +echo "Using public layer: $VAULT_LAYER_ARN" +echo "" + +# Create deployment package +echo "Creating deployment package..." +cd "$LAMBDA_DIR" +zip -q -r /tmp/vault-lambda.zip handler.py + +# Create IAM role for Lambda +echo "Creating IAM role..." +awslocal iam create-role \ + --role-name vault-lambda-role \ + --assume-role-policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole" + }] + }' 2>/dev/null || echo "Role already exists" + +# Attach basic execution policy +awslocal iam attach-role-policy \ + --role-name vault-lambda-role \ + --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 2>/dev/null || true + +# Delete existing function if present +awslocal lambda delete-function --function-name "$FUNCTION_NAME" 2>/dev/null || true + +echo "Creating Lambda function: $FUNCTION_NAME" + +# Environment variables for the Vault Lambda Extension +# Required: VAULT_ADDR, VAULT_AUTH_PROVIDER, VAULT_AUTH_ROLE +# Optional: VAULT_SECRET_PATH_*, VAULT_SECRET_FILE_* +ENV_VARS=$(cat < /dev/null 2>&1; then + echo "Vault is ready!" + break + fi + if [ $i -eq 30 ]; then + echo "ERROR: Vault not ready after 30 seconds" + exit 1 + fi + sleep 1 +done + +# Create test secrets using the Vault HTTP API +echo "Creating test secrets at secret/data/myapp/config..." + +curl -sf -X POST \ + -H "X-Vault-Token: $VAULT_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "data": { + "api_key": "sk-test-12345", + "db_host": "localhost", + "db_password": "supersecret", + "feature_flags": "enable_new_ui,beta_features" + } + }' \ + "$VAULT_ADDR/v1/secret/data/myapp/config" + +echo "" +echo "=== Verifying secret was created ===" + +curl -sf \ + -H "X-Vault-Token: $VAULT_TOKEN" \ + "$VAULT_ADDR/v1/secret/data/myapp/config" | jq '.data.data | keys' + +echo "" +echo "=== Creating Vault IAM auth role for Lambda ===" + +# Create the IAM auth role that accepts Lambda functions from LocalStack +# This is needed because the extension's auto-creation may not work with older code +curl -sf -X POST \ + -H "X-Vault-Token: $VAULT_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "auth_type": "iam", + "bound_iam_principal_arn": ["arn:aws:iam::000000000000:role/vault-lambda-role"], + "resolve_aws_unique_ids": false, + "policies": ["default-lambda-policy"], + "token_ttl": "24h", + "token_max_ttl": "24h" + }' \ + "$VAULT_ADDR/v1/auth/aws/role/default-lambda-role" || echo "(role may already exist)" + +echo "IAM auth role 'default-lambda-role' configured" + +echo "" +echo "=== Vault setup complete ===" +echo "" +echo "Secret path: secret/data/myapp/config" +echo "Keys: api_key, db_host, db_password, feature_flags" +echo "IAM auth role: default-lambda-role" diff --git a/vault/sample-app-extension/bin/test-lambda.sh b/vault/sample-app-extension/bin/test-lambda.sh new file mode 100755 index 00000000..5960a6c8 --- /dev/null +++ b/vault/sample-app-extension/bin/test-lambda.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Test the Lambda function's Vault integration via the Lambda Extension + +set -euo pipefail + +FUNCTION_NAME="vault-test-function" +OUTPUT_FILE="/tmp/vault-lambda-output.json" + +echo "=== Testing Vault Lambda Extension ===" +echo "" + +awslocal lambda invoke \ + --function-name "$FUNCTION_NAME" \ + "$OUTPUT_FILE" \ + --cli-read-timeout 60 + +echo "" +cat "$OUTPUT_FILE" | jq '.' + +SUCCESS=$(cat "$OUTPUT_FILE" | jq -r '.success' 2>/dev/null) + +echo "" +if [ "$SUCCESS" = "true" ]; then + echo "✅ SUCCESS" +else + echo "❌ FAILED" + exit 1 +fi + +rm -f "$OUTPUT_FILE" diff --git a/vault/sample-app-extension/lambda/handler.py b/vault/sample-app-extension/lambda/handler.py new file mode 100644 index 00000000..d8c7adee --- /dev/null +++ b/vault/sample-app-extension/lambda/handler.py @@ -0,0 +1,74 @@ +""" +Sample Lambda function for Vault Lambda Extension integration. + +This function reads secrets written by the Vault Lambda Extension to /tmp/secrets/. +The extension handles authentication with Vault via IAM auth and fetches secrets +during Lambda initialization. + +For more details on the Vault Lambda Extension, see: +https://github.com/hashicorp/vault-lambda-extension +""" + +import json +import os + + +def handler(event, context): + """ + Lambda handler that reads secrets from the Vault Lambda Extension. + + The extension writes secrets to files specified by VAULT_SECRET_FILE_* env vars. + This handler reads from those files. + """ + + result = { + "message": "Vault Lambda Extension integration", + "success": False, + "secrets": {}, + "env": { + "VAULT_ADDR": os.environ.get("VAULT_ADDR", "not set"), + "VAULT_AUTH_PROVIDER": os.environ.get("VAULT_AUTH_PROVIDER", "not set"), + "VAULT_AUTH_ROLE": os.environ.get("VAULT_AUTH_ROLE", "not set"), + "VAULT_SECRET_PATH_MYAPP": os.environ.get("VAULT_SECRET_PATH_MYAPP", "not set"), + "VAULT_SECRET_FILE_MYAPP": os.environ.get("VAULT_SECRET_FILE_MYAPP", "not set"), + }, + } + + # Path where Vault Lambda Extension writes secrets + secret_file = os.environ.get("VAULT_SECRET_FILE_MYAPP", "/tmp/secrets/myapp") + + if not os.path.exists(secret_file): + result["message"] = f"Secret file not found: {secret_file}" + result["error"] = "EXTENSION_NOT_RUNNING" + result["hint"] = "Ensure the Vault Lambda Extension layer is attached and configured correctly" + return result + + try: + with open(secret_file, "r") as f: + secrets = json.load(f) + + # Extract the actual secret data from Vault's response format + # The extension writes the full Vault response, which has data nested + secret_data = secrets.get("data", {}).get("data", {}) + if not secret_data: + # Fallback if the structure is different + secret_data = secrets.get("data", secrets) + + result["success"] = True + result["secrets"] = { + "source": "vault-lambda-extension", + "keys_found": list(secret_data.keys()) if isinstance(secret_data, dict) else [], + "count": len(secret_data) if isinstance(secret_data, dict) else 0, + } + result["message"] = f"Successfully loaded {len(secret_data)} secrets via Vault Lambda Extension" + return result + + except json.JSONDecodeError as e: + result["message"] = f"Failed to parse secrets file: {e}" + result["error"] = "PARSE_ERROR" + return result + + except Exception as e: + result["message"] = f"Error reading secrets: {e}" + result["error"] = type(e).__name__ + return result diff --git a/vault/tests/__init__.py b/vault/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vault/tests/test_extension.py b/vault/tests/test_extension.py new file mode 100644 index 00000000..5b48b1a9 --- /dev/null +++ b/vault/tests/test_extension.py @@ -0,0 +1,260 @@ +import base64 + +import boto3 +import requests +from localstack.utils.strings import short_uid + + +# Vault connection details +VAULT_ADDR = "http://vault.localhost.localstack.cloud:4566" +VAULT_TOKEN = "root" +LOCALSTACK_ENDPOINT = "http://localhost:4566" + + +def vault_request(method, path, data=None, token=VAULT_TOKEN): + """Make a request to Vault API.""" + url = f"{VAULT_ADDR}/v1/{path}" + headers = {"X-Vault-Token": token} + if data: + headers["Content-Type"] = "application/json" + return requests.request(method, url, headers=headers, json=data) + return requests.request(method, url, headers=headers) + + +def test_vault_health(): + """Test that Vault is running and healthy.""" + response = requests.get(f"{VAULT_ADDR}/v1/sys/health") + assert response.status_code == 200 + + data = response.json() + assert data["initialized"] is True + assert data["sealed"] is False + + +def test_vault_auth_with_token(): + """Test authentication with root token.""" + response = vault_request("GET", "auth/token/lookup-self") + assert response.status_code == 200 + + data = response.json() + assert "data" in data + assert data["data"]["id"] == VAULT_TOKEN + + +def test_kv_secrets_engine(): + """Test KV v2 secrets engine operations.""" + secret_path = f"myapp/config-{short_uid()}" + + # Write a secret + secret_data = { + "data": { + "api_key": "test-api-key-123", + "db_password": "supersecret", + } + } + response = vault_request("POST", f"secret/data/{secret_path}", secret_data) + assert response.status_code == 200 + + # Read the secret back + response = vault_request("GET", f"secret/data/{secret_path}") + assert response.status_code == 200 + + data = response.json() + assert data["data"]["data"]["api_key"] == "test-api-key-123" + assert data["data"]["data"]["db_password"] == "supersecret" + + # Delete the secret + response = vault_request("DELETE", f"secret/data/{secret_path}") + assert response.status_code == 204 + + +def test_kv_list_secrets(): + """Test listing secrets in KV engine.""" + # Create a few secrets + for i in range(3): + secret_data = {"data": {"value": f"secret-{i}"}} + vault_request("POST", f"secret/data/list-test/item-{i}", secret_data) + + # List secrets + response = vault_request("LIST", "secret/metadata/list-test") + assert response.status_code == 200 + + data = response.json() + keys = data["data"]["keys"] + assert len(keys) == 3 + assert "item-0" in keys + assert "item-1" in keys + assert "item-2" in keys + + # Cleanup + for i in range(3): + vault_request("DELETE", f"secret/metadata/list-test/item-{i}") + + +def test_transit_engine(): + """Test Transit secrets engine for encryption.""" + key_name = f"test-key-{short_uid()}" + + # Create an encryption key + response = vault_request("POST", f"transit/keys/{key_name}") + assert response.status_code in (200, 204) # Vault may return either + + # Encrypt some data + plaintext = "Hello, Vault!" + plaintext_b64 = base64.b64encode(plaintext.encode()).decode() + + response = vault_request( + "POST", + f"transit/encrypt/{key_name}", + {"plaintext": plaintext_b64}, + ) + assert response.status_code == 200 + ciphertext = response.json()["data"]["ciphertext"] + assert ciphertext.startswith("vault:v1:") + + # Decrypt the data + response = vault_request( + "POST", + f"transit/decrypt/{key_name}", + {"ciphertext": ciphertext}, + ) + assert response.status_code == 200 + decrypted_b64 = response.json()["data"]["plaintext"] + decrypted = base64.b64decode(decrypted_b64).decode() + assert decrypted == plaintext + + # Delete the key + vault_request("POST", f"transit/keys/{key_name}/config", {"deletion_allowed": True}) + vault_request("DELETE", f"transit/keys/{key_name}") + + +def test_aws_auth_method_enabled(): + """Test that AWS auth method is enabled and configured.""" + response = vault_request("GET", "sys/auth") + assert response.status_code == 200 + + data = response.json() + assert "aws/" in data["data"] + assert data["data"]["aws/"]["type"] == "aws" + + +def test_default_lambda_role_exists(): + """Test that the default Lambda IAM auth role exists or can be created.""" + response = vault_request("GET", "auth/aws/role/default-lambda-role") + + # If role doesn't exist (e.g., after Terraform testing), create it + if response.status_code == 404: + role_config = { + "auth_type": "iam", + "bound_iam_principal_arn": ["arn:aws:iam::000000000000:role/*"], + "token_policies": ["default-lambda-policy"], + "resolve_aws_unique_ids": False, + } + create_response = vault_request( + "POST", "auth/aws/role/default-lambda-role", role_config + ) + assert create_response.status_code == 204 + response = vault_request("GET", "auth/aws/role/default-lambda-role") + + assert response.status_code == 200 + data = response.json() + assert data["data"]["auth_type"] == "iam" + assert data["data"]["resolve_aws_unique_ids"] is False + + +def test_default_lambda_policy_exists(): + """Test that the default Lambda policy exists with correct permissions.""" + response = vault_request("GET", "sys/policies/acl/default-lambda-policy") + assert response.status_code == 200 + + data = response.json() + policy = data["data"]["policy"] + assert "secret/*" in policy + assert "transit/*" in policy + + +def test_mixed_vault_and_aws_traffic(): + """ + Test that Vault HTTP traffic and AWS API traffic work together. + + This verifies that the Vault extension properly proxies Vault requests + while not interfering with regular AWS API requests. + """ + # Test Vault API + response = vault_request("GET", "sys/health") + assert response.status_code == 200 + assert response.json()["sealed"] is False + + # Test AWS S3 API + s3_client = boto3.client( + "s3", + endpoint_url=LOCALSTACK_ENDPOINT, + aws_access_key_id="test", + aws_secret_access_key="test", + region_name="us-east-1", + ) + + bucket_name = f"vault-test-bucket-{short_uid()}" + s3_client.create_bucket(Bucket=bucket_name) + + buckets = s3_client.list_buckets() + bucket_names = [b["Name"] for b in buckets["Buckets"]] + assert bucket_name in bucket_names + + # Cleanup + s3_client.delete_bucket(Bucket=bucket_name) + + # Test AWS STS API (used by Vault for IAM auth) + sts_client = boto3.client( + "sts", + endpoint_url=LOCALSTACK_ENDPOINT, + aws_access_key_id="test", + aws_secret_access_key="test", + region_name="us-east-1", + ) + + identity = sts_client.get_caller_identity() + assert "Account" in identity + assert "Arn" in identity + + # Verify Vault still works after AWS calls + response = vault_request("GET", "auth/token/lookup-self") + assert response.status_code == 200 + + +def test_create_custom_policy_and_role(): + """Test creating custom Vault policies and IAM auth roles.""" + policy_name = f"custom-policy-{short_uid()}" + role_name = f"custom-role-{short_uid()}" + + # Create a custom policy + policy_hcl = """ + path "secret/data/custom/*" { + capabilities = ["read"] + } + """ + response = vault_request( + "PUT", f"sys/policies/acl/{policy_name}", {"policy": policy_hcl} + ) + assert response.status_code == 204 + + # Create a custom IAM auth role + role_config = { + "auth_type": "iam", + "bound_iam_principal_arn": [ + "arn:aws:iam::000000000000:role/custom-lambda-role" + ], + "token_policies": [policy_name], + "resolve_aws_unique_ids": False, + } + response = vault_request("POST", f"auth/aws/role/{role_name}", role_config) + assert response.status_code == 204 + + # Verify the role was created + response = vault_request("GET", f"auth/aws/role/{role_name}") + assert response.status_code == 200 + assert policy_name in response.json()["data"]["token_policies"] + + # Cleanup + vault_request("DELETE", f"auth/aws/role/{role_name}") + vault_request("DELETE", f"sys/policies/acl/{policy_name}")