diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 0000000..de0a007 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,67 @@ +name: Deploy VitePress Docs + +on: + push: + branches: [main, feature/docs-site] + paths: + - 'docs/**' + - '.github/workflows/deploy-docs.yml' + workflow_dispatch: # Allow manual triggers + +# Sets permissions for GitHub Pages deployment +permissions: + contents: read + pages: write + id-token: write + +# Prevent concurrent deployments +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # For lastUpdated feature + + - name: Retrieve Node.js version + id: node-version + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + cache: 'npm' + cache-dependency-path: docs/package-lock.json + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Install dependencies + working-directory: docs + run: npm ci + + - name: Build with VitePress + working-directory: docs + run: npm run build + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/.vitepress/dist + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..57a09c3 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +node_modules +.vitepress/dist +.vitepress/cache diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts new file mode 100644 index 0000000..561ba29 --- /dev/null +++ b/docs/.vitepress/config.mts @@ -0,0 +1,60 @@ +import { defineConfig } from 'vitepress' + +// https://vitepress.dev/reference/site-config +export default defineConfig({ + title: "Trusted Server", + description: "Privacy-preserving edge computing for ad serving and synthetic ID generation", + themeConfig: { + // https://vitepress.dev/reference/default-theme-config + nav: [ + { text: 'Home', link: '/' }, + { text: 'Guide', link: '/guide/getting-started' }, + ], + + sidebar: [ + { + text: 'Introduction', + items: [ + { text: 'What is Trusted Server?', link: '/guide/what-is-trusted-server' }, + { text: 'Getting Started', link: '/guide/getting-started' } + ] + }, + { + text: 'Core Concepts', + items: [ + { text: 'Synthetic IDs', link: '/guide/synthetic-ids' }, + { text: 'GDPR Compliance', link: '/guide/gdpr-compliance' }, + { text: 'Ad Serving', link: '/guide/ad-serving' }, + { text: 'First-Party Proxy', link: '/guide/first-party-proxy' }, + { text: 'Creative Processing', link: '/guide/creative-processing' } + ] + }, + { + text: 'Security', + items: [ + { text: 'Request Signing', link: '/guide/request-signing' }, + { text: 'Key Rotation', link: '/guide/key-rotation' } + ] + }, + { + text: 'Development', + items: [ + { text: 'Architecture', link: '/guide/architecture' }, + { text: 'Configuration', link: '/guide/configuration' }, + { text: 'Configuration Reference', link: '/guide/configuration-reference' }, + { text: 'Testing', link: '/guide/testing' }, + { text: 'Integration Guide', link: '/guide/integration-guide' } + ] + } + ], + + socialLinks: [ + { icon: 'github', link: 'https://github.com/IABTechLab/trusted-server' } + ], + + footer: { + message: 'Released under the Apache License 2.0.', + copyright: 'Copyright © 2018-present IAB Technology Laboratory' + } + } +}) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..b25785a --- /dev/null +++ b/docs/README.md @@ -0,0 +1,163 @@ +# Trusted Server Documentation + +VitePress documentation site for Trusted Server. + +## Local Development + +### Prerequisites + +- Node.js 24.10.0 (or version specified in `.tool-versions`) +- npm + +### Setup + +```bash +# Install dependencies +npm install + +# Start dev server (available at localhost:5173) +npm run dev +``` + +### Build + +```bash +# Build for production +npm run build + +# Preview production build +npm run preview +``` + +## GitHub Pages Deployment + +The documentation is automatically deployed to GitHub Pages when changes are pushed to the `main` branch. + +### Setup GitHub Pages + +1. Go to repository **Settings** → **Pages** +2. Under **Source**, select **GitHub Actions** +3. The workflow in `.github/workflows/deploy-docs.yml` will automatically deploy on push to `main` + +### Custom Domain Setup + +1. **Update CNAME file**: Edit `docs/public/CNAME` with your domain: + ``` + docs.yourdomain.com + ``` + +2. **Configure DNS**: Add DNS records at your domain provider: + + **Option A - CNAME Record** (recommended for subdomains): + ``` + Type: CNAME + Name: docs + Value: iabtechlab.github.io + ``` + + **Option B - A Records** (for apex domains): + ``` + Type: A + Name: @ + Value: 185.199.108.153 + Value: 185.199.109.153 + Value: 185.199.110.153 + Value: 185.199.111.153 + ``` + +3. **Verify in GitHub**: + - Go to **Settings** → **Pages** + - Enter your custom domain + - Wait for DNS check to pass + - Enable "Enforce HTTPS" + +### Workflow Details + +**Trigger**: +- Push to `main` branch (only when `docs/**` changes) +- Manual trigger via Actions tab + +**Build Process**: +1. Checkout repository with full history (for `lastUpdated` feature) +2. Setup Node.js (version from `.tool-versions`) +3. Install dependencies (`npm ci`) +4. Build VitePress site (`npm run build`) +5. Upload build artifact +6. Deploy to GitHub Pages + +**Permissions Required**: +- `contents: read` - Read repository +- `pages: write` - Deploy to Pages +- `id-token: write` - OIDC token for deployment + +## Troubleshooting + +### Build Fails in GitHub Actions + +**Check**: +- Node.js version matches `.tool-versions` +- All dependencies in `package.json` are correct +- Build succeeds locally (`npm run build`) + +**View Logs**: +1. Go to **Actions** tab in GitHub +2. Click on failed workflow run +3. Review build logs + +### Custom Domain Not Working + +**Check**: +- DNS records propagated (use `dig docs.yourdomain.com`) +- CNAME file exists in `docs/public/CNAME` +- Custom domain verified in GitHub Pages settings +- HTTPS enforced (may take up to 24 hours) + +**DNS Verification**: +```bash +# Check CNAME record +dig docs.yourdomain.com CNAME + +# Check A records (for apex domain) +dig yourdomain.com A +``` + +### 404 Errors + +**Check**: +- VitePress `base` config (should not be set for custom domains) +- Links use correct paths (start with `/`) +- Build output in `docs/.vitepress/dist` is correct + +## Project Structure + +``` +docs/ +├── .vitepress/ +│ ├── config.mts # VitePress configuration +│ └── dist/ # Build output (gitignored) +├── guide/ # Documentation pages +│ ├── getting-started.md +│ ├── configuration.md +│ └── ... +├── public/ # Static assets +│ └── CNAME # Custom domain file +├── index.md # Homepage +├── package.json # Dependencies +└── README.md # This file +``` + +## Contributing + +When adding new documentation: + +1. Create `.md` files in `docs/guide/` +2. Update sidebar in `docs/.vitepress/config.mts` +3. Test locally with `npm run dev` +4. Build and verify with `npm run build && npm run preview` +5. Commit and push to trigger deployment + +## Links + +- **Production**: (Configure your custom domain) +- **GitHub Repo**: https://github.com/IABTechLab/trusted-server +- **VitePress Docs**: https://vitepress.dev diff --git a/docs/guide/ad-serving.md b/docs/guide/ad-serving.md new file mode 100644 index 0000000..6a05ed6 --- /dev/null +++ b/docs/guide/ad-serving.md @@ -0,0 +1,115 @@ +# Ad Serving + +Learn how Trusted Server handles privacy-compliant ad serving. + +## Overview + +Trusted Server provides edge-based ad serving with built-in GDPR compliance and real-time bidding support. + +## Supported Integrations + +### Equativ + +Primary ad server integration with support for: +- Direct ad requests +- Creative proxying +- Click tracking +- Impression tracking + +### Prebid + +Real-time bidding integration: +- Header bidding support +- Bid caching +- Timeout management +- Winner selection + +## Ad Request Flow + +1. Request validation +2. GDPR consent check +3. Synthetic ID generation (if consented) +4. Ad server request +5. Response processing +6. Creative delivery + +## Configuration + +Configure ad servers in `trusted-server.toml`: + +```toml +[ad_servers.equativ] +endpoint = "https://ad-server.example.com" +timeout_ms = 1000 +enabled = true + +[prebid] +timeout_ms = 1500 +cache_ttl = 300 +``` + +## Creative Handling + +### Proxy Mode + +Creatives can be proxied through Trusted Server for: +- Security scanning +- Content modification +- Click tracking injection +- GDPR compliance + +### Direct Mode + +Creatives served directly from ad server: +- Lower latency +- Reduced edge load +- Less control over content + +## Tracking + +### Impression Tracking + +```javascript +// Placeholder example +trustedServer.trackImpression({ + adId: 'ad-123', + syntheticId: 'synthetic-xyz', + consent: true +}); +``` + +### Click Tracking + +Click tracking with privacy preservation: +- No PII in URLs +- Synthetic ID only (with consent) +- Encrypted parameters + +## Performance + +### Edge Caching + +- Bid responses cached at edge +- Creative assets cached +- Configuration cached +- Reduced origin requests + +### Timeouts + +Configurable timeouts for: +- Ad server requests +- Prebid auctions +- Creative fetching + +## Best Practices + +1. Set appropriate timeouts for your use case +2. Enable caching for frequently requested ads +3. Monitor ad server response times +4. Use proxy mode for security-sensitive content +5. Implement fallback ads + +## Next Steps + +- Review [Architecture](/guide/architecture) +- Configure [Testing](/guide/testing) diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md new file mode 100644 index 0000000..822b302 --- /dev/null +++ b/docs/guide/architecture.md @@ -0,0 +1,122 @@ +# Architecture + +Understanding the architecture of Trusted Server. + +## High-Level Overview + +Trusted Server is built as a Rust-based edge computing application that runs on Fastly Compute platform. + +``` +┌─────────────┐ +│ Browser │ +└──────┬──────┘ + │ + ▼ +┌─────────────────────┐ +│ Trusted Server │ +│ (Fastly Edge) │ +│ ┌───────────────┐ │ +│ │ GDPR Check │ │ +│ │ Synthetic IDs │ │ +│ │ Ad Serving │ │ +│ └───────────────┘ │ +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Ad Servers │ +│ KV Stores │ +│ External APIs │ +└─────────────────────┘ +``` + +## Core Components + +### trusted-server-common + +Core library containing shared functionality: +- Synthetic ID generation +- Cookie handling +- HTTP abstractions +- GDPR consent management +- Ad server integrations + +### trusted-server-fastly + +Fastly-specific implementation: +- Main application entry point +- Fastly SDK integration +- Request/response handling +- KV store access + +## Design Patterns + +### RequestWrapper Trait + +Abstracts HTTP request handling to support different backends: + +```rust +// Placeholder example +pub trait RequestWrapper { + fn get_header(&self, name: &str) -> Option; + fn get_cookie(&self, name: &str) -> Option; + // ... +} +``` + +### Settings-Driven Configuration + +External configuration via `trusted-server.toml` allows deployment-time customization without code changes. + +### Privacy-First Design + +All tracking operations require explicit GDPR consent checks before execution. + +## Data Flow + +1. **Request Ingress** - Request arrives at Fastly edge +2. **Consent Validation** - GDPR consent checked +3. **ID Generation** - Synthetic ID generated (if consented) +4. **Ad Request** - Backend ad server called +5. **Response Processing** - Creative processed and modified +6. **Response Egress** - Response sent to browser + +## Storage + +### Fastly KV Store + +Used for: +- Counter storage +- Domain mappings +- Configuration cache +- Synthetic ID state + +### No User Data Persistence + +User data is not persisted in storage - only processed in-flight at the edge. + +## Performance Characteristics + +- **Low Latency** - Edge execution near users +- **High Throughput** - Parallel request processing +- **Global Distribution** - Fastly's global network +- **Caching** - Aggressive edge caching + +## Security + +- **HMAC-based IDs** - Cryptographically secure identifiers +- **No PII Storage** - Privacy by design +- **Request Signing** - Optional request authentication +- **Content Security** - Creative scanning and modification + +## WebAssembly Target + +Compiled to `wasm32-wasip1` for Fastly Compute: +- Sandboxed execution +- Fast cold starts +- Efficient resource usage + +## Next Steps + +- Learn about [Configuration](/guide/configuration) +- Set up [Testing](/guide/testing) diff --git a/docs/guide/configuration-reference.md b/docs/guide/configuration-reference.md new file mode 100644 index 0000000..c820ca7 --- /dev/null +++ b/docs/guide/configuration-reference.md @@ -0,0 +1,855 @@ +# Configuration Reference + +Complete reference for all configuration options in Trusted Server. + +## Overview + +Trusted Server uses a TOML-based configuration system with environment variable overrides. Configuration is loaded from: + +1. **`trusted-server.toml`** - Base configuration file +2. **Environment Variables** - Runtime overrides with `TRUSTED_SERVER__` prefix + +### Quick Example + +```toml +# trusted-server.toml +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "your-secure-secret-here" + +[synthetic] +counter_store = "counter_store" +opid_store = "opid_store" +secret_key = "your-hmac-secret-here" +template = "{{ client_ip }}:{{ user_agent }}" +``` + +## Environment Variable Overrides + +### Format + +``` +TRUSTED_SERVER__SECTION__SUBSECTION__FIELD +``` + +**Rules**: +- Prefix: `TRUSTED_SERVER` +- Separator: `__` (double underscore) +- Case: UPPERCASE +- Sections: Match TOML hierarchy + +### Examples + +**Simple Field**: +```bash +TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com +``` + +**Nested Field**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid.example/auction +``` + +**Array Field (JSON)**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS='["kargo","rubicon"]' +``` + +**Array Field (Indexed)**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__0=kargo +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__1=rubicon +``` + +**Array Field (Comma-Separated)**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus +``` + +## Publisher Configuration + +Core publisher settings for domain, origin, and proxy configuration. + +### `[publisher]` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `domain` | String | Yes | Publisher's domain name | +| `cookie_domain` | String | Yes | Domain for setting cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | + +**Example**: +```toml +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" # Includes subdomains +origin_url = "https://origin.publisher.com" +proxy_secret = "change-me-to-secure-random-value" +``` + +**Environment Override**: +```bash +TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com +TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN=.publisher.com +TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=your-secret-here +``` + +### Field Details + +#### `domain` + +**Purpose**: Primary domain for the publisher. + +**Usage**: +- Displayed in synthetic ID generation +- Used in template variables (`publisher_domain`) +- Part of request context + +**Format**: Hostname without protocol or path +- ✅ `publisher.com` +- ✅ `www.publisher.com` +- ❌ `https://publisher.com` +- ❌ `publisher.com/path` + +#### `cookie_domain` + +**Purpose**: Domain scope for synthetic ID cookies. + +**Usage**: +- Set on `synthetic_id` cookie +- Controls cookie sharing across subdomains + +**Format**: Domain with optional leading dot +- `.publisher.com` - Shares across all subdomains +- `publisher.com` - Exact domain only + +**Best Practice**: Use leading dot (`.publisher.com`) for subdomain sharing. + +#### `origin_url` + +**Purpose**: Backend origin server URL for publisher content. + +**Usage**: +- Fallback proxy target for non-integration requests +- HTML processing rewrites origin URLs to request host +- Base for relative URL resolution + +**Format**: Full URL with protocol +- ✅ `https://origin.publisher.com` +- ✅ `https://origin.publisher.com:8080` +- ✅ `http://192.168.1.1:9000` +- ❌ `origin.publisher.com` (missing protocol) + +**Port Handling**: Includes port if non-standard (not 80/443). + +#### `proxy_secret` + +**Purpose**: Secret key for HMAC-SHA256 signing of proxy URLs. + +**Security**: +- Keep confidential and secure +- Rotate periodically (90 days recommended) +- Use cryptographically random values (32+ bytes) +- Never commit to version control + +**Generation**: +```bash +# Generate secure random secret +openssl rand -base64 32 +``` + +**Usage**: +- Signs `/first-party/proxy` URLs +- Signs `/first-party/click` URLs +- Validates incoming proxy requests +- Prevents URL tampering + +::: danger Security Warning +Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations carefully and use graceful transition periods. +::: + +## Synthetic ID Configuration + +Settings for generating privacy-preserving synthetic identifiers. + +### `[synthetic]` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `counter_store` | String | Yes | Fastly KV store name for counters | +| `opid_store` | String | Yes | Fastly KV store name for publisher ID mappings | +| `secret_key` | String | Yes | HMAC secret for ID generation | +| `template` | String | Yes | Handlebars template for ID composition | + +**Example**: +```toml +[synthetic] +counter_store = "counter_store" +opid_store = "opid_store" +secret_key = "your-secure-hmac-secret" +template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}" +``` + +**Environment Override**: +```bash +TRUSTED_SERVER__SYNTHETIC__COUNTER_STORE=counter_store +TRUSTED_SERVER__SYNTHETIC__OPID_STORE=opid_store +TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=your-secret +TRUSTED_SERVER__SYNTHETIC__TEMPLATE="{{ client_ip }}:{{ user_agent }}" +``` + +### Field Details + +#### `counter_store` + +**Purpose**: Fastly KV store for synthetic ID counters. + +**Usage**: +- Stores incrementing counters per domain +- Ensures ID uniqueness +- Accessed via Fastly KV Store API + +**Setup**: +```bash +# Create KV store +fastly kv-store create --name=counter_store +``` + +**Data Format**: +```json +{ + "publisher.com": 12345, + "another.com": 67890 +} +``` + +#### `opid_store` + +**Purpose**: Fastly KV store for publisher-provided ID mappings. + +**Usage**: +- Maps publisher IDs to synthetic IDs +- Enables first-party ID integration +- Optional (used if publisher provides IDs) + +**Setup**: +```bash +# Create KV store +fastly kv-store create --name=opid_store +``` + +**Data Format**: +```json +{ + "publisher-id-123": "synthetic-abc", + "publisher-id-456": "synthetic-def" +} +``` + +#### `secret_key` + +**Purpose**: HMAC secret for deterministic ID generation. + +**Security**: +- Minimum 8 bytes (validation enforced) +- Cannot be `"secret-key"` (reserved/invalid) +- Rotate periodically for security +- Store securely (environment variable recommended) + +**Generation**: +```bash +# Generate secure random key +openssl rand -hex 32 +``` + +**Validation**: Application startup fails if: +- Empty string +- Exactly `"secret-key"` (default placeholder) +- Less than 1 character + +#### `template` + +**Purpose**: Handlebars template defining ID composition. + +**Available Variables**: + +| Variable | Description | Example | +|----------|-------------|---------| +| `client_ip` | Client IP address | `192.168.1.1` | +| `user_agent` | User-Agent header | `Mozilla/5.0...` | +| `first_party_id` | Publisher-provided ID | `user-123` | +| `auth_user_id` | Authenticated user ID | `auth-456` | +| `publisher_domain` | Publisher domain | `publisher.com` | +| `accept_language` | Accept-Language header | `en-US,en;q=0.9` | + +**Template Examples**: + +**Simple (IP + UA)**: +```toml +template = "{{ client_ip }}:{{ user_agent }}" +``` + +**With First-Party ID**: +```toml +template = "{{ first_party_id }}:{{ client_ip }}" +``` + +**Comprehensive**: +```toml +template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}:{{ auth_user_id }}:{{ publisher_domain }}:{{ accept_language }}" +``` + +**Validation**: Must be non-empty string. + +::: tip Template Design +Choose template variables based on your privacy and uniqueness requirements: +- **More variables** = More unique IDs, less privacy +- **Fewer variables** = More privacy, potential collisions +- **Include `first_party_id`** for publisher ID integration +::: + +## Response Headers + +Custom headers added to all responses. + +### `[response_headers]` + +**Purpose**: Add custom HTTP headers to every response. + +**Format**: Key-value pairs + +**Example**: +```toml +[response_headers] +X-Custom-Header = "custom value" +X-Publisher-ID = "pub-12345" +X-Environment = "production" +Cache-Control = "public, max-age=3600" +``` + +**Environment Override**: +```bash +TRUSTED_SERVER__RESPONSE_HEADERS__X_CUSTOM_HEADER="custom value" +``` + +**Use Cases**: +- Custom tracking headers +- Cache control overrides +- Debugging identifiers +- CORS headers (if needed) + +::: warning Header Precedence +Custom headers may be overwritten by application logic. Standard headers (`Content-Type`, `Content-Length`) are controlled by the application. +::: + +## Request Signing + +Configuration for Ed25519 request signing and JWKS management. + +### `[request_signing]` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `enabled` | Boolean | No (default: false) | Enable request signing features | +| `config_store_id` | String | If enabled | Fastly Config Store ID for JWKS | +| `secret_store_id` | String | If enabled | Fastly Secret Store ID for private keys | + +**Example**: +```toml +[request_signing] +enabled = true +config_store_id = "01GXXX" # From Fastly dashboard +secret_store_id = "01GYYY" # From Fastly dashboard +``` + +**Environment Override**: +```bash +TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=true +TRUSTED_SERVER__REQUEST_SIGNING__CONFIG_STORE_ID=01GXXX +TRUSTED_SERVER__REQUEST_SIGNING__SECRET_STORE_ID=01GYYY +``` + +### Store Setup + +**Config Store** (for public keys): +```bash +# Create store +fastly config-store create --name=jwks_store + +# Get store ID +fastly config-store list +``` + +**Secret Store** (for private keys): +```bash +# Create store +fastly secret-store create --name=signing_keys + +# Get store ID +fastly secret-store list +``` + +**Link to Service** (`fastly.toml`): +```toml +[setup.config_stores.jwks_store] + +[setup.secret_stores.signing_keys] +``` + +See [Request Signing](/guide/request-signing) and [Key Rotation](/guide/key-rotation) for usage. + +## Basic Authentication Handlers + +Path-based HTTP Basic Authentication. + +### `[[handlers]]` + +**Purpose**: Protect specific paths with username/password authentication. + +**Format**: Array of handler objects + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `path` | String (Regex) | Yes | Regular expression matching paths | +| `username` | String | Yes | HTTP Basic Auth username | +| `password` | String | Yes | HTTP Basic Auth password | + +**Example**: +```toml +# Single handler +[[handlers]] +path = "^/admin" +username = "admin" +password = "secure-password" + +# Multiple handlers +[[handlers]] +path = "^/secure" +username = "user1" +password = "pass1" + +[[handlers]] +path = "^/api/private" +username = "api-user" +password = "api-pass" +``` + +**Environment Override**: +```bash +# Handler 0 +TRUSTED_SERVER__HANDLERS__0__PATH="^/admin" +TRUSTED_SERVER__HANDLERS__0__USERNAME="admin" +TRUSTED_SERVER__HANDLERS__0__PASSWORD="secure-password" + +# Handler 1 +TRUSTED_SERVER__HANDLERS__1__PATH="^/api/private" +TRUSTED_SERVER__HANDLERS__1__USERNAME="api-user" +TRUSTED_SERVER__HANDLERS__1__PASSWORD="api-pass" +``` + +### Path Patterns + +**Regex Syntax**: Standard Rust regex patterns + +**Examples**: + +```toml +# Exact path +path = "^/admin$" # Only /admin + +# Prefix match +path = "^/admin" # /admin, /admin/users, /admin/settings + +# Multiple paths +path = "^/(admin|secure|private)" + +# File extension +path = "\\.pdf$" # All PDF files + +# Complex pattern +path = "^/api/v[0-9]+/private" # /api/v1/private, /api/v2/private +``` + +**Validation**: Application startup fails if regex is invalid. + +### Security Considerations + +**Password Storage**: +- Stored in plain text in config +- Use environment variables in production +- Rotate passwords regularly +- Consider using Fastly Secret Store + +**Limitations**: +- HTTP Basic Auth (not OAuth/JWT) +- Single username/password per path +- No role-based access control +- No rate limiting (add at edge) + +::: warning Production Use +For production, store credentials in environment variables: +```bash +TRUSTED_SERVER__HANDLERS__0__PASSWORD=$(cat /run/secrets/admin_password) +``` +::: + +## URL Rewrite Configuration + +Control which domains are excluded from first-party rewriting. + +### `[rewrite]` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `exclude_domains` | Array[String] | No (default: []) | Domains to skip rewriting | + +**Example**: +```toml +[rewrite] +exclude_domains = [ + "*.cdn.trusted-partner.com", # Wildcard + "first-party.publisher.com", # Exact match + "localhost", # Development +] +``` + +**Environment Override**: +```bash +# JSON array +TRUSTED_SERVER__REWRITE__EXCLUDE_DOMAINS='["*.cdn.example.com","localhost"]' + +# Indexed +TRUSTED_SERVER__REWRITE__EXCLUDE_DOMAINS__0="*.cdn.example.com" +TRUSTED_SERVER__REWRITE__EXCLUDE_DOMAINS__1="localhost" + +# Comma-separated +TRUSTED_SERVER__REWRITE__EXCLUDE_DOMAINS="*.cdn.example.com,localhost" +``` + +### Pattern Matching + +**Wildcard Patterns** (`*`): +```toml +"*.cdn.example.com" +``` +Matches: +- ✅ `assets.cdn.example.com` +- ✅ `images.cdn.example.com` +- ✅ `cdn.example.com` (base domain) +- ❌ `cdn.example.com.evil.com` (different domain) + +**Exact Patterns** (no `*`): +```toml +"api.example.com" +``` +Matches: +- ✅ `api.example.com` +- ❌ `www.api.example.com` +- ❌ `api.example.com.evil.com` + +### Use Cases + +**Trusted Partners**: +```toml +exclude_domains = ["*.approved-cdn.com"] +``` + +**First-Party Resources**: +```toml +exclude_domains = ["assets.publisher.com", "static.publisher.com"] +``` + +**Development**: +```toml +exclude_domains = ["localhost", "127.0.0.1", "*.local"] +``` + +**Performance** (already first-party): +```toml +exclude_domains = ["*.publisher.com"] # Skip unnecessary proxying +``` + +See [Creative Processing](/guide/creative-processing#exclude-domains) for details. + +## Integration Configurations + +Settings for built-in integrations (Prebid, Next.js, Permutive, Testlight). + +### Common Fields + +All integrations support: + +| Field | Type | Description | +|-------|------|-------------| +| `enabled` | Boolean | Enable/disable integration (default: false) | + +### Prebid Integration + +**Section**: `[integrations.prebid]` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | Boolean | `false` | Enable Prebid integration | +| `server_url` | String | Required | Prebid Server endpoint URL | +| `timeout_ms` | Integer | `1000` | Request timeout in milliseconds | +| `bidders` | Array[String] | `[]` | List of enabled bidders | +| `auto_configure` | Boolean | `false` | Auto-inject Prebid.js shim | +| `debug` | Boolean | `false` | Enable debug logging | +| `script_handler` | String | Optional | Custom script endpoint path | + +**Example**: +```toml +[integrations.prebid] +enabled = true +server_url = "https://prebid-server.example/openrtb2/auction" +timeout_ms = 1200 +bidders = ["kargo", "rubicon", "appnexus", "openx"] +auto_configure = true +debug = false +``` + +**Environment Override**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=true +TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid.example/auction +TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1200 +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus +TRUSTED_SERVER__INTEGRATIONS__PREBID__AUTO_CONFIGURE=true +TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false +``` + +### Next.js Integration + +**Section**: `[integrations.nextjs]` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | Boolean | `false` | Enable Next.js integration | +| `rewrite_attributes` | Array[String] | `["href","link","url"]` | Attributes to rewrite | + +**Example**: +```toml +[integrations.nextjs] +enabled = true +rewrite_attributes = ["href", "link", "url", "src"] +``` + +**Environment Override**: +```bash +TRUSTED_SERVER__INTEGRATIONS__NEXTJS__ENABLED=true +TRUSTED_SERVER__INTEGRATIONS__NEXTJS__REWRITE_ATTRIBUTES=href,link,url,src +``` + +### Permutive Integration + +**Section**: `[integrations.permutive]` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | Boolean | `false` | Enable Permutive integration | +| `organization_id` | String | Required | Permutive organization ID | +| `workspace_id` | String | Required | Permutive workspace ID | +| `project_id` | String | Required | Permutive project ID | +| `api_endpoint` | String | `https://api.permutive.com` | Permutive API URL | +| `secure_signals_endpoint` | String | `https://secure-signals.permutive.app` | Secure signals URL | +| `cache_ttl_seconds` | Integer | `3600` | Cache TTL in seconds | +| `rewrite_sdk` | Boolean | `true` | Rewrite Permutive SDK references | + +**Example**: +```toml +[integrations.permutive] +enabled = true +organization_id = "org-12345" +workspace_id = "ws-67890" +project_id = "proj-abcde" +api_endpoint = "https://api.permutive.com" +secure_signals_endpoint = "https://secure-signals.permutive.app" +cache_ttl_seconds = 7200 +rewrite_sdk = true +``` + +### Testlight Integration + +**Section**: `[integrations.testlight]` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | Boolean | `false` | Enable Testlight integration | +| `endpoint` | String | Required | Testlight auction endpoint | +| `timeout_ms` | Integer | `1000` | Request timeout in milliseconds | +| `rewrite_scripts` | Boolean | `true` | Rewrite Testlight script references | + +**Example**: +```toml +[integrations.testlight] +enabled = true +endpoint = "https://testlight.example/openrtb2/auction" +timeout_ms = 1500 +rewrite_scripts = true +``` + +## Validation + +### Automatic Validation + +Configuration is validated at startup: + +**Publisher Validation**: +- All fields non-empty +- `origin_url` is valid URL + +**Synthetic Validation**: +- `secret_key` ≥ 1 character +- `secret_key` ≠ `"secret-key"` +- `template` non-empty + +**Handler Validation**: +- `path` is valid regex +- `username` non-empty +- `password` non-empty + +**Integration Validation**: +- Each integration implements `Validate` trait +- Custom rules per integration + +### Validation Errors + +**Startup Failure** if: +- Required fields missing +- Invalid data types +- Regex compilation fails +- Secret key is default value +- Integration config fails validation + +**Error Format**: +``` +Configuration error: Integration 'prebid' configuration failed validation: +server_url: must not be empty +``` + +## Best Practices + +### Configuration Management + +**Development**: +```toml +# trusted-server.dev.toml +[publisher] +domain = "localhost" +origin_url = "http://localhost:3000" +proxy_secret = "dev-secret" +``` + +**Staging**: +```bash +# .env.staging +TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://staging.publisher.com +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret_staging) +``` + +**Production**: +```bash +# All secrets from environment +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret) +TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=$(cat /run/secrets/synthetic_secret) +TRUSTED_SERVER__HANDLERS__0__PASSWORD=$(cat /run/secrets/admin_password) +``` + +### Secret Management + +**Do**: +✅ Use environment variables for secrets +✅ Rotate secrets periodically +✅ Generate cryptographically random values +✅ Store in secure secret management (Fastly Secret Store, Vault) +✅ Use different secrets per environment + +**Don't**: +❌ Commit secrets to version control +❌ Use default/placeholder values +❌ Share secrets across environments +❌ Log secret values +❌ Expose in error messages + +### File Organization + +**Recommended Structure**: +``` +trusted-server.toml # Base config +trusted-server.dev.toml # Development overrides +.env.development # Dev environment vars +.env.staging # Staging environment vars +.env.production # Production environment vars (not in git) +.env.example # Example template (in git) +``` + +**.gitignore**: +``` +.env.production +.env.staging +.env.local +*.secret +``` + +## Troubleshooting + +### Common Issues + +**"Failed to build configuration"**: +- Check TOML syntax (trailing commas, quotes) +- Verify all required fields present +- Check environment variable format + +**"Secret key is not valid"**: +- Cannot use `"secret-key"` (placeholder) +- Must be non-empty +- Change to secure random value + +**"Invalid regex"**: +- Handler `path` must be valid regex +- Test pattern: `echo "^/admin" | grep -E "^/admin"` +- Escape special characters: `\.`, `\$`, etc. + +**"Integration configuration could not be parsed"**: +- Check JSON syntax in env vars +- Verify indexed arrays (0, 1, 2...) +- Check field names match exactly + +**Environment Variables Not Applied**: +- Verify prefix: `TRUSTED_SERVER__` +- Check separator: `__` (double underscore) +- Confirm variable is exported: `echo $VARIABLE_NAME` +- Try explicit string: `VARIABLE='value'` not `VARIABLE=value` + +### Debug Configuration + +**Print Loaded Config** (test only): +```rust +use trusted_server_common::settings::Settings; + +let settings = Settings::new()?; +println!("{:#?}", settings); +``` + +**Check Environment**: +```bash +# List all TRUSTED_SERVER variables +env | grep TRUSTED_SERVER +``` + +**Validate TOML**: +```bash +# Use any TOML validator +cat trusted-server.toml | npx toml-cli validate +``` + +## Next Steps + +- Learn about [First-Party Proxy](/guide/first-party-proxy) for URL proxying +- Set up [Request Signing](/guide/request-signing) for secure API calls +- Configure [Creative Processing](/guide/creative-processing) rewrites +- Explore [Integration Guide](/guide/integration-guide) for custom integrations diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md new file mode 100644 index 0000000..962e440 --- /dev/null +++ b/docs/guide/configuration.md @@ -0,0 +1,505 @@ +# Configuration + +Learn how to configure Trusted Server for your deployment. + +## Overview + +Trusted Server uses a flexible configuration system based on: + +1. **TOML Files** - `trusted-server.toml` for base configuration +2. **Environment Variables** - Runtime overrides with `TRUSTED_SERVER__` prefix +3. **Fastly Stores** - KV/Config/Secret stores for runtime data + +::: tip Complete Reference +See [Configuration Reference](/guide/configuration-reference) for detailed documentation of all configuration options. +::: + +## Quick Start + +### Basic Configuration + +Create `trusted-server.toml` in your project root: + +```toml +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "your-secure-secret-here" + +[synthetic] +counter_store = "counter_store" +opid_store = "opid_store" +secret_key = "your-hmac-secret" +template = "{{ client_ip }}:{{ user_agent }}" +``` + +### Environment Overrides + +Override any setting with environment variables: + +```bash +# Publisher settings +export TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com +export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com + +# Synthetic ID settings +export TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=your-secret +export TRUSTED_SERVER__SYNTHETIC__TEMPLATE="{{ client_ip }}:{{ user_agent }}" +``` + +## Configuration Files + +### `trusted-server.toml` + +Main application configuration file. + +**Location**: Project root directory + +**Format**: TOML (Tom's Obvious, Minimal Language) + +**Sections**: +- `[publisher]` - Publisher domain and origin settings +- `[synthetic]` - Synthetic ID generation +- `[request_signing]` - Request signing and JWKS +- `[response_headers]` - Custom response headers +- `[rewrite]` - URL rewriting rules +- `[[handlers]]` - Basic auth handlers +- `[integrations.*]` - Integration configs (Prebid, Next.js, etc.) + +**Example**: +```toml +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "change-me-to-secure-value" + +[synthetic] +counter_store = "counter_store" +opid_store = "opid_store" +secret_key = "your-hmac-secret-key" +template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}" + +[response_headers] +X-Publisher-ID = "pub-12345" +X-Environment = "production" + +[request_signing] +enabled = false +config_store_id = "" +secret_store_id = "" + +[integrations.prebid] +enabled = true +server_url = "https://prebid-server.com/openrtb2/auction" +timeout_ms = 1200 +bidders = ["kargo", "rubicon", "appnexus"] +auto_configure = false +``` + +### `fastly.toml` + +Fastly Compute service configuration. + +**Purpose**: Build settings, local development, store links + +**Example**: +```toml +manifest_version = 2 +name = "trusted-server" +description = "Privacy-preserving ad serving" +authors = ["Your Team"] +language = "rust" + +[local_server] + [local_server.kv_stores.counter_store] + file = "test-data/counter_store.json" + + [local_server.kv_stores.opid_store] + file = "test-data/opid_store.json" + +[setup] + [setup.config_stores.jwks_store] + [setup.secret_stores.signing_keys] +``` + +### `.env.*` Files + +Environment-specific variable files. + +**`.env.dev`** - Local development: +```bash +TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=http://localhost:3000 +TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=dev-secret +LOG_LEVEL=debug +``` + +**`.env.staging`** - Staging environment: +```bash +TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://staging.publisher.com +TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=$(cat /run/secrets/synthetic_key_staging) +``` + +**`.env.production`** - Production (secrets from secure store): +```bash +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret) +TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=$(cat /run/secrets/synthetic_secret) +TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=true +``` + +## Environment Variables + +### Format + +``` +TRUSTED_SERVER__SECTION__FIELD=value +``` + +**Rules**: +- Prefix: `TRUSTED_SERVER` +- Separator: `__` (double underscore) +- Case: UPPERCASE +- Nested: Use additional `__` for each level + +### Examples + +**Simple Field**: +```bash +TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com +``` + +**Array (JSON)**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS='["kargo","rubicon"]' +``` + +**Array (Indexed)**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__0=kargo +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__1=rubicon +``` + +**Array (Comma-Separated)**: +```bash +TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus +``` + +## Key Configuration Sections + +### Publisher Settings + +Core settings for your publisher domain and origin. + +```toml +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "secure-random-secret" +``` + +**Key Fields**: +- `domain` - Your publisher domain +- `cookie_domain` - Domain for synthetic ID cookies (use `.domain.com` for subdomains) +- `origin_url` - Backend origin server URL +- `proxy_secret` - Secret for signing proxy URLs (HMAC-SHA256) + +::: warning Security +Generate `proxy_secret` with cryptographically random values: +```bash +openssl rand -base64 32 +``` +::: + +### Synthetic IDs + +Configure privacy-preserving ID generation. + +```toml +[synthetic] +counter_store = "counter_store" +opid_store = "opid_store" +secret_key = "your-hmac-secret" +template = "{{ client_ip }}:{{ user_agent }}:{{ first_party_id }}" +``` + +**Template Variables**: +- `client_ip` - Client IP address +- `user_agent` - User-Agent header +- `first_party_id` - Publisher-provided ID +- `auth_user_id` - Authenticated user ID +- `publisher_domain` - Publisher domain +- `accept_language` - Accept-Language header + +See [Synthetic IDs](/guide/synthetic-ids) for details. + +### Request Signing + +Enable Ed25519 request signing and JWKS management. + +```toml +[request_signing] +enabled = true +config_store_id = "01GXXX" # From Fastly dashboard +secret_store_id = "01GYYY" # From Fastly dashboard +``` + +**Setup**: +1. Create Fastly Config Store for JWKS +2. Create Fastly Secret Store for private keys +3. Copy store IDs to configuration +4. Enable request signing + +See [Request Signing](/guide/request-signing) and [Key Rotation](/guide/key-rotation) for setup. + +### Integrations + +Configure built-in integrations. + +**Prebid**: +```toml +[integrations.prebid] +enabled = true +server_url = "https://prebid-server.com/openrtb2/auction" +timeout_ms = 1200 +bidders = ["kargo", "rubicon", "appnexus"] +auto_configure = false +``` + +**Next.js**: +```toml +[integrations.nextjs] +enabled = true +rewrite_attributes = ["href", "link", "url"] +``` + +**Permutive**: +```toml +[integrations.permutive] +enabled = true +organization_id = "org-12345" +workspace_id = "ws-67890" +project_id = "proj-abcde" +``` + +See [Integration Guide](/guide/integration-guide) for custom integrations. + +## Fastly Store Setup + +### KV Stores + +Create stores for synthetic ID data: + +```bash +# Create counter store +fastly kv-store create --name=counter_store + +# Create publisher ID mapping store +fastly kv-store create --name=opid_store +``` + +**Link to Service** (`fastly.toml`): +```toml +[local_server.kv_stores.counter_store] + file = "test-data/counter_store.json" + +[local_server.kv_stores.opid_store] + file = "test-data/opid_store.json" +``` + +### Config Stores + +For JWKS public keys: + +```bash +# Create config store +fastly config-store create --name=jwks_store + +# Get store ID +fastly config-store list +``` + +### Secret Stores + +For private signing keys: + +```bash +# Create secret store +fastly secret-store create --name=signing_keys + +# Get store ID +fastly secret-store list +``` + +## Validation + +### Automatic Validation + +Configuration is validated at application startup: + +**Checks**: +- Required fields present +- Data types correct +- Regex patterns valid +- Secret keys not default values +- Integration configs valid + +**Failure Behavior**: Application exits with error message. + +### Manual Validation + +Validate before deployment: + +```bash +# Test TOML syntax +fastly compute validate + +# Test with local server +fastly compute serve +``` + +## Secrets Management + +### Best Practices + +**Development**: +```bash +# Use simple secrets for local dev +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=dev-secret +``` + +**Staging/Production**: +```bash +# Load from secure sources +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret) +TRUSTED_SERVER__SYNTHETIC__SECRET_KEY=$(vault kv get -field=value secret/synthetic_key) +``` + +**Do**: +✅ Use environment variables for secrets +✅ Generate cryptographically random values +✅ Rotate secrets periodically +✅ Store in Fastly Secret Store or Vault +✅ Use different secrets per environment + +**Don't**: +❌ Commit secrets to version control +❌ Use default placeholder values +❌ Share secrets across environments +❌ Log secret values + +### `.gitignore` + +Protect secret files: + +``` +.env.production +.env.staging +.env.local +*.secret +trusted-server.production.toml +``` + +## Configuration Patterns + +### Multi-Environment Setup + +**Directory Structure**: +``` +project/ +├── trusted-server.toml # Base config +├── trusted-server.dev.toml # Development overrides +├── .env.development # Dev environment vars +├── .env.staging # Staging environment vars +├── .env.production # Production (not in git) +├── .env.example # Template (in git) +└── .gitignore +``` + +**Base Config** (`trusted-server.toml`): +```toml +# Shared across all environments +[synthetic] +template = "{{ client_ip }}:{{ user_agent }}" + +[integrations.prebid] +timeout_ms = 1200 +bidders = ["kargo", "rubicon"] +``` + +**Environment Overrides**: +```bash +# Development +export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=http://localhost:3000 + +# Staging +export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://staging.publisher.com + +# Production +export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com +``` + +### Dynamic Configuration + +Use environment variables for runtime changes: + +```bash +# Enable/disable features +TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=true + +# Adjust timeouts +TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1500 + +# Update endpoints +TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://new-prebid.com/auction +``` + +## Troubleshooting + +### Common Issues + +**"Failed to build configuration"**: +- Check TOML syntax (commas, quotes, brackets) +- Verify all required fields present +- Check environment variable format + +**"Secret key is not valid"**: +- Cannot use `"secret-key"` placeholder +- Must be non-empty +- Change to secure random value + +**"Invalid regex"**: +- Handler `path` must be valid regex +- Escape special characters: `\.`, `\$` +- Test with: `echo "pattern" | grep -E "pattern"` + +**Environment variables not applied**: +- Verify prefix: `TRUSTED_SERVER__` +- Check separator: `__` (double underscore) +- Confirm exported: `echo $VAR_NAME` + +### Debug Commands + +**Check environment**: +```bash +env | grep TRUSTED_SERVER +``` + +**Validate TOML**: +```bash +cat trusted-server.toml | npx toml-cli validate +``` + +**Test local server**: +```bash +fastly compute serve --verbose +``` + +## Next Steps + +- See [Configuration Reference](/guide/configuration-reference) for complete options +- Set up [Request Signing](/guide/request-signing) for secure API calls +- Configure [First-Party Proxy](/guide/first-party-proxy) for URL proxying +- Learn about [Integration Guide](/guide/integration-guide) for custom integrations +- Review [Testing](/guide/testing) for validation strategies diff --git a/docs/guide/creative-processing.md b/docs/guide/creative-processing.md new file mode 100644 index 0000000..f7bca18 --- /dev/null +++ b/docs/guide/creative-processing.md @@ -0,0 +1,817 @@ +# Creative Processing + +Learn how Trusted Server automatically rewrites ad creative HTML and CSS to route all resources through first-party domains. + +## Overview + +Creative processing transforms third-party ad creatives by rewriting URLs to go through your first-party domain. This provides: + +- **Privacy Control** - All resources load through your domain +- **First-Party Context** - Cookies and storage use your domain +- **Synthetic ID Integration** - Automatic ID forwarding to trackers +- **Security** - Validated, signed URLs prevent tampering +- **GDPR Compliance** - Controlled data sharing + +## How It Works + +``` +┌──────────────────────────────────────────────────────┐ +│ Original Creative HTML │ +│ │ +│ + + + +``` + +::: warning Nested Iframes +If the iframe content itself contains HTML, it will be processed recursively. Each level of nesting gets its own URL rewriting pass. +::: + +### Links + +**Elements**: `` + +**Attributes**: +- `href` - Link target (stylesheets, preload, prefetch) +- `imagesrcset` - Responsive images in link preload + +**Conditions**: Only rewritten when `rel` attribute matches: +- `stylesheet` +- `preload` +- `prefetch` + +**Example**: +```html + + + + + + + +``` + +### Anchors (Click Tracking) + +**Elements**: ``, `` + +**Attributes**: +- `href` - Link destination + +**Rewrite Mode**: Uses `/first-party/click` for direct redirects + +**Example**: +```html + +Buy Now + + +Buy Now +``` + +::: tip Click vs Proxy +Anchors (``) use `/first-party/click` for 302 redirects, avoiding content downloads. Other elements use `/first-party/proxy` to fetch and potentially rewrite content. +::: + +### SVG Elements + +**Elements**: SVG ``, `` + +**Attributes**: +- `href` - SVG 2.0 syntax +- `xlink:href` - SVG 1.1 legacy syntax + +**Example**: +```html + + + + + + + + + + + +``` + +### Inline Styles + +**Attributes**: `style` attribute on any element + +**Patterns**: Rewrites `url(...)` values in CSS + +**Example**: +```html + +
+ + +
+``` + +### Style Blocks + +**Elements**: ` + + + +``` + +## URL Detection Rules + +### Absolute URLs + +**Pattern**: Starts with `http://` or `https://` + +**Rewritten**: ✅ Yes + +**Examples**: +``` +✅ https://cdn.example.com/image.png +✅ http://tracker.example.com/pixel.gif +❌ /relative/path.jpg (relative) +❌ ../images/logo.png (relative) +``` + +### Protocol-Relative URLs + +**Pattern**: Starts with `//` + +**Rewritten**: ✅ Yes (normalized to `https://`) + +**Examples**: +``` +Original: //cdn.example.com/script.js +Normalized: https://cdn.example.com/script.js +Rewritten: /first-party/proxy?tsurl=https://cdn.example.com/script.js&tstoken=sig +``` + +### Relative URLs + +**Patterns**: +- Starts with `/` (absolute path) +- Starts with `./` or `../` (relative path) +- No scheme prefix (relative) + +**Rewritten**: ❌ No + +**Examples**: +``` +❌ /assets/image.png +❌ ./local.jpg +❌ ../parent/file.css +❌ image.png +``` + +::: info Why Skip Relative URLs? +Relative URLs already point to your domain (publisher origin). Rewriting them would create unnecessary proxy loops and break functionality. +::: + +### Non-Network Schemes + +**Skipped Schemes**: +- `data:` - Data URIs (inline content) +- `javascript:` - JavaScript execution +- `mailto:` - Email links +- `tel:` - Phone numbers +- `blob:` - Blob objects +- `about:` - Browser internal pages + +**Rewritten**: ❌ No + +**Examples**: +``` +❌ data:image/png;base64,iVBORw0KGgo... +❌ javascript:void(0) +❌ mailto:contact@example.com +❌ tel:+1234567890 +❌ blob:https://example.com/uuid +❌ about:blank +``` + +## Srcset Processing + +### Srcset Syntax + +Srcset attributes contain comma-separated candidates with optional descriptors: + +**Format**: `url descriptor, url descriptor, ...` + +**Descriptors**: +- `1x`, `2x`, `3x` - Pixel density +- `100w`, `200w` - Width in pixels + +### Parsing Rules + +**Robust Comma Handling**: +- Splits on commas with or without spaces +- Preserves `data:` URIs (doesn't split on internal commas) +- Handles irregular spacing + +**Example**: +```html + +srcset="url1.jpg 1x, url2.jpg 2x" +srcset="url1.jpg 1x,url2.jpg 2x" +srcset="url1.jpg 1x , url2.jpg 2x" +``` + +### Descriptor Preservation + +Descriptors are preserved exactly as written: + +```html + + + + + +``` + +### Mixed URL Types + +Srcset can mix absolute and relative URLs: + +```html + + + + + +``` + +## CSS URL Rewriting + +### url() Syntax Variations + +CSS `url()` values support multiple quote styles: + +```css +/* No quotes */ +url(https://example.com/image.png) + +/* Single quotes */ +url('https://example.com/image.png') + +/* Double quotes */ +url("https://example.com/image.png") + +/* With spaces */ +url( "https://example.com/image.png" ) +``` + +**All are rewritten correctly**, preserving the original quote style. + +### CSS Properties + +Common properties with `url()` values: + +```css +/* Background images */ +background: url(https://cdn.com/bg.jpg); +background-image: url(https://cdn.com/pattern.png); + +/* Borders */ +border-image: url(https://cdn.com/border.svg); + +/* List styles */ +list-style-image: url(https://cdn.com/bullet.png); + +/* Cursors */ +cursor: url(https://cdn.com/cursor.cur), pointer; + +/* Masks */ +mask-image: url(https://cdn.com/mask.svg); + +/* Filters */ +filter: url(https://cdn.com/filter.svg#blur); +``` + +**All `url()` occurrences are rewritten** regardless of property. + +### Multiple url() Values + +Properties can have multiple `url()` values: + +```css +/* Original */ +.element { + background: + url(https://cdn.com/top.png) top, + url(https://cdn.com/bottom.png) bottom; +} + +/* Rewritten */ +.element { + background: + url(/first-party/proxy?tsurl=https://cdn.com/top.png&tstoken=sig1) top, + url(/first-party/proxy?tsurl=https://cdn.com/bottom.png&tstoken=sig2) bottom; +} +``` + +### @import Rules + +CSS `@import` with URLs: + +```css +/* Original */ +@import url(https://fonts.googleapis.com/css?family=Roboto); + +/* Rewritten */ +@import url(/first-party/proxy?tsurl=https://fonts.googleapis.com/css?family=Roboto&tstoken=sig); +``` + +## Exclude Domains + +### Configuration + +Prevent specific domains from being rewritten: + +```toml +[rewrite] +exclude_domains = [ + "*.cdn.trusted-partner.com", # Wildcard pattern + "first-party.example.com", # Exact match + "localhost", # Development +] +``` + +### Pattern Matching + +**Wildcard Patterns**: `*` matches any subdomain + +``` +Pattern: *.cdn.example.com +Matches: + ✅ assets.cdn.example.com + ✅ images.cdn.example.com + ❌ cdn.example.com (no subdomain) + ❌ cdn.example.com.evil.com (different domain) +``` + +**Exact Patterns**: No `*` requires exact host match + +``` +Pattern: api.example.com +Matches: + ✅ api.example.com + ❌ www.api.example.com + ❌ api.example.com.evil.com +``` + +### Use Cases + +**Trusted Partners**: +```toml +exclude_domains = ["*.trusted-cdn.com"] +``` +Skip rewriting for partners already providing first-party scripts. + +**Development**: +```toml +exclude_domains = ["localhost", "127.0.0.1"] +``` +Avoid proxying local development servers. + +**Same-Origin Resources**: +```toml +exclude_domains = ["assets.publisher.com"] +``` +Skip resources already on your domain. + +## Integration Hooks + +### Attribute Rewriters + +Integrations can override attribute rewriting: + +**Example**: Next.js integration rewrites origin URLs + +```rust +impl IntegrationAttributeRewriter for NextJsIntegration { + fn rewrite(&self, attr_name: &str, attr_value: &str, ctx: &Context) + -> AttributeRewriteAction + { + if attr_name == "href" && attr_value.contains(&ctx.origin_host) { + let rewritten = attr_value.replace(&ctx.origin_host, &ctx.request_host); + return AttributeRewriteAction::replace(rewritten); + } + AttributeRewriteAction::keep() + } +} +``` + +**Actions**: +- `keep()` - Leave attribute unchanged +- `replace(value)` - Change attribute value +- `remove_element()` - Delete entire element + +### Script Rewriters + +Integrations can modify ` +``` + +**Timing**: Injected **once per HTML response** before any other scripts. + +### Integration Bundles + +Integrations can request additional bundles: + +```rust +IntegrationRegistration::builder("my_integration") + .with_asset("my_integration") // Requests tsjs-my_integration.min.js + .build() +``` + +**Result**: +```html + + + + + +``` + +### Bundle Types + +Available bundles (from `crates/js/lib/src/integrations/`): + +- `tsjs-core.min.js` - Core API (always included) +- `tsjs-ext.min.js` - Extensions (Prebid integration) +- `tsjs-creative.min.js` - Creative tracking utilities +- `tsjs-permutive.min.js` - Permutive integration +- `tsjs-testlight.min.js` - Testlight integration + +## Performance Optimization + +### Streaming Processing + +HTML is processed in **chunks** (default 8192 bytes): + +**Benefits**: +- Low memory footprint +- Handles large creatives +- Incremental output +- Fast first byte + +**Trade-offs**: +- Cannot access full DOM +- Element-by-element processing +- No look-ahead + +### Compression + +**Buffered Mode** (with rewriting): +``` +Origin Response (gzipped) + ↓ Decompress +Processing + ↓ Rewrite URLs +Response (uncompressed) + ↓ Fastly edge can re-compress +Client +``` + +**Streaming Mode** (no rewriting): +``` +Origin Response (gzipped) + ↓ Passthrough +Client (stays gzipped) +``` + +Use streaming for binary/large files to preserve compression. + +### Caching + +Rewritten creatives can be cached: + +**Cache Key Components**: +- Original creative URL +- Publisher domain +- Integration configuration + +**Headers to Set**: +```http +Cache-Control: public, max-age=3600 +Vary: Accept-Encoding +``` + +## Debugging + +### Logging + +Enable debug logging for rewrite operations: + +```rust +log::debug!("creative: rewriting {} -> {}", original_url, proxy_url); +log::debug!("creative: excluded domain {}", url); +log::debug!("creative: skipped non-network scheme {}", url); +``` + +### Testing Rewrites + +**Manual Testing**: +1. Save original creative HTML to file +2. Pass through rewrite function +3. Compare output + +```rust +let original = ""; +let rewritten = rewrite_creative_html(original, &settings); +assert!(rewritten.contains("/first-party/proxy")); +``` + +**Integration Tests**: +```rust +#[test] +fn test_image_src_rewrite() { + let html = r#""#; + let result = rewrite_creative_html(html, &test_settings()); + assert!(result.contains("/first-party/proxy?tsurl=")); + assert!(result.contains("&tstoken=")); +} +``` + +### Common Issues + +**Relative URLs Not Working**: +- Ensure origin response includes proper `` tag +- Or convert to absolute URLs before rewriting + +**Data URIs Being Rewritten**: +- Should be automatically skipped +- Check for malformed `data:` scheme + +**Srcset Parsing Errors**: +- Verify comma-separated format +- Check for unclosed quotes + +## Security Considerations + +### URL Validation + +All rewritten URLs are validated: + +1. **Scheme Check**: Only `http://` and `https://` +2. **Signature**: HMAC-SHA256 token required +3. **Expiration**: Optional `tsexp` timestamp +4. **Exclusion List**: Configurable domain blacklist + +### Content Security Policy + +Recommended CSP headers for rewritten creatives: + +```http +Content-Security-Policy: + default-src 'self'; + img-src 'self' /first-party/proxy; + script-src 'self' /static/tsjs; + style-src 'self' 'unsafe-inline'; + frame-src 'self' /first-party/proxy; +``` + +### Injection Prevention + +**Automatic Protection**: +- URLs are properly encoded +- No raw user input in rewrites +- Signature prevents tampering + +**Manual Checks**: +- Validate origin creative sources +- Sanitize user-generated content +- Monitor for suspicious patterns + +## Best Practices + +### Configuration + +✅ **Do**: +- Use strong `proxy_secret` (32+ bytes random) +- Exclude trusted first-party domains +- Set appropriate cache headers +- Test rewrites before production + +❌ **Don't**: +- Hardcode secrets in source +- Rewrite same-origin URLs unnecessarily +- Skip signature validation +- Disable TSJS injection without reason + +### Performance + +✅ **Do**: +- Use streaming for large/binary responses +- Enable compression at edge +- Cache rewritten creatives +- Monitor rewrite latency + +❌ **Don't**: +- Buffer entire response unnecessarily +- Rewrite on every request (cache!) +- Process non-HTML/CSS with rewriter +- Chain multiple rewrites + +### Monitoring + +Track these metrics: + +- **Rewrite operations** - Count of rewrites per request +- **Excluded domains** - Frequency of exclusions +- **Processing time** - Latency added by rewriting +- **Cache hit rate** - Effectiveness of caching +- **TSJS injection** - Verify library loads + +## Next Steps + +- Learn about [First-Party Proxy](/guide/first-party-proxy) for URL handling +- Review [Integration Guide](/guide/integration-guide) for custom rewriters +- Set up [Configuration](/guide/configuration) for your creatives +- Explore [Synthetic IDs](/guide/synthetic-ids) for identity management diff --git a/docs/guide/first-party-proxy.md b/docs/guide/first-party-proxy.md new file mode 100644 index 0000000..c3f1167 --- /dev/null +++ b/docs/guide/first-party-proxy.md @@ -0,0 +1,651 @@ +# First-Party Proxy + +Learn how Trusted Server proxies third-party assets through first-party domains to improve privacy, security, and ad performance. + +## Overview + +The First-Party Proxy system rewrites third-party URLs in ad creatives to route through your domain, providing: + +- **Privacy Protection** - No direct third-party cookies or tracking +- **Synthetic ID Forwarding** - Controlled identity propagation +- **Creative Rewrites** - Automatic HTML/CSS URL transformation +- **Click Tracking** - First-party click redirects +- **Content Security** - Validated, signed URLs prevent tampering + +## How It Works + +``` +┌──────────────────────────────────────────────────┐ +│ Ad Creative (Original) │ +│ │ +└──────────────────────────────────────────────────┘ + ↓ Rewrite +┌──────────────────────────────────────────────────┐ +│ Ad Creative (Rewritten) │ +│ │ +└──────────────────────────────────────────────────┘ + ↓ Browser Request +┌──────────────────────────────────────────────────┐ +│ Trusted Server │ +│ 1. Validate tstoken signature │ +│ 2. Append synthetic_id parameter │ +│ 3. Proxy to tracker.com │ +│ 4. Return response │ +└──────────────────────────────────────────────────┘ +``` + +## Core Endpoints + +### `/first-party/proxy` - Asset Proxy + +Proxies third-party assets with automatic HTML/CSS rewriting. + +**Request**: +``` +GET /first-party/proxy?tsurl=https://example.com/ad.html&tstoken=signature +``` + +**Query Parameters**: +| Parameter | Required | Description | +|-----------|----------|-------------| +| `tsurl` | Yes | Base URL of the resource (without query params) | +| `tstoken` | Yes | HMAC-SHA256 signature of the full URL | +| `tsexp` | No | Unix timestamp expiration (30s default from signing) | +| *(others)* | No | Original query parameters from target URL | + +**Behavior**: + +1. **Validates** the `tstoken` signature against reconstructed URL +2. **Appends** `synthetic_id` query parameter (if available) +3. **Proxies** request to target URL with forwarded headers: + - `User-Agent` + - `Accept` + - `Accept-Language` + - `Accept-Encoding` + - `Referer` + - `X-Forwarded-For` +4. **Processes** response based on content type: + - **HTML** (`text/html`) - Rewrites all URLs, returns `text/html` + - **CSS** (`text/css`) - Rewrites `url()` values, returns `text/css` + - **Images** - Detects pixels, sets `image/*` if missing + - **Other** - Passthrough without modification + +**Example**: + +Original URL: +``` +https://tracker.com/pixel.gif?campaign=123&uid=abc +``` + +Signed proxy URL: +``` +/first-party/proxy? + tsurl=https://tracker.com/pixel.gif& + campaign=123& + uid=abc& + tstoken=HmacSha256Signature +``` + +Final proxied request: +``` +https://tracker.com/pixel.gif?campaign=123&uid=abc&synthetic_id=xyz +``` + +### `/first-party/click` - Click Redirects + +Handles click tracking with first-party redirects. + +**Request**: +``` +GET /first-party/click?tsurl=https://advertiser.com/landing&tstoken=signature +``` + +**Query Parameters**: Same as `/first-party/proxy` + +**Behavior**: + +1. **Validates** the `tstoken` signature +2. **Appends** `synthetic_id` parameter to target URL +3. **Issues** 302 redirect to target (browser navigates directly) +4. **Logs** click metadata: + - Target URL base (`tsurl`) + - Whether parameters were present + - Full reconstructed URL + - Referer, User-Agent + - Synthetic ID (if available) + +**Example**: + +Click URL in creative: +```html + + Buy Now + +``` + +User clicks → Server responds: +``` +HTTP/1.1 302 Found +Location: https://advertiser.com/buy?product=widget&synthetic_id=xyz +``` + +::: tip Click vs Proxy +Use `/first-party/click` for navigational links (anchors) since it avoids downloading content. Use `/first-party/proxy` for embedded resources (images, scripts, iframes) that need to be rendered inline. +::: + +### `/first-party/sign` - URL Signing + +Generates signed proxy URLs for dynamic use cases. + +**Request (GET)**: +``` +GET /first-party/sign?url=https://example.com/resource.jpg +``` + +**Request (POST)**: +```json +POST /first-party/sign +{ + "url": "https://example.com/resource.jpg?param=value" +} +``` + +**Response**: +```json +{ + "href": "/first-party/proxy?tsurl=https://example.com/resource.jpg¶m=value&tstoken=signature&tsexp=1234567890", + "base": "https://example.com/resource.jpg" +} +``` + +**Response Fields**: +| Field | Description | +|-------|-------------| +| `href` | Complete signed proxy URL ready to use | +| `base` | Original base URL (without query parameters) | + +**Expiration**: +- Default: 30 seconds from signing +- `tsexp` parameter included in signed URL +- Validation fails after expiration + +**Example Usage**: + +```javascript +// Client-side JavaScript dynamically signing URLs +fetch('/first-party/sign?url=' + encodeURIComponent(imageUrl)) + .then(r => r.json()) + .then(data => { + img.src = data.href; // Use signed URL + }); +``` + +### `/first-party/proxy-rebuild` - URL Modification + +Modifies existing signed URLs by adding or removing parameters. + +**Request**: +``` +POST /first-party/proxy-rebuild?tsclick=encoded_click_url&add=key:value&del=key +``` + +**Query Parameters**: +| Parameter | Required | Description | +|-----------|----------|-------------| +| `tsclick` | Yes | Base64-encoded original click/proxy URL | +| `add` | No | Parameter to add (format: `key:value`) | +| `del` | No | Parameter key to remove | + +**Behavior**: + +1. **Decodes** the `tsclick` base64 URL +2. **Parses** existing parameters +3. **Adds** specified parameters (`add`) +4. **Removes** specified parameters (`del`) +5. **Re-signs** the modified URL +6. **Returns** new signed URL + +**Example**: + +Original click URL: +``` +/first-party/click?tsurl=https://example.com&product=A&tstoken=sig1 +``` + +Modify URL (add `variant=red`, remove `product`): +``` +POST /first-party/proxy-rebuild? + tsclick=L2ZpcnN0LXBhcnR5L2NsaWNrP3RzdXJsPWh0dHBzOi8vZXhhbXBsZS5jb20mcHJvZHVjdD1BJnRzdG9rZW49c2lnMQ==& + add=variant:red& + del=product +``` + +Response: +```json +{ + "href": "/first-party/click?tsurl=https://example.com&variant=red&tstoken=sig2" +} +``` + +::: warning Use Cases +This endpoint is designed for advanced scenarios like A/B testing where you need to modify URLs without re-signing from scratch. Most implementations won't need this. +::: + +## URL Signing & Validation + +### Signature Generation + +Signatures use HMAC-SHA256 with the publisher's `proxy_secret`: + +``` +1. Reconstruct full URL: tsurl + query params (sorted) +2. Encrypt with XChaCha20-Poly1305 (deterministic nonce) +3. Hash encrypted bytes with SHA-256 +4. Base64 URL-safe encode (no padding) +5. Result = tstoken +``` + +**Configuration**: +```toml +[publisher] +proxy_secret = "your-secret-key-here" # Must be secure random string +``` + +### Signature Validation + +On incoming requests: + +``` +1. Extract tsurl and all query params (except tstoken, tsexp) +2. Reconstruct full URL in same order +3. Compute expected tstoken using proxy_secret +4. Compare with provided tstoken (constant-time) +5. Check tsexp hasn't passed (if present) +6. Reject if mismatch or expired +``` + +::: danger Security +- Keep `proxy_secret` confidential and secure +- Rotate secrets periodically +- Never expose in client-side code +- Use strong random values (32+ bytes) +::: + +## Content Type Handling + +### HTML Rewriting + +**Triggers**: Response `Content-Type: text/html` + +**Process**: +1. Parse HTML with streaming processor +2. Rewrite absolute URLs to `/first-party/proxy` or `/first-party/click` +3. Preserve relative URLs unchanged +4. Sign all rewritten URLs with `tstoken` +5. Return as `text/html; charset=utf-8` + +**Rewritten Elements**: See [Creative Processing](/guide/creative-processing) for full list. + +### CSS Rewriting + +**Triggers**: Response `Content-Type: text/css` + +**Process**: +1. Parse CSS for `url(...)` values +2. Rewrite absolute URLs to `/first-party/proxy` +3. Sign URLs with `tstoken` +4. Return as `text/css; charset=utf-8` + +**Example**: +```css +/* Original */ +.banner { background: url(https://cdn.com/bg.jpg); } + +/* Rewritten */ +.banner { background: url(/first-party/proxy?tsurl=https://cdn.com/bg.jpg&tstoken=sig); } +``` + +### Image Handling + +**Triggers**: +- Response `Content-Type: image/*`, OR +- Request `Accept` header contains `image/` + +**Process**: +1. Set `Content-Type: image/*` if missing +2. Detect likely pixels with heuristics: + - `Content-Length` ≤ 256 bytes + - URL contains `/pixel`, `/p.gif`, `/1x1`, `/track` +3. Log pixel detection (no response alteration) +4. Passthrough image data + +**Logging**: +``` +proxy: likely pixel detected size=43 url=https://tracker.com/p.gif +``` + +### Passthrough (Other Types) + +**Triggers**: Any other `Content-Type` + +**Process**: +- Forward response without modification +- Preserve original `Content-Type` +- No HTML/CSS/URL rewriting +- Useful for: JSON, JavaScript, binary files, etc. + +## Redirect Handling + +The proxy automatically follows HTTP redirects: + +**Supported Status Codes**: +- `301` - Moved Permanently +- `302` - Found +- `303` - See Other (switches to GET) +- `307` - Temporary Redirect +- `308` - Permanent Redirect + +**Behavior**: +1. Follow up to **4 redirect hops** +2. Re-apply `synthetic_id` on each hop +3. Switch to `GET` after `303` response +4. Log when redirect limit reached +5. Preserve request headers across hops + +**Example Flow**: +``` +Request: /first-party/proxy?tsurl=https://short.link&tstoken=sig + → 302 to https://cdn.com/ad.html + → 200 with HTML content + → Rewrite HTML and return +``` + +## Synthetic ID Propagation + +### Automatic Forwarding + +When proxying, Trusted Server automatically appends the `synthetic_id` parameter: + +**Source Priority**: +1. `x-psid-ts` request header +2. `synthetic_id` cookie +3. Generate new ID if missing + +**Example**: +``` +Original request to proxy: + /first-party/proxy?tsurl=https://tracker.com/pixel.gif&tstoken=sig + Cookie: synthetic_id=user123 + +Proxied backend request: + https://tracker.com/pixel.gif?synthetic_id=user123 +``` + +### Redirect Propagation + +Synthetic IDs are re-applied on **every redirect hop**: + +``` +/first-party/proxy?tsurl=https://redirect1.com&tstoken=sig + → https://redirect1.com?synthetic_id=user123 + → 302 to https://redirect2.com + → https://redirect2.com?synthetic_id=user123 + → 302 to https://final.com + → https://final.com?synthetic_id=user123 + → 200 response +``` + +This ensures downstream trackers receive consistent IDs even through redirect chains. + +### Click ID Forwarding + +Click redirects also forward synthetic IDs: + +```html + +``` + +User clicks → redirect includes ID: +``` +302 Found +Location: https://advertiser.com?synthetic_id=user123 +``` + +::: tip Privacy Control +Synthetic IDs are only forwarded when: +1. User has given GDPR consent (if required) +2. ID exists in request (header/cookie) +3. Integration hasn't disabled forwarding (`forward_synthetic_id: false`) +::: + +## Configuration + +### Publisher Settings + +Configure proxy behavior in `trusted-server.toml`: + +```toml +[publisher] +domain = "publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "your-secure-random-secret" +cookie_domain = ".publisher.com" # For synthetic_id cookies +``` + +### URL Rewrite Exclusions + +Exclude specific domains from rewriting: + +```toml +[rewrite] +exclude_domains = [ + "*.cdn.trusted.com", # Wildcard pattern + "first-party.example.com" # Exact match +] +``` + +URLs matching these patterns will NOT be rewritten to `/first-party/proxy`. + +### Streaming vs Buffered + +Control whether responses are streamed or buffered: + +```rust +// Integration code example +ProxyRequestConfig::new(url) + .with_streaming() // Enable streaming (no HTML/CSS rewrites) +``` + +**Streaming** (buffered rewrites disabled): +- Preserves origin compression (gzip/brotli) +- Lower memory usage +- No HTML/CSS URL rewriting +- Best for: large files, images, videos + +**Buffered** (default): +- Enables HTML/CSS rewriting +- Decompresses response +- Higher memory usage +- Best for: ad creatives, landing pages + +## Performance Optimization + +### Compression + +**Buffered Mode**: +- Decompresses origin response +- Processes content +- Returns uncompressed (Fastly can re-compress) + +**Streaming Mode**: +- Preserves origin `Content-Encoding` +- No decompression/recompression overhead +- Passes through gzip/brotli/deflate + +### Caching + +Proxy responses respect origin cache headers: +- `Cache-Control` +- `Expires` +- `ETag` +- `Last-Modified` + +**Best Practices**: +```http +Cache-Control: public, max-age=3600 +Vary: Accept-Encoding +``` + +### Header Forwarding + +Only essential headers are forwarded to reduce overhead: + +**Forwarded Headers**: +- `User-Agent` - Client identification +- `Accept` - Content negotiation +- `Accept-Language` - Language preferences +- `Accept-Encoding` - Compression support +- `Referer` - Page context +- `X-Forwarded-For` - Client IP chain + +**Not Forwarded**: +- Authentication headers (unless explicitly added) +- Cookies (except synthetic ID appended as query param) +- Custom headers (unless added via `ProxyRequestConfig`) + +## Error Handling + +### Common Errors + +**Invalid Signature**: +``` +HTTP 403 Forbidden +tstoken validation failed: signature mismatch +``` + +**Solutions**: +- Verify `proxy_secret` matches signing configuration +- Check URL reconstruction includes all parameters in correct order +- Ensure no URL encoding issues + +**Expired URL**: +``` +HTTP 403 Forbidden +tstoken expired +``` + +**Solutions**: +- URLs signed with `/first-party/sign` expire in 30s +- Re-sign URL if needed +- Check client/server clock sync + +**Missing Parameters**: +``` +HTTP 400 Bad Request +Missing required parameter: tsurl +``` + +**Solutions**: +- Ensure `tsurl` parameter is present +- Include `tstoken` in request +- Verify URL encoding is correct + +### Redirect Limit + +When redirect limit (4 hops) is reached: + +``` +Log: proxy: redirect limit reached for url=https://... +``` + +Response: Returns last redirect response (302/307/308) without following. + +**Solutions**: +- Contact origin to reduce redirect chain +- Increase limit in code (modify `MAX_REDIRECTS` constant) + +## Security Considerations + +### Token Security + +**Do**: +✅ Use cryptographically strong `proxy_secret` (32+ bytes random) +✅ Rotate secrets periodically +✅ Validate expiration on all requests +✅ Use constant-time comparison for signatures + +**Don't**: +❌ Expose `proxy_secret` in client-side code +❌ Reuse secrets across environments +❌ Accept unsigned URLs +❌ Skip validation for "trusted" domains + +### URL Injection Prevention + +Signed URLs prevent injection attacks: + +``` +Attacker tries: + /first-party/proxy?tsurl=https://evil.com&tstoken=forged + +Trusted Server: + 1. Computes expected token for https://evil.com + 2. Compares with provided token + 3. Rejects if mismatch (403 Forbidden) +``` + +### Content Security + +**Automatic Protection**: +- HTML/CSS rewriting removes malicious URLs +- Data URIs are skipped (`data:`, `javascript:`, `blob:`) +- Protocol validation (only `http://` and `https://`) + +**Considerations**: +- Origin content is still served (validate trusted sources) +- Streaming mode bypasses HTML inspection +- Enable CSP headers for additional protection + +## Monitoring & Debugging + +### Logging + +Proxy requests emit detailed logs: + +``` +proxy: origin response status=200 ct=text/html cl=1234 accept=text/html url=https://... +proxy: likely pixel detected size=43 url=https://tracker.com/p.gif +click: tsurl=https://advertiser.com had_params=true target=... referer=... ua=... tsid=... +``` + +### Diagnostic Headers + +Add custom headers for debugging: + +```toml +[response_headers] +X-Proxy-Mode = "rewrite" +X-TS-Version = "1.0" +``` + +### Metrics to Track + +**Key Metrics**: +- Proxy request count (total) +- Signature validation failures (rate) +- Redirect hops (average/max) +- Response time (p50/p95/p99) +- Content type distribution +- Pixel detection rate + +## Next Steps + +- Learn about [Creative Processing](/guide/creative-processing) for HTML rewriting details +- Review [Synthetic IDs](/guide/synthetic-ids) for identity management +- Set up [Configuration](/guide/configuration) for your deployment +- Explore [Integration Guide](/guide/integration-guide) for custom integrations diff --git a/docs/guide/gdpr-compliance.md b/docs/guide/gdpr-compliance.md new file mode 100644 index 0000000..b2d564e --- /dev/null +++ b/docs/guide/gdpr-compliance.md @@ -0,0 +1,96 @@ +# GDPR Compliance + +Understanding GDPR compliance and consent management in Trusted Server. + +## Overview + +Trusted Server enforces GDPR compliance at the edge, ensuring all tracking and data collection activities require explicit user consent. + +## Consent Management + +### Consent Validation + +All requests are validated for proper GDPR consent before any tracking occurs: + +```rust +// Placeholder example +if !validate_gdpr_consent(&request) { + return reject_tracking(); +} +``` + +### Consent Sources + +Trusted Server supports multiple consent frameworks: + +- TCF (Transparency & Consent Framework) +- Custom consent signals +- First-party consent cookies + +## Implementation + +### Checking Consent + +```javascript +// Placeholder example +const hasConsent = await trustedServer.checkConsent({ + purposes: ['storage', 'personalization'], + vendors: [vendor_id] +}); +``` + +### Consent Storage + +Consent signals are: +- Validated on every request +- Not persisted without explicit consent +- Respected across all operations + +## Privacy Controls + +### User Rights + +Trusted Server supports: + +- Right to access +- Right to erasure +- Right to data portability +- Right to object + +### Data Minimization + +Only essential data is collected: +- Synthetic IDs (with consent) +- Minimal request metadata +- No PII storage + +## Configuration + +Configure GDPR settings in `trusted-server.toml`: + +```toml +[gdpr] +require_consent = true +tcf_version = "2.2" +default_action = "reject" +``` + +## Compliance Features + +- Consent checks before ID generation +- Automatic rejection without consent +- Audit logging for compliance +- Regional enforcement rules + +## Best Practices + +1. Always require explicit consent +2. Respect user withdrawal of consent +3. Document consent mechanisms +4. Regular compliance audits +5. Keep consent records + +## Next Steps + +- Configure [Ad Serving](/guide/ad-serving) +- Review [Architecture](/guide/architecture) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md new file mode 100644 index 0000000..a1bc57f --- /dev/null +++ b/docs/guide/getting-started.md @@ -0,0 +1,81 @@ +# Getting Started + +Get up and running with Trusted Server quickly. + +## Prerequisites + +Before you begin, ensure you have: + +- Rust 1.91.1 or later +- Fastly CLI installed +- A Fastly account +- Basic familiarity with WebAssembly + +## Installation + +### Clone the Repository + +```bash +git clone https://github.com/yourusername/trusted-server.git +cd trusted-server +``` + +### Install Fastly CLI + +```bash +# macOS +brew install fastly/tap/fastly + +# Or download from fastly.com/cli +``` + +### Install Viceroy (Test Runtime) + +```bash +cargo install viceroy +``` + +## Local Development + +### Build the Project + +```bash +cargo build +``` + +### Run Tests + +```bash +cargo test +``` + +### Start Local Server + +```bash +fastly compute serve +``` + +The server will be available at `http://localhost:7676`. + +## Configuration + +Edit `trusted-server.toml` to configure: + +- Ad server integrations +- KV store mappings +- Synthetic ID templates +- GDPR settings + +See [Configuration](/guide/configuration) for details. + +## Deploy to Fastly + +```bash +fastly compute publish +``` + +## Next Steps + +- Learn about [Synthetic IDs](/guide/synthetic-ids) +- Understand [GDPR Compliance](/guide/gdpr-compliance) +- Configure [Ad Serving](/guide/ad-serving) diff --git a/docs/guide/index.md b/docs/guide/index.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md new file mode 100644 index 0000000..74f46cf --- /dev/null +++ b/docs/guide/integration-guide.md @@ -0,0 +1,315 @@ +# Integration Guide + +This document explains how to integrate a new integration module with the Trusted Server runtime. The workflow mirrors the built-in `testlight` sample in `crates/common/src/integrations/testlight.rs`. + +## Architecture Overview + +| Component | Purpose | +| --- | --- | +| `crates/common/src/integrations/registry.rs` | Defines the `IntegrationProxy`, `IntegrationAttributeRewriter`, and `IntegrationScriptRewriter` traits and hosts the `IntegrationRegistry`, which drives proxy routing and HTML/text rewrites. | +| `Settings::integrations` (`crates/common/src/settings.rs`) | Free-form JSON blob keyed by integration ID. Use `IntegrationSettings::insert_config` to seed configs; each module deserializes and validates (`validator::Validate`) its own config and exposes an `enabled` flag so the core settings schema stays stable. | +| Fastly entrypoint (`crates/fastly/src/main.rs`) | Instantiates the registry once per request, routes `/integrations//…` requests to the appropriate proxy, and passes the registry to the publisher origin proxy so HTML rewriting remains integration-aware. | +| `html_processor.rs` | Applies first-party URL rewrites, injects the Trusted Server JS shim, and lets integrations override attribute values (for example to swap script URLs). | + +## Step-by-Step Integration + +### 1. Define Integration Configuration + +Add a `trusted-server.toml` block and any environment overrides under `TRUSTED_SERVER__INTEGRATIONS____*`. Configuration values are exposed to your module via `Settings::integration_config()`. + +```toml +[integrations.my_integration] +endpoint = "https://example.com/api" +timeout_ms = 1000 +rewrite_scripts = true +``` + +### 2. Create the Integration Module + +Add a module under `crates/common/src/integrations//mod.rs` (see `crates/common/src/integrations/testlight.rs` for reference) and expose it in `crates/common/src/integrations/mod.rs`. + +Key pieces: + +```rust +#[derive(Deserialize, Validate)] +struct MyIntegrationConfig { + #[serde(default = "default_enabled")] + enabled: bool, + // … +} + +impl IntegrationConfig for MyIntegrationConfig { + fn is_enabled(&self) -> bool { self.enabled } +} + +pub struct MyIntegration { + config: MyIntegrationConfig, +} + +pub fn build(settings: &Settings) -> Option> { + let config = settings + .integration_config::("my_integration") + .ok() + .flatten()?; + Some(Arc::new(MyIntegration { config })) +} + +// Tests or scaffolding code can seed configs without hand-writing JSON: +settings + .integrations + .insert_config( + "my_integration", + &serde_json::json!({ + "enabled": true, + "endpoint": "https://example.com/api" + }), + )?; +``` + +`Settings::integration_config::` automatically deserializes the raw JSON blob, runs [`validator`](https://docs.rs/validator/latest/validator/) on the type, and drops configs whose `is_enabled` returns `false`. Always derive/implement `Validate` for schema enforcement and implement `IntegrationConfig` (typically wrapping a `#[serde(default)] enabled` flag) so operators can toggle integrations without code changes. + +### 3. Return an IntegrationRegistration + +Each integration registers itself via a `register` function that returns an `IntegrationRegistration`. This object describes which HTTP proxies and HTML rewrites the integration exposes: + +```rust +pub fn register(settings: &Settings) -> Option { + let integration = build(settings)?; + Some( + IntegrationRegistration::builder("my_integration") + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration.clone()) + .with_script_rewriter(integration) + .with_asset("my_integration") + .build(), + ) +} +``` + +Any combination of the three vectors may be populated. Modules that only need HTML rewrites can skip the `proxies` field altogether, and vice versa. The registry automatically iterates over the static builder list in `crates/common/src/integrations/mod.rs`, so adding the new `register` function is enough to make the integration discoverable. + +### 4. Implement IntegrationProxy for Endpoints + +Implement the trait from `registry.rs` when your integration needs its own HTTP entrypoint: + +```rust +#[async_trait(?Send)] +impl IntegrationProxy for MyIntegration { + fn integration_name(&self) -> &'static str { + "my_integration" + } + + fn routes(&self) -> Vec { + vec![ + self.post("/auction"), + self.get("/status"), + ] + } + + async fn handle( + &self, + settings: &Settings, + req: Request, + ) -> Result> { + // Parse/generate synthetic IDs, forward upstream, and return the response. + } +} +``` + +::: tip Route Helpers +Use the provided helper methods to automatically namespace your routes under `/integrations/{integration_name()}/`. Available helpers: `get()`, `post()`, `put()`, `delete()`, and `patch()`. This lets you define routes with just their relative paths (e.g., `self.post("/auction")` becomes `"/integrations/my_integration/auction"`). +::: + +Routes are matched verbatim in `crates/fastly/src/main.rs`, so stick to stable paths and register whichever HTTP methods you need. **New integrations should namespace their routes under `/integrations/{INTEGRATION_NAME}/`** using the helper methods for consistency, but you can define routes manually if needed (e.g., for backwards compatibility). + +The shared context already injects Trusted Server logging, headers, and error handling; the handler only needs to deserialize the request, call the upstream endpoint, and stamp integration-specific headers. + +#### Proxying Upstream Requests + +Use the shared helper in `crates/common/src/proxy.rs` to forward requests so you automatically get the same header copying, redirect handling, HTML/CSS rewrite behavior, and synthetic ID handling the first-party proxy uses: + +```rust +use crate::proxy::{proxy_request, ProxyRequestConfig}; +use fastly::http::{header, HeaderValue}; + +let payload = serde_json::to_vec(&my_body)?; +let response = proxy_request( + settings, + req, + ProxyRequestConfig::new(&self.config.endpoint) + .with_body(payload) + .with_header(header::CONTENT_TYPE, HeaderValue::from_static("application/json")) + .with_streaming(), // stream passthrough; disable if you need HTML rewrites +) +.await?; +``` + +Set `forward_synthetic_id` to `false` if the upstream should not receive the caller's synthetic ID (`Testlight` does this), and disable `follow_redirects` if you need to surface redirects directly to the caller. + +**Streaming passthrough example:** + +```rust +let response = proxy_request( + settings, + req, + ProxyRequestConfig::new("https://example.com/pixel") + .with_streaming() // no HTML/CSS rewrites; preserves origin compression +); +``` + +::: info When to Use Streaming +Use streaming when the upstream response is binary or large and you do not need creative rewrites. Keep the default (non-streaming) mode when you want HTML/CSS content rewritten through the existing creative pipeline. +::: + +### 5. Implement HTML Rewrite Hooks (Optional) + +If the integration needs to rewrite script/link tags or inject HTML, implement `IntegrationAttributeRewriter` for attribute mutation and `IntegrationScriptRewriter` for inline `