diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5f69c379..fff4db59 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,5 @@ backend/ @HardMax71 # Frontend frontend/ @HardMax71 -# DevOps / Helm / CI -helm/ @HardMax71 +# DevOps / CI .github/ @HardMax71 diff --git a/.github/actions/e2e-ready/action.yml b/.github/actions/e2e-ready/action.yml index 15d3e042..62c7fa46 100644 --- a/.github/actions/e2e-ready/action.yml +++ b/.github/actions/e2e-ready/action.yml @@ -1,5 +1,5 @@ name: 'E2E Ready' -description: 'Finalize k3s, wait for infra, start compose stack, health-check, seed test users' +description: 'Finalize k3s, wait for infra, start compose stack, health-check' inputs: image-tag: @@ -75,6 +75,3 @@ runs: echo "Frontend ready" fi - - name: Seed test users - shell: bash - run: docker compose exec -T backend uv run python scripts/seed_users.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9778e7a3..3d1be900 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,13 +35,6 @@ jobs: - base - backend - frontend - - coordinator - - k8s-worker - - pod-monitor - - result-processor - - saga-orchestrator - - event-replay - - dlq-processor - cert-generator - zookeeper-certgen steps: @@ -116,13 +109,6 @@ jobs: crane copy "$REGISTRY/$PREFIX/base:$TAG" "$REGISTRY/$PREFIX/base:latest" crane copy "$REGISTRY/$PREFIX/backend:$TAG" "$REGISTRY/$PREFIX/backend:latest" crane copy "$REGISTRY/$PREFIX/frontend:$TAG" "$REGISTRY/$PREFIX/frontend:latest" - crane copy "$REGISTRY/$PREFIX/coordinator:$TAG" "$REGISTRY/$PREFIX/coordinator:latest" - crane copy "$REGISTRY/$PREFIX/k8s-worker:$TAG" "$REGISTRY/$PREFIX/k8s-worker:latest" - crane copy "$REGISTRY/$PREFIX/pod-monitor:$TAG" "$REGISTRY/$PREFIX/pod-monitor:latest" - crane copy "$REGISTRY/$PREFIX/result-processor:$TAG" "$REGISTRY/$PREFIX/result-processor:latest" - crane copy "$REGISTRY/$PREFIX/saga-orchestrator:$TAG" "$REGISTRY/$PREFIX/saga-orchestrator:latest" - crane copy "$REGISTRY/$PREFIX/event-replay:$TAG" "$REGISTRY/$PREFIX/event-replay:latest" - crane copy "$REGISTRY/$PREFIX/dlq-processor:$TAG" "$REGISTRY/$PREFIX/dlq-processor:latest" crane copy "$REGISTRY/$PREFIX/cert-generator:$TAG" "$REGISTRY/$PREFIX/cert-generator:latest" crane copy "$REGISTRY/$PREFIX/zookeeper-certgen:$TAG" "$REGISTRY/$PREFIX/zookeeper-certgen:latest" @@ -156,4 +142,4 @@ jobs: echo "| Frontend | \`docker pull $REGISTRY/$PREFIX/frontend:latest\` |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Security Scans" >> $GITHUB_STEP_SUMMARY - echo "All 12 images scanned with Trivy (CRITICAL + HIGH, unfixed ignored)." >> $GITHUB_STEP_SUMMARY + echo "All 5 images scanned with Trivy (CRITICAL + HIGH, unfixed ignored)." >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/stack-tests.yml b/.github/workflows/stack-tests.yml index ff5e9649..7ff6aac4 100644 --- a/.github/workflows/stack-tests.yml +++ b/.github/workflows/stack-tests.yml @@ -168,17 +168,10 @@ jobs: if: steps.base-cache.outputs.cache-hit != 'true' run: docker save integr8scode-base:latest | zstd -T0 -3 > /tmp/base-image.tar.zst - # ── Backend + workers (depend on local base image) ─────────────── - - name: Build backend and worker images + # ── Backend (depends on local base image) ─────────────── + - name: Build backend image run: | docker build -t integr8scode-backend:latest --build-context base=docker-image://integr8scode-base:latest -f ./backend/Dockerfile ./backend - docker build -t integr8scode-coordinator:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.coordinator ./backend - docker build -t integr8scode-k8s-worker:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.k8s_worker ./backend - docker build -t integr8scode-pod-monitor:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.pod_monitor ./backend - docker build -t integr8scode-result-processor:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.result_processor ./backend - docker build -t integr8scode-saga-orchestrator:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.saga_orchestrator ./backend - docker build -t integr8scode-event-replay:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.event_replay ./backend - docker build -t integr8scode-dlq-processor:latest --build-context base=docker-image://integr8scode-base:latest -f backend/workers/Dockerfile.dlq_processor ./backend # ── Utility images (GHA-cached, independent of base) ──────────── - name: Build cert-generator image @@ -232,23 +225,14 @@ jobs: # Tag all images for GHCR docker tag integr8scode-base:latest "$IMG/base:$TAG" docker tag integr8scode-backend:latest "$IMG/backend:$TAG" - docker tag integr8scode-coordinator:latest "$IMG/coordinator:$TAG" - docker tag integr8scode-k8s-worker:latest "$IMG/k8s-worker:$TAG" - docker tag integr8scode-pod-monitor:latest "$IMG/pod-monitor:$TAG" - docker tag integr8scode-result-processor:latest "$IMG/result-processor:$TAG" - docker tag integr8scode-saga-orchestrator:latest "$IMG/saga-orchestrator:$TAG" - docker tag integr8scode-event-replay:latest "$IMG/event-replay:$TAG" - docker tag integr8scode-dlq-processor:latest "$IMG/dlq-processor:$TAG" docker tag integr8scode-cert-generator:latest "$IMG/cert-generator:$TAG" docker tag integr8scode-zookeeper-certgen:latest "$IMG/zookeeper-certgen:$TAG" docker tag integr8scode-frontend:latest "$IMG/frontend-dev:$TAG" docker tag integr8scode-frontend-prod:latest "$IMG/frontend:$TAG" - # Push all 13 images in parallel, tracking each PID + # Push all 6 images in parallel, tracking each PID declare -A PIDS - for name in base backend coordinator k8s-worker pod-monitor \ - result-processor saga-orchestrator event-replay \ - dlq-processor cert-generator zookeeper-certgen \ + for name in base backend cert-generator zookeeper-certgen \ frontend-dev frontend; do docker push "$IMG/$name:$TAG" & PIDS[$name]=$! diff --git a/.gitignore b/.gitignore index c49ae4a0..ee489c0a 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,3 @@ frontend/coverage/ frontend/playwright-report/ frontend/test-results/ -# Helm -helm/*/charts/*.tgz -helm/*/Chart.lock diff --git a/backend/workers/Dockerfile.coordinator b/backend/workers/Dockerfile.coordinator deleted file mode 100644 index ae97091b..00000000 --- a/backend/workers/Dockerfile.coordinator +++ /dev/null @@ -1,8 +0,0 @@ -# Coordinator worker -FROM base - -# Copy application code -COPY . . - -# Run the coordinator service -CMD ["python", "-m", "workers.run_coordinator"] diff --git a/backend/workers/Dockerfile.dlq_processor b/backend/workers/Dockerfile.dlq_processor deleted file mode 100644 index 3c53a72f..00000000 --- a/backend/workers/Dockerfile.dlq_processor +++ /dev/null @@ -1,8 +0,0 @@ -# DLQ Processor worker -FROM base - -# Copy application code -COPY . . - -# Run DLQ processor -CMD ["python", "workers/dlq_processor.py"] diff --git a/backend/workers/Dockerfile.event_replay b/backend/workers/Dockerfile.event_replay deleted file mode 100644 index 948da191..00000000 --- a/backend/workers/Dockerfile.event_replay +++ /dev/null @@ -1,8 +0,0 @@ -# Event Replay worker -FROM base - -# Copy application code -COPY . . - -# Run event replay service -CMD ["python", "workers/run_event_replay.py"] diff --git a/backend/workers/Dockerfile.k8s_worker b/backend/workers/Dockerfile.k8s_worker deleted file mode 100644 index dc02131a..00000000 --- a/backend/workers/Dockerfile.k8s_worker +++ /dev/null @@ -1,8 +0,0 @@ -# Kubernetes Worker -FROM base - -# Copy application code -COPY . . - -# Run Kubernetes worker -CMD ["python", "workers/run_k8s_worker.py"] diff --git a/backend/workers/Dockerfile.pod_monitor b/backend/workers/Dockerfile.pod_monitor deleted file mode 100644 index 77fe71dd..00000000 --- a/backend/workers/Dockerfile.pod_monitor +++ /dev/null @@ -1,8 +0,0 @@ -# Pod Monitor worker -FROM base - -# Copy application code -COPY . . - -# Run pod monitor -CMD ["python", "workers/run_pod_monitor.py"] diff --git a/backend/workers/Dockerfile.result_processor b/backend/workers/Dockerfile.result_processor deleted file mode 100644 index 9c878459..00000000 --- a/backend/workers/Dockerfile.result_processor +++ /dev/null @@ -1,8 +0,0 @@ -# Result Processor worker -FROM base - -# Copy application code -COPY . . - -# Run result processor -CMD ["python", "workers/run_result_processor.py"] diff --git a/backend/workers/Dockerfile.saga_orchestrator b/backend/workers/Dockerfile.saga_orchestrator deleted file mode 100644 index f69a90c5..00000000 --- a/backend/workers/Dockerfile.saga_orchestrator +++ /dev/null @@ -1,8 +0,0 @@ -# Saga Orchestrator worker -FROM base - -# Copy application code -COPY . . - -# Run saga orchestrator -CMD ["python", "workers/run_saga_orchestrator.py"] diff --git a/backend/workers/dlq_processor.py b/backend/workers/run_dlq_processor.py similarity index 100% rename from backend/workers/dlq_processor.py rename to backend/workers/run_dlq_processor.py diff --git a/deploy.sh b/deploy.sh index 8f20d0cb..3b4f7925 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,15 +1,12 @@ #!/bin/bash # ============================================================================= -# Integr8sCode Unified Deployment Script +# Integr8sCode Deployment Script (Docker Compose) # ============================================================================= # # Usage: # ./deploy.sh dev # Start local development (docker-compose) # ./deploy.sh dev --build # Rebuild and start local development # ./deploy.sh down # Stop local development -# ./deploy.sh prod # Deploy to K8s (builds images locally) -# ./deploy.sh prod --prod # Deploy with production values (uses registry) -# ./deploy.sh prod --dry-run # Test Helm deployment without applying # ./deploy.sh check # Run local quality checks (lint, type, security) # ./deploy.sh test # Run full test suite locally # ./deploy.sh logs [service] # View logs (dev mode) @@ -31,11 +28,6 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' -# Helm configuration -NAMESPACE="integr8scode" -RELEASE_NAME="integr8scode" -CHART_PATH="./helm/integr8scode" - print_header() { echo -e "${BLUE}" echo "===========================================================================" @@ -66,7 +58,6 @@ show_help() { echo " --wait Wait for services to be healthy" echo " --timeout Health check timeout (default: 120)" echo " down Stop all services" - echo " prod [options] Deploy to Kubernetes with Helm" echo " check Run quality checks (ruff, mypy, bandit)" echo " test Run full test suite" echo " logs [service] View logs (defaults to all services)" @@ -75,12 +66,6 @@ show_help() { echo " types Generate TypeScript types for frontend from OpenAPI spec" echo " help Show this help message" echo "" - echo "Prod options:" - echo " --dry-run Validate templates without applying" - echo " --prod Use production values (ghcr.io images, no local build)" - echo " --local Force local build even with --prod values" - echo " --set key=value Override Helm values" - echo "" echo "Configuration:" echo " All settings come from backend/config.toml (single source of truth)" echo " For CI/tests: cp backend/config.test.toml backend/config.toml" @@ -89,7 +74,6 @@ show_help() { echo " ./deploy.sh dev # Start dev environment" echo " ./deploy.sh dev --build # Rebuild and start" echo " ./deploy.sh dev --wait # Start and wait for healthy" - echo " ./deploy.sh prod # Deploy with local images" echo " ./deploy.sh logs backend # View backend logs" } @@ -214,10 +198,6 @@ cmd_status() { echo "" echo "Docker Compose Services:" docker compose ps 2>/dev/null || echo " No docker-compose services running" - - echo "" - echo "Kubernetes Pods (if deployed):" - kubectl get pods -n "$NAMESPACE" 2>/dev/null || echo " No Kubernetes deployment found" } # ============================================================================= @@ -281,132 +261,6 @@ cmd_test() { exit $TEST_RESULT } -# ============================================================================= -# KUBERNETES DEPLOYMENT (Helm) -# ============================================================================= -build_and_import_images() { - print_info "Building Docker images..." - - docker build -t base:latest -f ./backend/Dockerfile.base ./backend - docker build -t integr8scode-backend:latest -f ./backend/Dockerfile ./backend - - if [[ -f ./frontend/Dockerfile.prod ]]; then - docker build -t integr8scode-frontend:latest -f ./frontend/Dockerfile.prod ./frontend - else - docker build -t integr8scode-frontend:latest -f ./frontend/Dockerfile ./frontend - fi - - print_success "Images built" - - if command -v k3s &> /dev/null; then - print_info "Importing images to K3s..." - docker save base:latest | sudo k3s ctr images import - - docker save integr8scode-backend:latest | sudo k3s ctr images import - - docker save integr8scode-frontend:latest | sudo k3s ctr images import - - print_success "Images imported to K3s" - else - print_warning "K3s not found - skipping image import" - fi -} - -cmd_prod() { - print_header "Deploying to Kubernetes" - - local DRY_RUN="" - local VALUES_FILE="values.yaml" - local EXTRA_ARGS="" - local USE_REGISTRY=false - local FORCE_LOCAL=false - - # Parse arguments - while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) - DRY_RUN="--dry-run" - print_warning "DRY RUN MODE - No changes will be applied" - ;; - --prod) - VALUES_FILE="values-prod.yaml" - USE_REGISTRY=true - print_info "Using production values (ghcr.io images)" - ;; - --local) - FORCE_LOCAL=true - print_info "Forcing local image build" - ;; - --set) - shift - EXTRA_ARGS="$EXTRA_ARGS --set $1" - ;; - *) - print_error "Unknown option: $1" - exit 1 - ;; - esac - shift - done - - deploy_helm "$VALUES_FILE" "$DRY_RUN" "$EXTRA_ARGS" "$USE_REGISTRY" "$FORCE_LOCAL" -} - -deploy_helm() { - local VALUES_FILE="$1" - local DRY_RUN="$2" - local EXTRA_ARGS="$3" - local USE_REGISTRY="$4" - local FORCE_LOCAL="$5" - - # Build images if: - # - Not dry-run AND - # - Not using registry OR force local build - if [[ -z "$DRY_RUN" ]]; then - if [[ "$USE_REGISTRY" != "true" ]] || [[ "$FORCE_LOCAL" == "true" ]]; then - build_and_import_images - else - print_info "Using pre-built images from ghcr.io (skipping local build)" - fi - fi - - print_info "Updating Helm dependencies..." - helm dependency update "$CHART_PATH" - - print_info "Creating namespace..." - if [[ -z "$DRY_RUN" ]]; then - kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - - fi - - print_info "Deploying with Helm..." - helm upgrade --install "$RELEASE_NAME" "$CHART_PATH" \ - --namespace "$NAMESPACE" \ - --values "$CHART_PATH/$VALUES_FILE" \ - --wait \ - --timeout 10m \ - $DRY_RUN \ - $EXTRA_ARGS - - if [[ -z "$DRY_RUN" ]]; then - echo "" - print_success "Deployment complete!" - echo "" - echo "Pods:" - kubectl get pods -n "$NAMESPACE" - echo "" - echo "Services:" - kubectl get services -n "$NAMESPACE" - echo "" - if [[ "$VALUES_FILE" == "values-prod.yaml" ]]; then - echo "Note: Passwords must be set via --set flags for production" - else - echo "Default credentials: user/user123, admin/admin123" - fi - echo "" - echo "Commands:" - echo " kubectl logs -n $NAMESPACE -l app.kubernetes.io/component=backend" - echo " kubectl port-forward -n $NAMESPACE svc/$RELEASE_NAME-backend 8443:443" - echo " helm uninstall $RELEASE_NAME -n $NAMESPACE" - fi -} - # ============================================================================= # OPENAPI SPEC GENERATION # ============================================================================= @@ -477,10 +331,6 @@ case "${1:-help}" in test) cmd_test ;; - prod) - shift - cmd_prod "$@" - ;; openapi) cmd_openapi "$2" ;; diff --git a/docker-compose.yaml b/docker-compose.yaml index 78ca0763..5c04c5a7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -342,15 +342,8 @@ services: # Kafka topic initialization kafka-init: image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: Dockerfile - additional_contexts: - base: service:base container_name: kafka-init depends_on: - base: - condition: service_completed_successfully kafka: condition: service_healthy environment: @@ -366,15 +359,8 @@ services: # Seed default users (runs once after mongo is ready) user-seed: image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: Dockerfile - additional_contexts: - base: service:base container_name: user-seed depends_on: - base: - condition: service_completed_successfully mongo: condition: service_healthy environment: @@ -390,16 +376,10 @@ services: # Event-driven workers coordinator: - image: ghcr.io/hardmax71/integr8scode/coordinator:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.coordinator - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: coordinator + command: ["python", "workers/run_coordinator.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully mongo: @@ -415,16 +395,10 @@ services: restart: unless-stopped k8s-worker: - image: ghcr.io/hardmax71/integr8scode/k8s-worker:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.k8s_worker - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: k8s-worker + command: ["python", "workers/run_k8s_worker.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully mongo: @@ -443,16 +417,10 @@ services: restart: unless-stopped pod-monitor: - image: ghcr.io/hardmax71/integr8scode/pod-monitor:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.pod_monitor - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: pod-monitor + command: ["python", "workers/run_pod_monitor.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully volumes: @@ -469,16 +437,10 @@ services: restart: unless-stopped result-processor: - image: ghcr.io/hardmax71/integr8scode/result-processor:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.result_processor - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: result-processor + command: ["python", "workers/run_result_processor.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully mongo: @@ -497,16 +459,10 @@ services: restart: unless-stopped saga-orchestrator: - image: ghcr.io/hardmax71/integr8scode/saga-orchestrator:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.saga_orchestrator - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: saga-orchestrator + command: ["python", "workers/run_saga_orchestrator.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully mongo: @@ -543,21 +499,14 @@ services: # Event replay service event-replay: - image: ghcr.io/hardmax71/integr8scode/event-replay:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.event_replay - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: event-replay + command: ["python", "workers/run_event_replay.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully mongo: condition: service_started - command: ["python", "workers/run_event_replay.py"] volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro @@ -570,21 +519,14 @@ services: # DLQ Processor Service dlq-processor: - image: ghcr.io/hardmax71/integr8scode/dlq-processor:${IMAGE_TAG:-latest} - build: - context: ./backend - dockerfile: workers/Dockerfile.dlq_processor - additional_contexts: - base: service:base + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} container_name: dlq-processor + command: ["python", "workers/run_dlq_processor.py"] depends_on: - base: - condition: service_completed_successfully kafka-init: condition: service_completed_successfully mongo: condition: service_started - command: ["python", "workers/dlq_processor.py"] volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro diff --git a/docs/components/dead-letter-queue.md b/docs/components/dead-letter-queue.md index 0cdc59cd..9881594a 100644 --- a/docs/components/dead-letter-queue.md +++ b/docs/components/dead-letter-queue.md @@ -36,7 +36,7 @@ This handler tracks retry counts per event. If an event fails 3 times, it gets s ### DLQ processor -The `dlq_processor` is a separate service that monitors the dead letter queue topic. It's responsible for the retry orchestration. When it sees a message in the DLQ, it applies topic-specific retry policies to determine when (or if) to retry sending that message back to its original topic. +The `run_dlq_processor` is a separate service that monitors the dead letter queue topic. It's responsible for the retry orchestration. When it sees a message in the DLQ, it applies topic-specific retry policies to determine when (or if) to retry sending that message back to its original topic. Different topics have different retry strategies configured: @@ -92,7 +92,7 @@ The system is designed to be resilient but not perfect. In catastrophic scenario | File | Purpose | |--------------------------------------------------------------------------------------------------------------------------|------------------------| -| [`dlq_processor.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/workers/dlq_processor.py) | DLQ processor worker | +| [`run_dlq_processor.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/workers/run_dlq_processor.py) | DLQ processor worker | | [`manager.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/manager.py) | DLQ management logic | | [`unified_producer.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/events/unified_producer.py) | `send_to_dlq()` method | | [`dlq.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/api/routes/dlq.py) | Admin API routes | diff --git a/docs/components/workers/coordinator.md b/docs/components/workers/coordinator.md index f6015cfd..1f5cc781 100644 --- a/docs/components/workers/coordinator.md +++ b/docs/components/workers/coordinator.md @@ -55,8 +55,8 @@ Executions are processed in priority order. Lower numeric values are processed f ```yaml coordinator: - build: - dockerfile: workers/Dockerfile.coordinator + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_coordinator.py"] ``` Usually runs as a single replica. Leader election via Redis is available if scaling is needed. diff --git a/docs/components/workers/dlq_processor.md b/docs/components/workers/dlq_processor.md index 146bc57f..c210bd3e 100644 --- a/docs/components/workers/dlq_processor.md +++ b/docs/components/workers/dlq_processor.md @@ -53,7 +53,7 @@ The DLQ can be monitored via the admin API: | File | Purpose | |-----------------------------------------------------------------------------------------------------------|----------------------| -| [`dlq_processor.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/workers/dlq_processor.py) | Entry point | +| [`run_dlq_processor.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/workers/run_dlq_processor.py) | Entry point | | [`manager.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/manager.py) | DLQ management logic | | [`dlq.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/api/routes/dlq.py) | Admin API routes | @@ -61,8 +61,8 @@ The DLQ can be monitored via the admin API: ```yaml dlq-processor: - build: - dockerfile: workers/Dockerfile.dlq_processor + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_dlq_processor.py"] ``` Usually runs as a single replica. The processor is designed to handle periodic retries, not real-time processing. diff --git a/docs/components/workers/event_replay.md b/docs/components/workers/event_replay.md index 1695ba3c..b18b92c0 100644 --- a/docs/components/workers/event_replay.md +++ b/docs/components/workers/event_replay.md @@ -46,8 +46,8 @@ filter criteria before running a real replay. ```yaml event-replay: - build: - dockerfile: workers/Dockerfile.event_replay + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_event_replay.py"] ``` -Typically runs on-demand with 0 replicas. Scale up when replay is needed, scale back down when complete. +Typically runs on-demand. Scale up when replay is needed, scale back down when complete. diff --git a/docs/components/workers/index.md b/docs/components/workers/index.md index 6c863c4d..64c59a5b 100644 --- a/docs/components/workers/index.md +++ b/docs/components/workers/index.md @@ -33,7 +33,7 @@ graph LR | [Pod Monitor](pod_monitor.md) | Watches pods, translates K8s events to domain events | `run_pod_monitor.py` | | [Result Processor](result_processor.md) | Persists execution results, cleans up resources | `run_result_processor.py` | | [Event Replay](event_replay.md) | Re-emits historical events for debugging | `run_event_replay.py` | -| [DLQ Processor](dlq_processor.md) | Retries failed messages from the dead letter queue | `dlq_processor.py` | +| [DLQ Processor](dlq_processor.md) | Retries failed messages from the dead letter queue | `run_dlq_processor.py` | All entry points live in [`backend/workers/`](https://github.com/HardMax71/Integr8sCode/tree/main/backend/workers). diff --git a/docs/components/workers/k8s_worker.md b/docs/components/workers/k8s_worker.md index 4513b40c..4c44124c 100644 --- a/docs/components/workers/k8s_worker.md +++ b/docs/components/workers/k8s_worker.md @@ -56,8 +56,8 @@ The worker refuses to run pods in the `default` namespace to ensure network poli ```yaml k8s-worker: - build: - dockerfile: workers/Dockerfile.k8s_worker + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_k8s_worker.py"] ``` Scale based on pod creation throughput. Each instance needs access to the Kubernetes API via kubeconfig. diff --git a/docs/components/workers/pod_monitor.md b/docs/components/workers/pod_monitor.md index de6fcb89..b73dcaf4 100644 --- a/docs/components/workers/pod_monitor.md +++ b/docs/components/workers/pod_monitor.md @@ -55,8 +55,8 @@ doing a periodic inventory check to make sure nothing slipped through. ```yaml pod-monitor: - build: - dockerfile: workers/Dockerfile.pod_monitor + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_pod_monitor.py"] ``` Runs as a single replica. The watch connection handles high throughput well. diff --git a/docs/components/workers/result_processor.md b/docs/components/workers/result_processor.md index dc24d130..b94fb005 100644 --- a/docs/components/workers/result_processor.md +++ b/docs/components/workers/result_processor.md @@ -39,8 +39,8 @@ module deletes ConfigMaps and pods that are no longer needed, keeping the Kubern ```yaml result-processor: - build: - dockerfile: workers/Dockerfile.result_processor + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_result_processor.py"] ``` Runs in the `result-processor-group` consumer group. Can scale horizontally if result throughput becomes a bottleneck. diff --git a/docs/components/workers/saga_orchestrator.md b/docs/components/workers/saga_orchestrator.md index afeb341d..da145ae8 100644 --- a/docs/components/workers/saga_orchestrator.md +++ b/docs/components/workers/saga_orchestrator.md @@ -62,8 +62,8 @@ stateDiagram-v2 ```yaml saga-orchestrator: - build: - dockerfile: workers/Dockerfile.saga_orchestrator + image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest} + command: ["python", "workers/run_saga_orchestrator.py"] ``` The orchestrator runs as a single replica since it's stateful. Event sourcing allows recovery after restarts. diff --git a/docs/getting-started.md b/docs/getting-started.md index 64769e04..9aa3dcd7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -112,6 +112,6 @@ docker compose down -v ## Next steps - [Architecture Overview](architecture/overview.md) - How the pieces fit together -- [Deployment](operations/deployment.md) - Production deployment with Helm +- [Deployment](operations/deployment.md) - Docker Compose deployment - [API Reference](reference/api-reference.md) - Full endpoint documentation - [Configuration Reference](reference/configuration.md) - TOML configuration options diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index 68edba42..53a63e22 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -32,7 +32,7 @@ graph LR end subgraph "Docker Scan & Promote" - Scan["Trivy Scan (12 images)"] + Scan["Trivy Scan (5 images)"] Promote["Promote SHA → latest"] Scan --> Promote end @@ -73,7 +73,7 @@ the backend and frontend E2E jobs, which both need k3s and the full docker compo | Action | File | Purpose | |-------------------------|----------------------------------------------|--------------------------------------------| | E2E Boot | `.github/actions/e2e-boot/action.yml` | GHCR login, background image pull + infra pre-warm, k3s install | -| E2E Ready | `.github/actions/e2e-ready/action.yml` | Finalize k3s, start compose stack, health checks, seed users | +| E2E Ready | `.github/actions/e2e-ready/action.yml` | Finalize k3s, start compose stack, health checks | The split is intentional. Frontend E2E needs to install Node.js and Playwright browsers _between_ boot and ready, overlapping that work with k3s installation to save wall-clock time. Backend E2E calls them back-to-back since it has @@ -81,7 +81,7 @@ no setup to overlap. ## Stack Tests (the main workflow) -This is the core testing workflow. It builds all 13 container images, pushes them to GHCR with immutable SHA-based +This is the core testing workflow. It builds all 6 container images, pushes them to GHCR with immutable SHA-based tags, then runs E2E tests on separate runners that pull images from the registry. ```mermaid @@ -92,7 +92,7 @@ graph TD end subgraph "Phase 2: Build" - C["Build & Push 13 Images to GHCR"] + C["Build & Push 6 Images to GHCR"] end subgraph "Phase 3: E2E (parallel runners)" @@ -120,37 +120,31 @@ the image build is skipped entirely. ### Phase 2: Build and push -All 13 images are built on a single runner and pushed to GHCR with an immutable `sha-<7chars>` tag: +All 6 images are built on a single runner and pushed to GHCR with an immutable `sha-<7chars>` tag: | Image | Source | |----------------------|---------------------------------------------| | `base` | `backend/Dockerfile.base` | | `backend` | `backend/Dockerfile` | -| `coordinator` | `backend/workers/Dockerfile.coordinator` | -| `k8s-worker` | `backend/workers/Dockerfile.k8s_worker` | -| `pod-monitor` | `backend/workers/Dockerfile.pod_monitor` | -| `result-processor` | `backend/workers/Dockerfile.result_processor` | -| `saga-orchestrator` | `backend/workers/Dockerfile.saga_orchestrator` | -| `event-replay` | `backend/workers/Dockerfile.event_replay` | -| `dlq-processor` | `backend/workers/Dockerfile.dlq_processor` | | `cert-generator` | `cert-generator/Dockerfile` | | `zookeeper-certgen` | `backend/zookeeper/Dockerfile.certgen` | | `frontend-dev` | `frontend/Dockerfile` | | `frontend` | `frontend/Dockerfile.prod` | -Of these 13 images, 12 are scanned by Trivy and promoted to `latest` in the +Workers reuse the `backend` image with different `command:` overrides in docker-compose, so no separate worker images +are needed. Of these 6 images, 5 are scanned by Trivy and promoted to `latest` in the [Docker Scan & Promote](#docker-scan--promote) workflow. The `frontend-dev` image is excluded — it's the Vite dev server build used only for E2E tests in CI and is never deployed to production. -The base image is cached separately as a zstd-compressed tarball since its dependencies rarely change. Worker images -depend on it via `--build-context base=docker-image://integr8scode-base:latest`. Utility and frontend images use GHA -layer caching. +The base image is cached separately as a zstd-compressed tarball since its dependencies rarely change. The backend +image depends on it via `--build-context base=docker-image://integr8scode-base:latest`. Utility and frontend images +use GHA layer caching. -All 13 images are pushed to GHCR in parallel, with each push tracked by PID so individual failures are reported: +All 6 images are pushed to GHCR in parallel, with each push tracked by PID so individual failures are reported: ```yaml declare -A PIDS -for name in base backend coordinator k8s-worker ...; do +for name in base backend cert-generator zookeeper-certgen ...; do docker push "$IMG/$name:$TAG" & PIDS[$name]=$! done @@ -194,7 +188,6 @@ This action finalizes the environment after boot tasks complete: `/tmp/infra-pull.exit`, failing fast if the background process had errors 5. **Start compose stack** with `docker compose up -d --no-build` 6. **Health checks** — waits for backend (`/api/v1/health/live`), and optionally frontend (`https://localhost:5001`) -7. **Seed test users** via `scripts/seed_users.py` #### Frontend E2E sharding @@ -236,7 +229,7 @@ sets it, and only after all tests pass. ```mermaid graph LR ST["Stack Tests
(main, success)"] -->|workflow_run trigger| Scan - Scan["Trivy Scan
(12 images in parallel)"] --> Promote["crane copy
sha-xxx → latest"] + Scan["Trivy Scan
(5 images in parallel)"] --> Promote["crane copy
sha-xxx → latest"] Promote --> Summary["Step Summary"] ``` @@ -247,7 +240,7 @@ Runs automatically when `Stack Tests` completes successfully on `main`. Can also ### Scan -Uses [Trivy](https://trivy.dev/) (pinned at `v0.68.2`) to scan all 12 deployed images in parallel via matrix strategy. +Uses [Trivy](https://trivy.dev/) (pinned at `v0.68.2`) to scan all 5 deployed images in parallel via matrix strategy. Scans for `CRITICAL` and `HIGH` severity vulnerabilities with unfixed issues ignored. Results are uploaded as SARIF files to GitHub's Security tab. @@ -376,7 +369,7 @@ Playwright browsers are cached by `package-lock.json` hash. On cache hit, only s ### Parallel image push -All 13 images are pushed to GHCR concurrently using background processes with PID tracking. Each push failure is +All 6 images are pushed to GHCR concurrently using background processes with PID tracking. Each push failure is reported individually via `::error::` annotations. ## Running locally diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 31ac8bd7..a158e456 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -1,42 +1,36 @@ # Deployment -Integr8sCode supports two deployment modes: local development using Docker Compose and production deployment to -Kubernetes using Helm. Both modes share the same container images and configuration patterns, so what works locally -translates directly to production. +Integr8sCode uses Docker Compose for deployment. All services — backend, frontend, workers, and infrastructure — +run as containers orchestrated by a single `docker-compose.yaml`. Workers reuse the backend image with different +`command:` overrides, so there's only one application image to build. Kubernetes is used only for executor pods +(running user code); workers just need a kubeconfig to talk to the K8s API. ## Architecture ```mermaid flowchart TB - Script[deploy.sh] --> Local & Prod + Script[deploy.sh] --> DC subgraph Images["Container Images"] Base[Dockerfile.base] --> Backend[Backend Image] - Base --> Workers[Worker Images] end - subgraph Local["Local Development"] - DC[docker-compose.yaml] --> Containers - end - - subgraph Prod["Kubernetes"] - Helm[Helm Chart] --> Pods + subgraph DC["Docker Compose"] + Compose[docker-compose.yaml] --> Containers end Images --> Containers - Images --> Pods ``` ## Deployment script -The unified [`deploy.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/deploy.sh) script handles both modes: +The [`deploy.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/deploy.sh) script wraps Docker Compose: ```bash --8<-- "deploy.sh:6:18" ``` -The script abstracts away the differences between environments. For local development it orchestrates Docker Compose, -while for production it builds images, imports them to the container runtime, and runs Helm with appropriate values. +The script wraps Docker Compose with convenience commands for building, starting, stopping, and running tests. ## Local development @@ -84,11 +78,10 @@ flowchart LR end subgraph Services["Service Images"] - S1[Backend] - S2[Workers] + S1[Backend + Workers] end - Base --> S1 & S2 + Base --> S1 ``` The base image installs all production dependencies: @@ -141,199 +134,33 @@ services define healthchecks in `docker-compose.yaml`: Services without explicit healthchecks (workers, Grafana, Kafdrop) are considered "started" when their container is running. The test suite doesn't require worker containers since tests instantiate worker classes directly. -## Kubernetes deployment - -Production deployment targets Kubernetes using a Helm chart that packages all manifests and configuration. The chart -lives in `helm/integr8scode/` and includes templates for every component of the stack. - -### Prerequisites - -Before deploying, ensure you have Helm 3.x installed and kubectl configured to talk to your cluster. If you're using -K3s, the deploy script handles image import automatically. For other distributions, you'll need to push images to a -registry and update the image references in your values file. - -### Chart structure - -The Helm chart organizes templates by function: - -| Directory | Contents | -|-----------------------------|-------------------------------------------| -| `templates/rbac/` | ServiceAccount, Role, RoleBinding | -| `templates/secrets/` | Kubeconfig and Kafka JAAS | -| `templates/configmaps/` | TOML configuration and environment | -| `templates/infrastructure/` | Zookeeper, Kafka, Jaeger | -| `templates/app/` | Backend and Frontend deployments | -| `templates/workers/` | All seven worker deployments | -| `templates/jobs/` | Kafka topic init and user seed | -| `charts/` | Bitnami sub-charts (Redis, MongoDB) | - -The chart uses Bitnami sub-charts for Redis and MongoDB. Kafka uses custom templates because Confluent images require -unsetting Kubernetes auto-generated environment variables like `KAFKA_PORT=tcp://...` that conflict with expected -numeric values. - -### Running a deployment - -The simplest deployment uses default values, which work for development and testing clusters. +## Monitoring -```bash -./deploy.sh prod -``` - -This builds the Docker images, imports them to K3s (if available), updates Helm dependencies to download the Redis and -MongoDB sub-charts, creates the namespace, and runs `helm upgrade --install`. The `--wait` flag ensures the command -blocks until all pods are ready. - -For production environments, pass additional flags to set secure passwords. - -```bash -./deploy.sh prod --prod \ - --set userSeed.defaultUserPassword=secure-user-pass \ - --set userSeed.adminUserPassword=secure-admin-pass \ - --set mongodb.auth.rootPassword=mongo-root-pass -``` - -The `--prod` flag tells the script to use `values-prod.yaml`, which increases replica counts, resource limits, and -enables MongoDB authentication. Without the password flags, the user seed job will fail since the production values -intentionally leave passwords empty to force explicit configuration. - -To validate templates without applying anything: - -```bash -./deploy.sh prod --dry-run -``` - -This renders the templates and prints what would be applied, useful for catching configuration errors before they hit -the cluster. - -### Configuration - -The `values.yaml` file contains all configurable options. Key sections: - -```yaml ---8<-- "helm/integr8scode/values.yaml:10:19" -``` - -Application settings are loaded from TOML files (`config.toml` + `secrets.toml`), not environment variables. The Helm chart mounts these as files into each pod. Service-specific overrides go in their respective sections. For example, to increase backend replicas and memory: - -```yaml -backend: - replicas: 3 - resources: - limits: - memory: "2Gi" -``` - -Worker configuration follows a similar pattern. Each worker has its own section under `workers` where you can enable or -disable it, set replicas, and override the default command. - -```yaml -workers: - k8sWorker: - enabled: true - replicas: 2 - dlqProcessor: - enabled: false # Disable if not needed -``` - -The infrastructure section controls Confluent platform components. You can adjust heap sizes, resource limits, and -enable or disable optional services like Jaeger. - -```yaml -infrastructure: - kafka: - heapOpts: "-Xms1G -Xmx1G" - resources: - limits: - memory: "2Gi" - jaeger: - enabled: false # Disable tracing in resource-constrained environments -``` - -### Post-install jobs - -Two Helm hooks run after the main deployment completes. The kafka-init job waits for Kafka to become -healthy, then creates all required topics using the `scripts/create_topics.py` module. Topics are created with the -prefix defined in settings (default `pref`) to avoid conflicts with Kubernetes-generated environment variables. - -The user-seed job waits for MongoDB, then runs `scripts/seed_users.py` to create the default and admin users. If users -already exist, the script updates their roles without creating duplicates, making it safe to run on upgrades. - -Both jobs have a hook-weight that controls ordering. Kafka init runs first (weight 5), followed by user seed (weight -10). The `before-hook-creation` delete policy ensures old jobs are cleaned up before new ones run, preventing conflicts -from previous releases. - -### Accessing deployed services - -After deployment, services are only accessible within the cluster by default. Use kubectl port-forward to access them -locally. - -```bash -kubectl port-forward -n integr8scode svc/integr8scode-backend 8443:443 -kubectl port-forward -n integr8scode svc/integr8scode-frontend 5001:5001 -``` - -For production exposure, enable the ingress section in your values file and configure it for your ingress controller. -The chart supports standard Kubernetes ingress annotations for TLS termination and path routing. - -### Monitoring the deployment - -Check pod status and logs using standard kubectl commands. - -```bash -kubectl get pods -n integr8scode -kubectl logs -n integr8scode -l app.kubernetes.io/component=backend -kubectl logs -n integr8scode -l app.kubernetes.io/component=coordinator -``` - -The deploy script's `status` command shows both Docker Compose and Kubernetes status in one view. +Check service status using the deploy script or Docker Compose directly. ```bash ./deploy.sh status +docker compose ps +docker compose logs backend ``` -### Rollback and uninstall - -Helm maintains release history, so you can roll back to a previous version if something goes wrong. - -```bash -helm rollback integr8scode 1 -n integr8scode -``` - -To completely remove the deployment: - -```bash -helm uninstall integr8scode -n integr8scode -kubectl delete namespace integr8scode -``` - -This removes all resources created by the chart. Persistent volume claims for MongoDB and Redis may remain depending on -your storage class's reclaim policy. - ## Troubleshooting | Issue | Cause | Solution | |-----------------------|-----------------------------------|---------------------------------------------------| -| Unknown topic errors | kafka-init failed or wrong prefix | Check `kubectl logs job/integr8scode-kafka-init` | -| Confluent port errors | K8s auto-generated `KAFKA_PORT` | Ensure `unset KAFKA_PORT` in container startup | -| ImagePullBackOff | Images not in cluster | Use ghcr.io images or import with K3s | -| MongoDB auth errors | Password mismatch | Verify secret matches `values-prod.yaml` | -| OOMKilled workers | Resource limits too low | Increase `workers.common.resources.limits.memory` | +| Unknown topic errors | kafka-init failed or wrong prefix | Check `docker compose logs kafka-init` | +| MongoDB auth errors | Password mismatch | Verify `secrets.toml` matches compose env vars | +| Worker crash loop | Config file missing | Ensure `config..toml` exists | ### Kafka topic debugging ```bash -kubectl logs -n integr8scode job/integr8scode-kafka-init -kubectl exec -n integr8scode integr8scode-kafka-0 -- kafka-topics --list --bootstrap-server localhost:29092 +docker compose logs kafka-init +docker compose exec kafka kafka-topics --list --bootstrap-server localhost:29092 ``` Topics should be prefixed (e.g., `prefexecution_events` not `execution_events`). -### MongoDB password verification - -```bash -kubectl get secret -n integr8scode integr8scode-mongodb -o jsonpath='{.data.mongodb-root-password}' | base64 -d -``` - ### k3s crash loop after VPN or IP change **Symptoms:** @@ -421,64 +248,25 @@ When using VPN (e.g., NordVPN with WireGuard/NordLynx): ## Pre-built images -For production deployments, you can skip the local build step entirely by using pre-built images from GitHub Container -Registry. The CI pipeline automatically builds and pushes images on every merge to main. - -### Using registry images - -The `--prod` flag configures the deployment to pull images from `ghcr.io/hardmax71/integr8scode/`: +The CI pipeline automatically builds and pushes images to GitHub Container Registry on every merge to main. To use +pre-built images instead of building locally, set `IMAGE_TAG`: ```bash -./deploy.sh prod --prod \ - --set userSeed.defaultUserPassword=secure-pass \ - --set userSeed.adminUserPassword=secure-admin +IMAGE_TAG=sha-abc1234 docker compose up -d --no-build ``` -This skips the local Docker build and tells Kubernetes to pull images from the registry. The `values-prod.yaml` file -sets `imagePullPolicy: IfNotPresent`, so images are cached locally after the first pull. - ### Available tags -Each push to main produces multiple tags: - | Tag | Description | |---------------|------------------------------------| | `latest` | Most recent build from main branch | | `sha-abc1234` | Specific commit SHA | -| `v1.0.0` | Release version (from git tags) | - -For production, pin to a specific SHA or version rather than `latest`: - -```yaml -images: - backend: - repository: ghcr.io/hardmax71/integr8scode/backend - tag: sha-abc1234 -``` - -### Hybrid approach - -If you need production resource limits but want to build locally (for testing changes before pushing): - -```bash -./deploy.sh prod --prod --local -``` - -The `--local` flag forces a local build even when using `values-prod.yaml`. - -### CI/CD integration - -The GitHub Actions workflow in `.github/workflows/docker.yml` handles image building and publishing. On every push to -main, it builds the base, backend, and frontend images, scans them with Trivy for vulnerabilities, and pushes to -ghcr.io. Pull requests build and scan but don't push, ensuring only tested code reaches the registry. ## Key files | File | Purpose | |--------------------------------------------------------------------------------------------------------------------------------|-----------------------------| -| [`deploy.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/deploy.sh) | Unified deployment script | -| [`docker-compose.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/docker-compose.yaml) | Local development stack | +| [`deploy.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/deploy.sh) | Deployment script | +| [`docker-compose.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/docker-compose.yaml) | Full stack definition | | [`backend/Dockerfile.base`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/Dockerfile.base) | Shared base image with deps | -| [`helm/integr8scode/values.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/helm/integr8scode/values.yaml) | Default Helm configuration | -| [`helm/integr8scode/values-prod.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/helm/integr8scode/values-prod.yaml) | Production overrides | | [`.github/workflows/docker.yml`](https://github.com/HardMax71/Integr8sCode/blob/main/.github/workflows/docker.yml) | CI/CD image build pipeline | diff --git a/docs/operations/nginx-configuration.md b/docs/operations/nginx-configuration.md index 6ff6edb1..fe9c87d1 100644 --- a/docs/operations/nginx-configuration.md +++ b/docs/operations/nginx-configuration.md @@ -25,7 +25,7 @@ backend. | Directive | Purpose | |-----------------|--------------------------------------------------| -| `listen 5001` | Internal container port (mapped via K8s Service) | +| `listen 5001` | Internal container port (mapped via Docker Compose) | | `server_name _` | Catch-all server name | | `root` | Static files from Svelte build | @@ -132,16 +132,15 @@ The nginx image automatically processes files in `/etc/nginx/templates/*.templat |---------------|-----------------------------------|-----------------------| | `BACKEND_URL` | Backend service URL for API proxy | `https://backend:443` | -Set this via docker-compose environment section or Kubernetes deployment env vars. +Set this via the docker-compose environment section. ### Rebuilding To apply nginx configuration changes: ```bash -docker build --no-cache -t integr8scode-frontend:latest \ - -f frontend/Dockerfile.prod frontend/ -kubectl rollout restart deployment/frontend -n integr8scode +docker compose build --no-cache frontend +docker compose restart frontend ``` ## Troubleshooting @@ -150,5 +149,5 @@ kubectl rollout restart deployment/frontend -n integr8scode |--------------------------|----------------------------------|-------------------------------------------| | SSE connections dropping | Default 60s `proxy_read_timeout` | Verify 86400s timeout is set | | CSP blocking resources | Missing source in directive | Check browser console, add blocked source | -| 502 Bad Gateway | Backend unreachable | `kubectl get svc backend -n integr8scode` | +| 502 Bad Gateway | Backend unreachable | `docker compose logs backend` | | Assets not updating | Browser cache | Clear cache or verify `no-cache` on HTML | diff --git a/docs/security/policies.md b/docs/security/policies.md index f77edd97..8010dfa0 100644 --- a/docs/security/policies.md +++ b/docs/security/policies.md @@ -44,7 +44,7 @@ The network policy and RBAC resources are created by the setup script during ini ./cert-generator/setup-k8s.sh ``` -This script creates the `integr8scode` namespace, a ServiceAccount with appropriate permissions, and the deny-all NetworkPolicy. For Helm deployments, only the ServiceAccount is managed by templates in the chart (`helm/integr8scode/templates/rbac/`). The namespace is created by `deploy.sh` before Helm runs, and the NetworkPolicy must be applied separately using the setup script or manually. +This script creates the `integr8scode` namespace, a ServiceAccount with appropriate permissions, and the deny-all NetworkPolicy. ## Notes diff --git a/helm/integr8scode/.helmignore b/helm/integr8scode/.helmignore deleted file mode 100644 index b3e48029..00000000 --- a/helm/integr8scode/.helmignore +++ /dev/null @@ -1,39 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ -# Helm -charts/*.tgz -# Testing -tests/ -# CI/CD -.github/ -.gitlab-ci.yml -.circleci/ -Jenkinsfile -# Documentation -*.md -!templates/NOTES.txt -# Development -Makefile -Dockerfile* -docker-compose* diff --git a/helm/integr8scode/Chart.yaml b/helm/integr8scode/Chart.yaml deleted file mode 100644 index 96125a88..00000000 --- a/helm/integr8scode/Chart.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v2 -name: integr8scode -description: A Helm chart for Integr8sCode - Kubernetes-based code execution platform -type: application -version: 1.0.0 -appVersion: "1.0.0" - -keywords: - - kubernetes - - code-execution - - kafka - - event-driven - - microservices - -home: https://github.com/HardMax71/Integr8sCode -sources: - - https://github.com/HardMax71/Integr8sCode - -maintainers: - - name: Integr8sCode Team - -dependencies: - # Redis using Bitnami chart - - name: redis - version: "20.6.3" - repository: "https://charts.bitnami.com/bitnami" - condition: redis.enabled - - # MongoDB using Bitnami chart - - name: mongodb - version: "16.4.3" - repository: "https://charts.bitnami.com/bitnami" - condition: mongodb.enabled - -# NOTE: Kafka NOT using Bitnami sub-chart -# Reason: Confluent images have hardcoded port detection that conflicts -# with K8s auto-generated KAFKA_PORT env vars (tcp://IP:PORT format). -# Custom templates with `unset KAFKA_PORT` workaround are required. diff --git a/helm/integr8scode/charts/.gitkeep b/helm/integr8scode/charts/.gitkeep deleted file mode 100644 index 485d1a21..00000000 --- a/helm/integr8scode/charts/.gitkeep +++ /dev/null @@ -1,2 +0,0 @@ -# This directory is populated by `helm dependency update` -# Downloaded sub-charts (*.tgz) are gitignored diff --git a/helm/integr8scode/templates/NOTES.txt b/helm/integr8scode/templates/NOTES.txt deleted file mode 100644 index dfba105a..00000000 --- a/helm/integr8scode/templates/NOTES.txt +++ /dev/null @@ -1,111 +0,0 @@ -=========================================================================== - ___ _ ___ ____ _ -|_ _|_ __ | |_ ___ __ _ _ __ ( _ )___ / ___|___ __| | ___ - | || '_ \| __/ _ \/ _` | '__| / _ \___ \| | / _ \ / _` |/ _ \ - | || | | | || __/ (_| | | | (_) |__) | |__| (_) | (_| | __/ -|___|_| |_|\__\___|\__, |_| \___/____/ \____\___/ \__,_|\___| - |___/ -=========================================================================== - -Thank you for installing {{ .Chart.Name }}! - -Release: {{ .Release.Name }} -Namespace: {{ .Release.Namespace }} -Version: {{ .Chart.AppVersion }} - -=========================================================================== -DEPLOYMENT STATUS -=========================================================================== - -To check the status of your deployment: - - kubectl get pods -n {{ .Release.Namespace }} - -To view logs for a specific component: - - # Backend - kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=backend - - # Workers - kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=k8s-worker - kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=pod-monitor - kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=result-processor - -=========================================================================== -SERVICES -=========================================================================== - -Backend API: -{{- if contains "LoadBalancer" .Values.backend.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - Watch the status: - kubectl get svc -n {{ .Release.Namespace }} {{ include "integr8scode.fullname" . }}-backend -w -{{- else if contains "NodePort" .Values.backend.service.type }} - export NODE_PORT=$(kubectl get svc -n {{ .Release.Namespace }} {{ include "integr8scode.fullname" . }}-backend -o jsonpath="{.spec.ports[0].nodePort}") - export NODE_IP=$(kubectl get nodes -o jsonpath="{.items[0].status.addresses[0].address}") - echo "Backend: https://$NODE_IP:$NODE_PORT" -{{- else }} - kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "integr8scode.fullname" . }}-backend 8443:443 - echo "Backend: https://localhost:8443" -{{- end }} - -Frontend: - kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "integr8scode.fullname" . }}-frontend 5001:5001 - echo "Frontend: http://localhost:5001" - -{{- if .Values.infrastructure.jaeger.enabled }} - -Jaeger UI (Tracing): - kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "integr8scode.fullname" . }}-jaeger 16686:16686 - echo "Jaeger: http://localhost:16686" -{{- end }} - -=========================================================================== -KAFKA TOPICS -=========================================================================== - -{{- if .Values.kafkaInit.enabled }} - -Kafka topics should be automatically created by the post-install hook. -To verify topics were created: - - kubectl exec -n {{ .Release.Namespace }} {{ include "integr8scode.fullname" . }}-kafka-0 -- \ - kafka-topics --bootstrap-server localhost:29092 --list - -{{- else }} - -Kafka topic auto-creation is disabled. Please create topics manually: - - kubectl exec -n {{ .Release.Namespace }} {{ include "integr8scode.fullname" . }}-kafka-0 -- \ - kafka-topics --bootstrap-server localhost:29092 --create --topic execution_events --partitions 10 --replication-factor 1 - -{{- end }} - -=========================================================================== -TROUBLESHOOTING -=========================================================================== - -If pods are not starting: -1. Check events: kubectl get events -n {{ .Release.Namespace }} --sort-by='.lastTimestamp' -2. Check pod status: kubectl describe pod -n {{ .Release.Namespace }} -3. Check logs: kubectl logs -n {{ .Release.Namespace }} - -Common issues: -- Image not found: Ensure images are imported to K3s containerd -- RBAC errors: Check ServiceAccount and Role bindings -- Kafka connection: Wait for Kafka to be ready before workers start - -=========================================================================== -UPGRADE / ROLLBACK -=========================================================================== - -To upgrade: - helm upgrade {{ .Release.Name }} ./helm/integr8scode -n {{ .Release.Namespace }} - -To rollback: - helm rollback {{ .Release.Name }} 1 -n {{ .Release.Namespace }} - -To uninstall: - helm uninstall {{ .Release.Name }} -n {{ .Release.Namespace }} - -=========================================================================== diff --git a/helm/integr8scode/templates/_helpers.tpl b/helm/integr8scode/templates/_helpers.tpl deleted file mode 100644 index b145728e..00000000 --- a/helm/integr8scode/templates/_helpers.tpl +++ /dev/null @@ -1,145 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "integr8scode.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "integr8scode.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "integr8scode.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "integr8scode.labels" -}} -helm.sh/chart: {{ include "integr8scode.chart" . }} -{{ include "integr8scode.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/part-of: integr8scode -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "integr8scode.selectorLabels" -}} -app.kubernetes.io/name: {{ include "integr8scode.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "integr8scode.serviceAccountName" -}} -{{- if .Values.rbac.create }} -{{- default (include "integr8scode.fullname" .) .Values.rbac.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.rbac.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Backend image reference -*/}} -{{- define "integr8scode.backendImage" -}} -{{- printf "%s:%s" .Values.images.backend.repository .Values.images.backend.tag }} -{{- end }} - -{{/* -Frontend image reference -*/}} -{{- define "integr8scode.frontendImage" -}} -{{- printf "%s:%s" .Values.images.frontend.repository .Values.images.frontend.tag }} -{{- end }} - -{{/* -Redis host - uses Bitnami sub-chart naming convention -*/}} -{{- define "integr8scode.redisHost" -}} -{{- if .Values.redis.enabled }} -{{- printf "%s-redis-master" .Release.Name }} -{{- else }} -{{- .Values.externalRedis.host }} -{{- end }} -{{- end }} - -{{/* -MongoDB host - uses Bitnami sub-chart naming convention -*/}} -{{- define "integr8scode.mongodbHost" -}} -{{- if .Values.mongodb.enabled }} -{{- printf "%s-mongodb" .Release.Name }} -{{- else }} -{{- .Values.externalMongodb.host }} -{{- end }} -{{- end }} - -{{/* -MongoDB URL - with URL-encoded credentials (RFC 3986) -*/}} -{{- define "integr8scode.mongodbUrl" -}} -{{- if .Values.mongodb.enabled }} -{{- if .Values.mongodb.auth.enabled }} -{{- printf "mongodb://%s:%s@%s:27017/integr8scode?authSource=admin" (.Values.mongodb.auth.rootUser | urlquery) (.Values.mongodb.auth.rootPassword | urlquery) (include "integr8scode.mongodbHost" .) }} -{{- else }} -{{- printf "mongodb://%s:27017/integr8scode" (include "integr8scode.mongodbHost" .) }} -{{- end }} -{{- else }} -{{- .Values.externalMongodb.url }} -{{- end }} -{{- end }} - -{{/* -Kafka bootstrap servers -*/}} -{{- define "integr8scode.kafkaBootstrapServers" -}} -{{- printf "%s-kafka:29092" .Release.Name }} -{{- end }} - -{{/* -Jaeger host -*/}} -{{- define "integr8scode.jaegerHost" -}} -{{- printf "%s-jaeger" .Release.Name }} -{{- end }} - -{{/* -Generate worker labels -*/}} -{{- define "integr8scode.workerLabels" -}} -{{ include "integr8scode.labels" .root }} -app.kubernetes.io/component: {{ .name }} -{{- end }} - -{{/* -Generate worker selector labels -*/}} -{{- define "integr8scode.workerSelectorLabels" -}} -app.kubernetes.io/name: {{ include "integr8scode.name" .root }} -app.kubernetes.io/instance: {{ .root.Release.Name }} -app.kubernetes.io/component: {{ .name }} -{{- end }} diff --git a/helm/integr8scode/templates/app/backend-deployment.yaml b/helm/integr8scode/templates/app/backend-deployment.yaml deleted file mode 100644 index d45faceb..00000000 --- a/helm/integr8scode/templates/app/backend-deployment.yaml +++ /dev/null @@ -1,90 +0,0 @@ -{{- if .Values.backend.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-backend - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: backend -spec: - replicas: {{ .Values.backend.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-backend - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-backend - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: backend - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: backend - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - ports: - - containerPort: 443 - name: https - - containerPort: 9090 - name: metrics - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KUBECONFIG - value: "/app/kubeconfig.yaml" - {{- if .Values.backend.extraEnv }} - {{- if .Values.backend.extraEnv.SECRET_KEY }} - - name: SECRET_KEY - value: {{ .Values.backend.extraEnv.SECRET_KEY | quote }} - {{- end }} - {{- if .Values.backend.extraEnv.WEB_CONCURRENCY }} - - name: WEB_CONCURRENCY - value: {{ .Values.backend.extraEnv.WEB_CONCURRENCY | quote }} - {{- end }} - {{- end }} - resources: - {{- toYaml .Values.backend.resources | nindent 10 }} - volumeMounts: - - name: kubeconfig - mountPath: /app/kubeconfig.yaml - subPath: kubeconfig.yaml - readOnly: true - {{- if .Values.backend.ssl.enabled }} - - name: ssl-certs - mountPath: /app/certs - readOnly: true - {{- end }} - # Correct health probe path (fixes 405 error) - livenessProbe: - httpGet: - path: {{ .Values.backend.healthProbe.path | default "/api/v1/health/live" }} - port: 443 - scheme: {{ .Values.backend.healthProbe.scheme | default "HTTPS" }} - initialDelaySeconds: 30 - periodSeconds: 30 - failureThreshold: 3 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: {{ .Values.backend.healthProbe.readyPath | default "/api/v1/health/ready" }} - port: 443 - scheme: {{ .Values.backend.healthProbe.scheme | default "HTTPS" }} - initialDelaySeconds: 10 - periodSeconds: 10 - failureThreshold: 3 - timeoutSeconds: 5 - volumes: - - name: kubeconfig - secret: - secretName: {{ .Values.kubeconfig.secretName }} - {{- if .Values.backend.ssl.enabled }} - - name: ssl-certs - secret: - secretName: {{ .Values.backend.ssl.secretName }} - {{- end }} -{{- end }} diff --git a/helm/integr8scode/templates/app/backend-service.yaml b/helm/integr8scode/templates/app/backend-service.yaml deleted file mode 100644 index e6ac1789..00000000 --- a/helm/integr8scode/templates/app/backend-service.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.backend.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "integr8scode.fullname" . }}-backend - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: backend -spec: - type: {{ .Values.backend.service.type | default "ClusterIP" }} - selector: - app: {{ include "integr8scode.fullname" . }}-backend - ports: - - name: https - port: {{ .Values.backend.service.port | default 443 }} - targetPort: 443 - - name: metrics - port: 9090 - targetPort: 9090 -{{- end }} diff --git a/helm/integr8scode/templates/app/frontend-deployment.yaml b/helm/integr8scode/templates/app/frontend-deployment.yaml deleted file mode 100644 index 6ac1ebb9..00000000 --- a/helm/integr8scode/templates/app/frontend-deployment.yaml +++ /dev/null @@ -1,51 +0,0 @@ -{{- if .Values.frontend.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-frontend - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: frontend -spec: - replicas: {{ .Values.frontend.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-frontend - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-frontend - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: frontend - spec: - automountServiceAccountToken: false - containers: - - name: frontend - image: {{ include "integr8scode.frontendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - ports: - - containerPort: {{ .Values.frontend.containerPort | default 5001 }} - name: http - env: - # BACKEND_URL is used by nginx template (envsubst at container startup) - - name: BACKEND_URL - value: "https://{{ include "integr8scode.fullname" . }}-backend:443" - # VITE_BACKEND_URL is baked into JS at build time (not used in prod nginx image) - - name: VITE_BACKEND_URL - value: "https://{{ include "integr8scode.fullname" . }}-backend:443" - resources: - {{- toYaml .Values.frontend.resources | nindent 10 }} - livenessProbe: - httpGet: - path: / - port: {{ .Values.frontend.containerPort | default 5001 }} - initialDelaySeconds: 10 - periodSeconds: 30 - readinessProbe: - httpGet: - path: / - port: {{ .Values.frontend.containerPort | default 5001 }} - initialDelaySeconds: 5 - periodSeconds: 10 -{{- end }} diff --git a/helm/integr8scode/templates/app/frontend-service.yaml b/helm/integr8scode/templates/app/frontend-service.yaml deleted file mode 100644 index 83fb19df..00000000 --- a/helm/integr8scode/templates/app/frontend-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.frontend.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "integr8scode.fullname" . }}-frontend - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: frontend -spec: - type: {{ .Values.frontend.service.type | default "ClusterIP" }} - selector: - app: {{ include "integr8scode.fullname" . }}-frontend - ports: - - name: http - port: {{ .Values.frontend.service.port | default 5001 }} - targetPort: {{ .Values.frontend.containerPort | default 5001 }} -{{- end }} diff --git a/helm/integr8scode/templates/configmaps/zookeeper-config.yaml b/helm/integr8scode/templates/configmaps/zookeeper-config.yaml deleted file mode 100644 index ec63ab83..00000000 --- a/helm/integr8scode/templates/configmaps/zookeeper-config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.infrastructure.zookeeper.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "integr8scode.fullname" . }}-zookeeper-config - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} -data: - zoo.cfg: | - tickTime=2000 - dataDir=/var/lib/zookeeper/data - dataLogDir=/var/lib/zookeeper/log - clientPort=2181 - serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory - authProvider.1=org.apache.zookeeper.server.auth.DigestAuthenticationProvider - autopurge.snapRetainCount=3 - autopurge.purgeInterval=24 - admin.enableServer=false - admin.serverPort=0 - 4lw.commands.whitelist=* - log4j.properties: | - log4j.rootLogger=WARN, CONSOLE - log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender - log4j.appender.CONSOLE.Threshold=WARN - log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout - log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n -{{- end }} diff --git a/helm/integr8scode/templates/infrastructure/jaeger.yaml b/helm/integr8scode/templates/infrastructure/jaeger.yaml deleted file mode 100644 index 49fccb18..00000000 --- a/helm/integr8scode/templates/infrastructure/jaeger.yaml +++ /dev/null @@ -1,75 +0,0 @@ -{{- if .Values.infrastructure.jaeger.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-jaeger - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: jaeger -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-jaeger - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-jaeger - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: jaeger - spec: - automountServiceAccountToken: false - containers: - - name: jaeger - image: {{ .Values.infrastructure.jaeger.image }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - ports: - - containerPort: 4317 - name: otlp-grpc - - containerPort: 6831 - name: thrift - protocol: UDP - - containerPort: 16686 - name: ui - env: - - name: COLLECTOR_OTLP_ENABLED - value: "true" - resources: - {{- toYaml .Values.infrastructure.jaeger.resources | nindent 10 }} - readinessProbe: - httpGet: - path: / - port: 16686 - initialDelaySeconds: 10 - periodSeconds: 10 - livenessProbe: - httpGet: - path: / - port: 16686 - initialDelaySeconds: 30 - periodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "integr8scode.fullname" . }}-jaeger - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: jaeger -spec: - selector: - app: {{ include "integr8scode.fullname" . }}-jaeger - ports: - - name: otlp-grpc - port: 4317 - targetPort: 4317 - - name: thrift - port: 6831 - targetPort: 6831 - protocol: UDP - - name: ui - port: 16686 - targetPort: 16686 -{{- end }} diff --git a/helm/integr8scode/templates/infrastructure/kafka.yaml b/helm/integr8scode/templates/infrastructure/kafka.yaml deleted file mode 100644 index b4142eba..00000000 --- a/helm/integr8scode/templates/infrastructure/kafka.yaml +++ /dev/null @@ -1,114 +0,0 @@ -{{- if .Values.infrastructure.kafka.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-kafka - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: kafka -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-kafka - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-kafka - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: kafka - spec: - automountServiceAccountToken: false - containers: - - name: kafka - image: {{ .Values.infrastructure.kafka.image }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - # CRITICAL: Unset KAFKA_PORT to prevent Confluent image conflict - # K8s auto-generates KAFKA_PORT as tcp://IP:PORT format which - # conflicts with Confluent's internal port detection - command: - - bash - - -c - - | - unset KAFKA_PORT - /etc/confluent/docker/run - ports: - - containerPort: 29092 - name: broker - env: - - name: KAFKA_BROKER_ID - value: "1" - - name: KAFKA_ZOOKEEPER_CONNECT - value: "{{ include "integr8scode.fullname" . }}-zookeeper:2181" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: "PLAINTEXT:PLAINTEXT" - - name: KAFKA_LISTENERS - value: "PLAINTEXT://0.0.0.0:29092" - - name: KAFKA_ADVERTISED_LISTENERS - value: "PLAINTEXT://{{ include "integr8scode.fullname" . }}-kafka:29092" - - name: KAFKA_INTER_BROKER_LISTENER_NAME - value: "PLAINTEXT" - - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR - value: "1" - - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR - value: "1" - - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR - value: "1" - - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS - value: "0" - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE - value: {{ .Values.infrastructure.kafka.autoCreateTopics | default "true" | quote }} - - name: KAFKA_DELETE_TOPIC_ENABLE - value: "true" - - name: KAFKA_OPTS - value: "-Dzookeeper.sasl.client=true -Dzookeeper.sasl.clientconfig=Client -Djava.security.auth.login.config=/etc/kafka/secrets/kafka_jaas.conf" - - name: KAFKA_ZOOKEEPER_SET_ACL - value: "false" - - name: KAFKA_HEAP_OPTS - value: {{ .Values.infrastructure.kafka.heapOpts | default "-Xms256M -Xmx256M" | quote }} - resources: - {{- toYaml .Values.infrastructure.kafka.resources | nindent 10 }} - volumeMounts: - - name: jaas-config - mountPath: /etc/kafka/secrets - readOnly: true - readinessProbe: - exec: - command: - - sh - - -c - - "kafka-broker-api-versions --bootstrap-server localhost:29092" - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 10 - livenessProbe: - exec: - command: - - sh - - -c - - "kafka-broker-api-versions --bootstrap-server localhost:29092" - initialDelaySeconds: 60 - periodSeconds: 30 - timeoutSeconds: 10 - volumes: - - name: jaas-config - secret: - secretName: {{ include "integr8scode.fullname" . }}-kafka-jaas ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "integr8scode.fullname" . }}-kafka - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: kafka -spec: - selector: - app: {{ include "integr8scode.fullname" . }}-kafka - ports: - - name: broker - port: 29092 - targetPort: 29092 -{{- end }} diff --git a/helm/integr8scode/templates/infrastructure/zookeeper.yaml b/helm/integr8scode/templates/infrastructure/zookeeper.yaml deleted file mode 100644 index b8d6c06d..00000000 --- a/helm/integr8scode/templates/infrastructure/zookeeper.yaml +++ /dev/null @@ -1,99 +0,0 @@ -{{- if .Values.infrastructure.zookeeper.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-zookeeper - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: zookeeper -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-zookeeper - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-zookeeper - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: zookeeper - spec: - automountServiceAccountToken: false - containers: - - name: zookeeper - image: {{ .Values.infrastructure.zookeeper.image }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - # CRITICAL: Unset ZOOKEEPER_PORT to prevent Confluent image conflict - # K8s auto-generates ZOOKEEPER_PORT as tcp://IP:PORT format - command: - - bash - - -c - - | - unset ZOOKEEPER_PORT - /etc/confluent/docker/run - ports: - - containerPort: 2181 - name: client - env: - - name: ZOOKEEPER_CLIENT_PORT - value: "2181" - - name: ZOOKEEPER_TICK_TIME - value: "2000" - - name: ZOOKEEPER_AUTH_PROVIDER_1 - value: "org.apache.zookeeper.server.auth.DigestAuthenticationProvider" - - name: ZOOKEEPER_SERVER_CNXN_FACTORY - value: "org.apache.zookeeper.server.NettyServerCnxnFactory" - - name: KAFKA_OPTS - value: "-Djava.security.auth.login.config=/etc/kafka/secrets/kafka_jaas.conf" - - name: ZOOKEEPER_4LW_COMMANDS_WHITELIST - value: "*" - - name: KAFKA_HEAP_OPTS - value: {{ .Values.infrastructure.zookeeper.heapOpts | default "-Xms256M -Xmx256M" | quote }} - - name: ZOOKEEPER_LOG4J_ROOT_LOGLEVEL - value: "WARN" - resources: - {{- toYaml .Values.infrastructure.zookeeper.resources | nindent 10 }} - volumeMounts: - - name: jaas-config - mountPath: /etc/kafka/secrets - readOnly: true - readinessProbe: - exec: - command: - - sh - - -c - - "echo ruok | nc localhost 2181 | grep imok" - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 5 - livenessProbe: - exec: - command: - - sh - - -c - - "echo ruok | nc localhost 2181 | grep imok" - initialDelaySeconds: 30 - periodSeconds: 30 - timeoutSeconds: 5 - volumes: - - name: jaas-config - secret: - secretName: {{ include "integr8scode.fullname" . }}-kafka-jaas ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "integr8scode.fullname" . }}-zookeeper - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: zookeeper -spec: - selector: - app: {{ include "integr8scode.fullname" . }}-zookeeper - ports: - - name: client - port: 2181 - targetPort: 2181 -{{- end }} diff --git a/helm/integr8scode/templates/jobs/kafka-init.yaml b/helm/integr8scode/templates/jobs/kafka-init.yaml deleted file mode 100644 index 202c7304..00000000 --- a/helm/integr8scode/templates/jobs/kafka-init.yaml +++ /dev/null @@ -1,58 +0,0 @@ -{{- if .Values.kafkaInit.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "integr8scode.fullname" . }}-kafka-init - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: kafka-init - annotations: - # Helm hook: Run after install and upgrade, after all other resources are created - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "5" - # Delete previous job before creating new one to ensure idempotency - "helm.sh/hook-delete-policy": before-hook-creation -spec: - ttlSecondsAfterFinished: {{ .Values.kafkaInit.ttlSecondsAfterFinished | default 300 }} - backoffLimit: {{ .Values.kafkaInit.backoffLimit | default 3 }} - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-kafka-init - {{- include "integr8scode.labels" . | nindent 8 }} - spec: - automountServiceAccountToken: false - restartPolicy: OnFailure - initContainers: - # Wait for Kafka to be ready before attempting topic creation - - name: wait-for-kafka - image: busybox:1.36 - command: - - sh - - -c - - | - echo "Waiting for Kafka to be ready at {{ include "integr8scode.fullname" . }}-kafka:29092..." - until nc -z {{ include "integr8scode.fullname" . }}-kafka 29092; do - echo "Kafka not ready, waiting 5 seconds..." - sleep 5 - done - echo "Kafka is ready!" - containers: - - name: kafka-init - image: {{ .Values.kafkaInit.image | default (include "integr8scode.backendImage" .) }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - - uv - - run - - python - - -m - - scripts.create_topics - env: - - name: KAFKA_BOOTSTRAP_SERVERS - value: {{ include "integr8scode.kafkaBootstrapServers" . | quote }} - - name: LOG_LEVEL - value: "INFO" - resources: - {{- toYaml .Values.kafkaInit.resources | nindent 10 }} -{{- end }} diff --git a/helm/integr8scode/templates/jobs/user-seed.yaml b/helm/integr8scode/templates/jobs/user-seed.yaml deleted file mode 100644 index b6bdb8b6..00000000 --- a/helm/integr8scode/templates/jobs/user-seed.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{{- if .Values.userSeed.enabled }} -{{/* Dev defaults provided - override via values-prod.yaml or --set for production */}} -{{- $defaultPwd := .Values.userSeed.defaultUserPassword | default "user123" }} -{{- $adminPwd := .Values.userSeed.adminUserPassword | default "admin123" }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "integr8scode.fullname" . }}-user-seed - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: user-seed - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "10" - "helm.sh/hook-delete-policy": before-hook-creation -spec: - ttlSecondsAfterFinished: {{ .Values.userSeed.ttlSecondsAfterFinished | default 300 }} - backoffLimit: {{ .Values.userSeed.backoffLimit | default 3 }} - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-user-seed - {{- include "integr8scode.labels" . | nindent 8 }} - spec: - automountServiceAccountToken: false - restartPolicy: OnFailure - initContainers: - - name: wait-for-mongodb - image: busybox:1.36 - command: - - sh - - -c - - | - echo "Waiting for MongoDB to be ready..." - until nc -z {{ include "integr8scode.fullname" . }}-mongodb 27017; do - echo "MongoDB not ready, waiting 5 seconds..." - sleep 5 - done - echo "MongoDB is ready!" - containers: - - name: user-seed - image: {{ .Values.userSeed.image | default (include "integr8scode.backendImage" .) }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - - uv - - run - - python - - -m - - scripts.seed_users - env: - - name: MONGODB_URL - value: {{ include "integr8scode.mongodbUrl" . | quote }} - - name: DEFAULT_USER_PASSWORD - value: {{ $defaultPwd | quote }} - - name: ADMIN_USER_PASSWORD - value: {{ $adminPwd | quote }} - resources: - {{- toYaml .Values.userSeed.resources | nindent 10 }} -{{- end }} diff --git a/helm/integr8scode/templates/rbac/role.yaml b/helm/integr8scode/templates/rbac/role.yaml deleted file mode 100644 index 4b1e35d5..00000000 --- a/helm/integr8scode/templates/rbac/role.yaml +++ /dev/null @@ -1,72 +0,0 @@ -{{- if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "integr8scode.fullname" . }}-executor - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} -rules: - # Core API resources for pod management - - apiGroups: [""] - resources: - - pods - - pods/log - - pods/status - verbs: - - create - - get - - list - - watch - - delete - - update - - patch - # ConfigMaps for script storage - - apiGroups: [""] - resources: - - configmaps - verbs: - - create - - get - - list - - watch - - delete - - update - - patch - # Secrets for kubeconfig and other credentials - - apiGroups: [""] - resources: - - secrets - verbs: - - get - - list - - watch - # Services for pod networking (optional) - - apiGroups: [""] - resources: - - services - verbs: - - get - - list - - watch - # Apps API for DaemonSet management (runtime-image-pre-puller) - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - create - - get - - list - - watch - - delete - - update - - patch - # Events for debugging - - apiGroups: [""] - resources: - - events - verbs: - - get - - list - - watch -{{- end }} diff --git a/helm/integr8scode/templates/rbac/rolebinding.yaml b/helm/integr8scode/templates/rbac/rolebinding.yaml deleted file mode 100644 index 66d22351..00000000 --- a/helm/integr8scode/templates/rbac/rolebinding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.rbac.create }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "integr8scode.fullname" . }}-executor - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "integr8scode.fullname" . }}-executor -subjects: - - kind: ServiceAccount - name: {{ include "integr8scode.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/helm/integr8scode/templates/rbac/serviceaccount.yaml b/helm/integr8scode/templates/rbac/serviceaccount.yaml deleted file mode 100644 index c0fed3bb..00000000 --- a/helm/integr8scode/templates/rbac/serviceaccount.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.rbac.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "integr8scode.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - {{- with .Values.rbac.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: true -{{- end }} diff --git a/helm/integr8scode/templates/secrets/env-secret.yaml b/helm/integr8scode/templates/secrets/env-secret.yaml deleted file mode 100644 index 1d826f1a..00000000 --- a/helm/integr8scode/templates/secrets/env-secret.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "integr8scode.fullname" . }}-env - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} -type: Opaque -stringData: - # ========================================================================== - # CRITICAL: Explicit port settings to override K8s auto-generated env vars - # K8s auto-creates SERVICE_PORT variables in tcp://IP:PORT format which - # conflicts with application expectations of integer port numbers. - # ========================================================================== - REDIS_HOST: {{ include "integr8scode.redisHost" . | quote }} - REDIS_PORT: "6379" - REDIS_DB: {{ .Values.env.REDIS_DB | default "0" | quote }} - - # Kafka configuration - KAFKA_BOOTSTRAP_SERVERS: {{ include "integr8scode.kafkaBootstrapServers" . | quote }} - - # MongoDB (contains credentials - URL-encoded) - MONGODB_URL: {{ include "integr8scode.mongodbUrl" . | quote }} - - # Kubernetes - K8S_NAMESPACE: {{ .Release.Namespace | quote }} - - # Application settings - PROJECT_NAME: {{ .Values.env.PROJECT_NAME | default "integr8scode" | quote }} - API_V1_STR: {{ .Values.env.API_V1_STR | default "/api/v1" | quote }} - LOG_LEVEL: {{ .Values.env.LOG_LEVEL | default "INFO" | quote }} - SERVICE_NAME: {{ .Values.env.SERVICE_NAME | default "integr8scode-backend" | quote }} - SERVICE_VERSION: {{ .Values.env.SERVICE_VERSION | default .Chart.AppVersion | quote }} - - # Server configuration - SERVER_HOST: "0.0.0.0" - SERVER_PORT: "443" - - # Event streaming - ENABLE_EVENT_STREAMING: {{ .Values.env.ENABLE_EVENT_STREAMING | default "true" | quote }} - EVENT_RETENTION_DAYS: {{ .Values.env.EVENT_RETENTION_DAYS | default "30" | quote }} - - # Tracing - ENABLE_TRACING: {{ .Values.env.ENABLE_TRACING | default "false" | quote }} - {{- if .Values.infrastructure.jaeger.enabled }} - JAEGER_AGENT_HOST: {{ include "integr8scode.jaegerHost" . | quote }} - JAEGER_AGENT_PORT: "6831" - {{- end }} - TRACING_SAMPLING_RATE: {{ .Values.env.TRACING_SAMPLING_RATE | default "0.1" | quote }} - - # DLQ configuration - DLQ_RETRY_MAX_ATTEMPTS: {{ .Values.env.DLQ_RETRY_MAX_ATTEMPTS | default "5" | quote }} - DLQ_RETENTION_DAYS: {{ .Values.env.DLQ_RETENTION_DAYS | default "7" | quote }} - - # Rate limiting - RATE_LIMIT_DEFAULT_REQUESTS: {{ .Values.env.RATE_LIMIT_DEFAULT_REQUESTS | default "100" | quote }} - RATE_LIMIT_DEFAULT_WINDOW: {{ .Values.env.RATE_LIMIT_DEFAULT_WINDOW | default "60" | quote }} - - # WebSocket / SSE - WEBSOCKET_PING_INTERVAL: {{ .Values.env.WEBSOCKET_PING_INTERVAL | default "30" | quote }} - WEBSOCKET_PING_TIMEOUT: {{ .Values.env.WEBSOCKET_PING_TIMEOUT | default "10" | quote }} - SSE_CONSUMER_POOL_SIZE: {{ .Values.env.SSE_CONSUMER_POOL_SIZE | default "10" | quote }} - SSE_HEARTBEAT_INTERVAL: {{ .Values.env.SSE_HEARTBEAT_INTERVAL | default "30" | quote }} diff --git a/helm/integr8scode/templates/secrets/kafka-jaas-secret.yaml b/helm/integr8scode/templates/secrets/kafka-jaas-secret.yaml deleted file mode 100644 index 0e36e82c..00000000 --- a/helm/integr8scode/templates/secrets/kafka-jaas-secret.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{- if .Values.infrastructure.kafka.enabled }} -{{- $superPassword := .Values.infrastructure.kafka.jaas.superPassword | default "admin-secret" }} -{{- $kafkaPassword := .Values.infrastructure.kafka.jaas.kafkaPassword | default "kafka-secret" }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "integr8scode.fullname" . }}-kafka-jaas - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} -type: Opaque -stringData: - kafka_jaas.conf: | - Server { - org.apache.zookeeper.server.auth.DigestLoginModule required - user_super="{{ $superPassword }}" - user_kafka="{{ $kafkaPassword }}"; - }; - - Client { - org.apache.zookeeper.server.auth.DigestLoginModule required - username="kafka" - password="{{ $kafkaPassword }}"; - }; -{{- end }} diff --git a/helm/integr8scode/templates/secrets/kubeconfig-secret.yaml b/helm/integr8scode/templates/secrets/kubeconfig-secret.yaml deleted file mode 100644 index 24e76d74..00000000 --- a/helm/integr8scode/templates/secrets/kubeconfig-secret.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if .Values.kubeconfig.create }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.kubeconfig.secretName }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} -type: Opaque -stringData: - kubeconfig.yaml: | - apiVersion: v1 - kind: Config - clusters: - - cluster: - {{- if .Values.kubeconfig.caCertificate }} - certificate-authority-data: {{ .Values.kubeconfig.caCertificate }} - {{- else }} - # For in-cluster access, use the mounted CA - certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - {{- end }} - server: {{ .Values.kubeconfig.server | default "https://kubernetes.default.svc:443" }} - name: default-cluster - contexts: - - context: - cluster: default-cluster - namespace: {{ .Release.Namespace }} - user: default-user - name: default-context - current-context: default-context - users: - - name: default-user - user: - {{- if .Values.kubeconfig.token }} - token: {{ .Values.kubeconfig.token }} - {{- else }} - # For in-cluster access, use the mounted service account token - tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - {{- end }} -{{- end }} diff --git a/helm/integr8scode/templates/workers/_worker.tpl b/helm/integr8scode/templates/workers/_worker.tpl deleted file mode 100644 index 480f4d19..00000000 --- a/helm/integr8scode/templates/workers/_worker.tpl +++ /dev/null @@ -1,72 +0,0 @@ -{{/* -Generate a worker deployment. -Usage: {{ include "integr8scode.worker" (dict "root" . "name" "k8s-worker" "config" .Values.workers.k8sWorker) }} -*/}} -{{- define "integr8scode.worker" -}} -{{- $root := .root -}} -{{- $name := .name -}} -{{- $config := .config -}} -{{- if $config.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" $root }}-{{ $name }} - namespace: {{ $root.Release.Namespace }} - labels: - {{- include "integr8scode.labels" $root | nindent 4 }} - app.kubernetes.io/component: {{ $name }} -spec: - replicas: {{ $config.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" $root }}-{{ $name }} - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" $root }}-{{ $name }} - {{- include "integr8scode.labels" $root | nindent 8 }} - app.kubernetes.io/component: {{ $name }} - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") $root | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" $root }} - containers: - - name: {{ $name }} - image: {{ include "integr8scode.backendImage" $root }} - imagePullPolicy: {{ $root.Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml $config.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" $root }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ $config.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: {{ $name | quote }} - {{- if $config.requiresKubeconfig }} - - name: KUBECONFIG - value: "/app/kubeconfig.yaml" - {{- end }} - resources: - {{- if $config.resources }} - {{- toYaml $config.resources | nindent 10 }} - {{- else }} - {{- toYaml $root.Values.workers.common.resources | nindent 10 }} - {{- end }} - {{- if $config.requiresKubeconfig }} - volumeMounts: - - name: kubeconfig - mountPath: /app/kubeconfig.yaml - subPath: kubeconfig.yaml - readOnly: true - {{- end }} - {{- if $config.requiresKubeconfig }} - volumes: - - name: kubeconfig - secret: - secretName: {{ $root.Values.kubeconfig.secretName }} - {{- end }} - restartPolicy: Always -{{- end }} -{{- end -}} diff --git a/helm/integr8scode/templates/workers/coordinator.yaml b/helm/integr8scode/templates/workers/coordinator.yaml deleted file mode 100644 index 1f00b403..00000000 --- a/helm/integr8scode/templates/workers/coordinator.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if .Values.workers.coordinator.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-coordinator - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: coordinator -spec: - replicas: {{ .Values.workers.coordinator.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-coordinator - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-coordinator - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: coordinator - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: coordinator - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.coordinator.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.coordinator.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "execution-coordinator" - resources: - {{- if .Values.workers.coordinator.resources }} - {{- toYaml .Values.workers.coordinator.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/templates/workers/dlq-processor.yaml b/helm/integr8scode/templates/workers/dlq-processor.yaml deleted file mode 100644 index 2c22d3dc..00000000 --- a/helm/integr8scode/templates/workers/dlq-processor.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if .Values.workers.dlqProcessor.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-dlq-processor - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: dlq-processor -spec: - replicas: {{ .Values.workers.dlqProcessor.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-dlq-processor - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-dlq-processor - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: dlq-processor - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: dlq-processor - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.dlqProcessor.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.dlqProcessor.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "dlq-processor" - resources: - {{- if .Values.workers.dlqProcessor.resources }} - {{- toYaml .Values.workers.dlqProcessor.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/templates/workers/event-replay.yaml b/helm/integr8scode/templates/workers/event-replay.yaml deleted file mode 100644 index da6139d5..00000000 --- a/helm/integr8scode/templates/workers/event-replay.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if .Values.workers.eventReplay.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-event-replay - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: event-replay -spec: - replicas: {{ .Values.workers.eventReplay.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-event-replay - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-event-replay - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: event-replay - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: event-replay - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.eventReplay.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.eventReplay.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "event-replay" - resources: - {{- if .Values.workers.eventReplay.resources }} - {{- toYaml .Values.workers.eventReplay.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/templates/workers/k8s-worker.yaml b/helm/integr8scode/templates/workers/k8s-worker.yaml deleted file mode 100644 index 73d8b026..00000000 --- a/helm/integr8scode/templates/workers/k8s-worker.yaml +++ /dev/null @@ -1,57 +0,0 @@ -{{- if .Values.workers.k8sWorker.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-k8s-worker - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: k8s-worker -spec: - replicas: {{ .Values.workers.k8sWorker.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-k8s-worker - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-k8s-worker - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: k8s-worker - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: k8s-worker - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.k8sWorker.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.k8sWorker.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "k8s-worker" - - name: KUBECONFIG - value: "/app/kubeconfig.yaml" - resources: - {{- if .Values.workers.k8sWorker.resources }} - {{- toYaml .Values.workers.k8sWorker.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - volumeMounts: - - name: kubeconfig - mountPath: /app/kubeconfig.yaml - subPath: kubeconfig.yaml - readOnly: true - volumes: - - name: kubeconfig - secret: - secretName: {{ .Values.kubeconfig.secretName }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/templates/workers/pod-monitor.yaml b/helm/integr8scode/templates/workers/pod-monitor.yaml deleted file mode 100644 index f2ab0f3f..00000000 --- a/helm/integr8scode/templates/workers/pod-monitor.yaml +++ /dev/null @@ -1,57 +0,0 @@ -{{- if .Values.workers.podMonitor.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-pod-monitor - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: pod-monitor -spec: - replicas: {{ .Values.workers.podMonitor.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-pod-monitor - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-pod-monitor - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: pod-monitor - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: pod-monitor - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.podMonitor.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.podMonitor.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "pod-monitor" - - name: KUBECONFIG - value: "/app/kubeconfig.yaml" - resources: - {{- if .Values.workers.podMonitor.resources }} - {{- toYaml .Values.workers.podMonitor.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - volumeMounts: - - name: kubeconfig - mountPath: /app/kubeconfig.yaml - subPath: kubeconfig.yaml - readOnly: true - volumes: - - name: kubeconfig - secret: - secretName: {{ .Values.kubeconfig.secretName }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/templates/workers/result-processor.yaml b/helm/integr8scode/templates/workers/result-processor.yaml deleted file mode 100644 index 7b3661dc..00000000 --- a/helm/integr8scode/templates/workers/result-processor.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if .Values.workers.resultProcessor.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-result-processor - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: result-processor -spec: - replicas: {{ .Values.workers.resultProcessor.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-result-processor - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-result-processor - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: result-processor - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: result-processor - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.resultProcessor.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.resultProcessor.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "result-processor" - resources: - {{- if .Values.workers.resultProcessor.resources }} - {{- toYaml .Values.workers.resultProcessor.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/templates/workers/saga-orchestrator.yaml b/helm/integr8scode/templates/workers/saga-orchestrator.yaml deleted file mode 100644 index 7ff7e9f0..00000000 --- a/helm/integr8scode/templates/workers/saga-orchestrator.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if .Values.workers.sagaOrchestrator.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "integr8scode.fullname" . }}-saga-orchestrator - namespace: {{ .Release.Namespace }} - labels: - {{- include "integr8scode.labels" . | nindent 4 }} - app.kubernetes.io/component: saga-orchestrator -spec: - replicas: {{ .Values.workers.sagaOrchestrator.replicas | default 1 }} - selector: - matchLabels: - app: {{ include "integr8scode.fullname" . }}-saga-orchestrator - template: - metadata: - labels: - app: {{ include "integr8scode.fullname" . }}-saga-orchestrator - {{- include "integr8scode.labels" . | nindent 8 }} - app.kubernetes.io/component: saga-orchestrator - annotations: - checksum/config: {{ include (print $.Template.BasePath "/secrets/env-secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ include "integr8scode.serviceAccountName" . }} - containers: - - name: saga-orchestrator - image: {{ include "integr8scode.backendImage" . }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} - command: - {{- toYaml .Values.workers.sagaOrchestrator.command | nindent 10 }} - envFrom: - - secretRef: - name: {{ include "integr8scode.fullname" . }}-env - env: - - name: KAFKA_CONSUMER_GROUP_ID - value: {{ .Values.workers.sagaOrchestrator.consumerGroupId | quote }} - - name: TRACING_SERVICE_NAME - value: "saga-orchestrator" - resources: - {{- if .Values.workers.sagaOrchestrator.resources }} - {{- toYaml .Values.workers.sagaOrchestrator.resources | nindent 10 }} - {{- else }} - {{- toYaml .Values.workers.common.resources | nindent 10 }} - {{- end }} - restartPolicy: Always -{{- end }} diff --git a/helm/integr8scode/values-prod.yaml b/helm/integr8scode/values-prod.yaml deleted file mode 100644 index af592206..00000000 --- a/helm/integr8scode/values-prod.yaml +++ /dev/null @@ -1,205 +0,0 @@ -# ============================================================================= -# INTEGR8SCODE HELM CHART - PRODUCTION VALUES -# ============================================================================= -# Production-specific overrides for the Integr8sCode Helm chart. -# Usage: helm upgrade --install integr8scode ./helm/integr8scode -f values-prod.yaml - -# ============================================================================= -# IMAGE CONFIGURATION - USE REGISTRY IMAGES -# ============================================================================= -# Pre-built images from GitHub Container Registry (no local build required) -global: - imagePullPolicy: IfNotPresent - -images: - backend: - repository: ghcr.io/hardmax71/integr8scode/backend - tag: latest # Or pin to specific version: v1.0.0, sha-abc1234 - frontend: - repository: ghcr.io/hardmax71/integr8scode/frontend - tag: latest - base: - repository: ghcr.io/hardmax71/integr8scode/base - tag: latest - -# ============================================================================= -# ENVIRONMENT SETTINGS -# ============================================================================= -env: - LOG_LEVEL: "INFO" - ENABLE_TRACING: "true" - TRACING_SAMPLING_RATE: "0.1" - -# ============================================================================= -# BACKEND APPLICATION -# ============================================================================= -backend: - replicas: 2 - - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "2000m" - - extraEnv: - WEB_CONCURRENCY: "4" - # IMPORTANT: Set SECRET_KEY via --set or external secrets management - # SECRET_KEY: "" - -# ============================================================================= -# FRONTEND APPLICATION -# ============================================================================= -frontend: - replicas: 2 - - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "512Mi" - cpu: "1000m" - -# ============================================================================= -# WORKERS - PRODUCTION SCALING -# ============================================================================= -workers: - common: - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "1000m" - - # Scale critical workers for production load - k8sWorker: - replicas: 2 - - podMonitor: - replicas: 1 # Single instance is usually sufficient - - resultProcessor: - replicas: 2 - - sagaOrchestrator: - replicas: 2 - - coordinator: - replicas: 2 - - eventReplay: - replicas: 1 # Low traffic service - - dlqProcessor: - replicas: 1 # Low traffic service - -# ============================================================================= -# INFRASTRUCTURE - PRODUCTION SETTINGS -# ============================================================================= -infrastructure: - zookeeper: - heapOpts: "-Xms512M -Xmx512M" - resources: - requests: - memory: "512Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" - - kafka: - heapOpts: "-Xms1G -Xmx1G" - # IMPORTANT: Set JAAS credentials via --set for production: - # --set infrastructure.kafka.jaas.superPassword= - # --set infrastructure.kafka.jaas.kafkaPassword= - jaas: - superPassword: "" # REQUIRED: Set via --set - kafkaPassword: "" # REQUIRED: Set via --set - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "2000m" - - jaeger: - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" - -# ============================================================================= -# REDIS - PRODUCTION SETTINGS -# ============================================================================= -redis: - master: - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "1Gi" - cpu: "1000m" - persistence: - enabled: true - size: 5Gi - -# ============================================================================= -# MONGODB - PRODUCTION SETTINGS -# ============================================================================= -mongodb: - auth: - enabled: true - rootUser: "root" - # IMPORTANT: Set password via --set mongodb.auth.rootPassword= - # rootPassword: "" - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "2Gi" - cpu: "2000m" - persistence: - enabled: true - size: 50Gi - -# ============================================================================= -# USER SEED - PRODUCTION SETTINGS -# ============================================================================= -userSeed: - enabled: true - # IMPORTANT: Set passwords via --set for production: - # --set userSeed.defaultUserPassword= - # --set userSeed.adminUserPassword= - defaultUserPassword: "" # REQUIRED: Set via --set - adminUserPassword: "" # REQUIRED: Set via --set - -# ============================================================================= -# INGRESS - PRODUCTION SETTINGS -# ============================================================================= -# Uncomment and configure for production ingress -# ingress: -# enabled: true -# className: "nginx" -# annotations: -# cert-manager.io/cluster-issuer: "letsencrypt-prod" -# nginx.ingress.kubernetes.io/ssl-redirect: "true" -# hosts: -# - host: integr8scode.example.com -# paths: -# - path: / -# pathType: Prefix -# tls: -# - secretName: integr8scode-tls -# hosts: -# - integr8scode.example.com diff --git a/helm/integr8scode/values.yaml b/helm/integr8scode/values.yaml deleted file mode 100644 index 856dde9a..00000000 --- a/helm/integr8scode/values.yaml +++ /dev/null @@ -1,425 +0,0 @@ -# ============================================================================= -# INTEGR8SCODE HELM CHART - DEFAULT VALUES -# ============================================================================= -# This file contains default configuration values for the Integr8sCode Helm chart. -# Override these values using --set flags or a custom values-prod.yaml file. - -# ============================================================================= -# GLOBAL SETTINGS -# ============================================================================= -global: - # Image pull policy: - # - "Never" for K3s with locally imported images (default for local dev) - # - "IfNotPresent" for registry-pulled images - # - "Always" for latest tags from registry - imagePullPolicy: Never - - # Container registry (leave empty for local images) - # Set to "ghcr.io/hardmax71/integr8scode" for GitHub Container Registry - imageRegistry: "" - -# Name overrides -nameOverride: "" -fullnameOverride: "" - -# ============================================================================= -# IMAGE CONFIGURATION -# ============================================================================= -images: - # For local development (imagePullPolicy: Never): - # repository: integr8scode-backend - # For registry (imagePullPolicy: IfNotPresent/Always): - # repository: ghcr.io/hardmax71/integr8scode/backend - backend: - repository: integr8scode-backend - tag: latest - frontend: - repository: integr8scode-frontend - tag: latest - # Base image (used by backend Dockerfile) - base: - repository: base - tag: latest - -# ============================================================================= -# RBAC CONFIGURATION -# ============================================================================= -rbac: - # Create ServiceAccount, Role, and RoleBinding - create: true - serviceAccount: - name: integr8scode-sa - annotations: {} - -# ============================================================================= -# KUBECONFIG SECRET -# ============================================================================= -kubeconfig: - # Create kubeconfig secret for workers that need K8s API access - create: true - secretName: kubeconfig-secret - # Server URL - use in-cluster service address - server: "https://kubernetes.default.svc:443" - # Optional: Base64-encoded CA certificate (leave empty for in-cluster token) - caCertificate: "" - # Optional: ServiceAccount token (leave empty for in-cluster token) - token: "" - -# ============================================================================= -# SHARED ENVIRONMENT VARIABLES -# ============================================================================= -env: - # Application settings - PROJECT_NAME: "integr8scode" - API_V1_STR: "/api/v1" - LOG_LEVEL: "INFO" - SERVICE_NAME: "integr8scode-backend" - SERVICE_VERSION: "1.0.0" - - # Redis (explicit to override K8s auto-generated vars) - REDIS_DB: "0" - - # Event streaming - ENABLE_EVENT_STREAMING: "true" - EVENT_RETENTION_DAYS: "30" - - # Tracing - ENABLE_TRACING: "false" - TRACING_SAMPLING_RATE: "0.1" - - # DLQ configuration - DLQ_RETRY_MAX_ATTEMPTS: "5" - DLQ_RETENTION_DAYS: "7" - - # Rate limiting - RATE_LIMIT_DEFAULT_REQUESTS: "100" - RATE_LIMIT_DEFAULT_WINDOW: "60" - - # WebSocket / SSE - WEBSOCKET_PING_INTERVAL: "30" - WEBSOCKET_PING_TIMEOUT: "10" - SSE_CONSUMER_POOL_SIZE: "10" - SSE_HEARTBEAT_INTERVAL: "30" - -# ============================================================================= -# BACKEND APPLICATION -# ============================================================================= -backend: - enabled: true - replicas: 1 - - resources: - requests: - memory: "512Mi" - cpu: "250m" - ephemeral-storage: "128Mi" - limits: - memory: "1Gi" - cpu: "1000m" - ephemeral-storage: "512Mi" - - service: - type: ClusterIP - port: 443 - - # Health probe configuration (fixes 405 error) - healthProbe: - path: /api/v1/health/live - readyPath: /api/v1/health/ready - scheme: HTTPS - - # SSL configuration - ssl: - enabled: false - secretName: backend-ssl - - # Additional environment variables - extraEnv: - WEB_CONCURRENCY: "4" - # SECRET_KEY: "" # Set in values-prod.yaml or via --set - -# ============================================================================= -# FRONTEND APPLICATION -# ============================================================================= -frontend: - enabled: true - replicas: 1 - containerPort: 5001 - - resources: - requests: - memory: "128Mi" - cpu: "100m" - ephemeral-storage: "64Mi" - limits: - memory: "256Mi" - cpu: "500m" - ephemeral-storage: "128Mi" - - service: - type: ClusterIP - port: 5001 - -# ============================================================================= -# WORKERS CONFIGURATION -# ============================================================================= -workers: - # Shared settings for all workers - common: - resources: - requests: - memory: "256Mi" - cpu: "100m" - ephemeral-storage: "128Mi" - limits: - memory: "512Mi" - cpu: "500m" - ephemeral-storage: "256Mi" - - # K8s Worker - Creates executor pods - k8sWorker: - enabled: true - replicas: 1 - requiresKubeconfig: true - command: - - uv - - run - - python - - workers/run_k8s_worker.py - consumerGroupId: "k8s-worker" - - # Pod Monitor - Watches pod lifecycle events - podMonitor: - enabled: true - replicas: 1 - requiresKubeconfig: true - command: - - uv - - run - - python - - workers/run_pod_monitor.py - consumerGroupId: "pod-monitor" - - # Result Processor - Stores execution results - resultProcessor: - enabled: true - replicas: 1 - requiresKubeconfig: false - command: - - uv - - run - - python - - workers/run_result_processor.py - consumerGroupId: "result-processor-group" - - # Saga Orchestrator - Manages distributed transactions - sagaOrchestrator: - enabled: true - replicas: 1 - requiresKubeconfig: false - command: - - uv - - run - - python - - workers/run_saga_orchestrator.py - consumerGroupId: "saga-orchestrator" - - # Coordinator - Schedules executions - coordinator: - enabled: true - replicas: 1 - requiresKubeconfig: false - command: - - uv - - run - - python - - -m - - workers.run_coordinator - consumerGroupId: "execution-coordinator" - - # Event Replay - Replays historical events - eventReplay: - enabled: true - replicas: 1 - requiresKubeconfig: false - command: - - uv - - run - - python - - workers/run_event_replay.py - consumerGroupId: "event-replay" - - # DLQ Processor - Handles failed messages - dlqProcessor: - enabled: true - replicas: 1 - requiresKubeconfig: false - command: - - uv - - run - - python - - workers/dlq_processor.py - consumerGroupId: "dlq-processor" - -# ============================================================================= -# INFRASTRUCTURE - CONFLUENT PLATFORM -# ============================================================================= -# NOTE: Custom templates are used instead of Bitnami sub-charts because -# Confluent images require `unset *_PORT` workaround for K8s compatibility - -infrastructure: - # Zookeeper - zookeeper: - enabled: true - image: confluentinc/cp-zookeeper:7.8.2 - heapOpts: "-Xms256M -Xmx256M" - resources: - requests: - memory: "256Mi" - cpu: "100m" - ephemeral-storage: "128Mi" - limits: - memory: "512Mi" - cpu: "500m" - ephemeral-storage: "256Mi" - - # Kafka - kafka: - enabled: true - image: confluentinc/cp-kafka:7.8.2 - heapOpts: "-Xms256M -Xmx256M" - autoCreateTopics: "true" - # JAAS authentication credentials for Zookeeper communication - # Override these in production via --set or values-prod.yaml - jaas: - superPassword: "admin-secret" - kafkaPassword: "kafka-secret" - resources: - requests: - memory: "384Mi" - cpu: "200m" - ephemeral-storage: "256Mi" - limits: - memory: "768Mi" - cpu: "1000m" - ephemeral-storage: "512Mi" - - # Jaeger (distributed tracing) - jaeger: - enabled: true - image: jaegertracing/all-in-one:1.52 - resources: - requests: - memory: "128Mi" - cpu: "100m" - ephemeral-storage: "128Mi" - limits: - memory: "512Mi" - cpu: "500m" - ephemeral-storage: "512Mi" - -# ============================================================================= -# REDIS (BITNAMI SUB-CHART) -# ============================================================================= -redis: - enabled: true - architecture: standalone - auth: - enabled: false - master: - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" - persistence: - enabled: false # Disable for lightweight K3s deployment - -# ============================================================================= -# MONGODB (BITNAMI SUB-CHART) -# ============================================================================= -mongodb: - enabled: true - architecture: standalone - auth: - enabled: false # Disable auth for development; enable for production - rootUser: "root" - rootPassword: "" - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" - persistence: - enabled: true - size: 8Gi - -# ============================================================================= -# KAFKA INIT JOB -# ============================================================================= -kafkaInit: - enabled: true - # Image to use (defaults to backend image which has create_topics.py) - image: "" # Leave empty to use backend image - backoffLimit: 3 - ttlSecondsAfterFinished: 300 - resources: - requests: - memory: "128Mi" - cpu: "50m" - ephemeral-storage: "64Mi" - limits: - memory: "256Mi" - cpu: "200m" - ephemeral-storage: "128Mi" - -# ============================================================================= -# USER SEED JOB -# ============================================================================= -userSeed: - enabled: true - # Image to use (defaults to backend image which has seed_users.py) - image: "" # Leave empty to use backend image - backoffLimit: 3 - ttlSecondsAfterFinished: 300 - # Default user credentials (override in values-prod.yaml for production) - defaultUserPassword: "user123" - adminUserPassword: "admin123" - resources: - requests: - memory: "128Mi" - cpu: "50m" - ephemeral-storage: "64Mi" - limits: - memory: "256Mi" - cpu: "200m" - ephemeral-storage: "128Mi" - -# ============================================================================= -# EXTERNAL SERVICES (when sub-charts are disabled) -# ============================================================================= -externalRedis: - host: "" - port: 6379 - -externalMongodb: - host: "" - url: "" - -# ============================================================================= -# INGRESS (OPTIONAL) -# ============================================================================= -ingress: - enabled: false - className: "" - annotations: {} - hosts: - - host: integr8scode.local - paths: - - path: / - pathType: Prefix - tls: []