docker-compose.yaml

version: '3'
x-airflow-common: &airflow-common
  image: apache/airflow:2.7.1
  environment:
    &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://root:root@pgdatabase:5432/project1
    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__LOGGING__LOGGING_LEVEL: 'INFO'
    AIRFLOW__WEBSERVER__SECRET_KEY: 'your-secret-key'  
    AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: 'true'       
  volumes:
    - ./dags:/opt/airflow/dags
    - ./logs:/opt/airflow/logs
    - ./plugins:/opt/airflow/plugins
    - ./config:/opt/airflow/config
    - ./lab:/opt/airflow/lab
    - ./spark-apps:/opt/airflow/spark-apps
    - ./spark-data:/opt/airflow/data

services:
  airflow-init:
    <<: *airflow-common
    command: version
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-admin}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-admin}
      _PIP_ADDITIONAL_REQUIREMENTS: apache-airflow-providers-apache-spark==4.1.5

  airflow-webserver:
    <<: *airflow-common
    build:
      context: .
    command: bash -c "airflow db upgrade && airflow webserver"
    ports:
      - "8081:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 30s
      retries: 3
    restart: always
    depends_on:
      - airflow-init
    volumes:
      - ./dags:/opt/airflow/dags
      - ./logs:/opt/airflow/logs
      - ./plugins:/opt/airflow/plugins
      - ./config:/opt/airflow/config
      - ./lab:/opt/airflow/lab
      - ./spark-apps:/opt/airflow/spark-apps
      - ./spark-data:/opt/airflow/data 

  airflow-scheduler:
    <<: *airflow-common
    build:
      context: .
    command: bash -c "airflow db upgrade && airflow scheduler"
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
      interval: 30s
      timeout: 30s
      retries: 3
    restart: always
    depends_on:
      - airflow-init
    volumes:
      - ./dags:/opt/airflow/dags
      - ./logs:/opt/airflow/logs
      - ./plugins:/opt/airflow/plugins
      - ./config:/opt/airflow/config
      - ./lab:/opt/airflow/lab
      - ./spark-apps:/opt/airflow/spark-apps
      - ./spark-data:/opt/airflow/data 

  # Spark 服务
  spark-master:
    image: bitnami/spark:3.3.0
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - "8090:8080"
      - "7077:7077"
    volumes:
      - ./spark-data:/opt/spark/data
      - ./spark-apps:/opt/spark/apps
      - ./lab:/opt/airflow/lab

  spark-worker-1:
    image: bitnami/spark:3.3.0
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=2G
      - SPARK_WORKER_CORES=2
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - "8091:8081"
    volumes:
      - ./spark-data:/opt/spark/data
      - ./spark-apps:/opt/spark/apps
      - ./lab:/opt/airflow/lab
    depends_on:
      - spark-master

  spark-worker-2:
    image: bitnami/spark:3.3.0
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=2G
      - SPARK_WORKER_CORES=2
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - "8092:8081"
    volumes:
      - ./spark-data:/opt/spark/data
      - ./spark-apps:/opt/spark/apps
      - ./lab:/opt/airflow/lab
    depends_on:
      - spark-master

  # PostgreSQL 服务
  pgdatabase:
    image: postgres:13
    environment:
      - POSTGRES_USER=root
      - POSTGRES_PASSWORD=root
      - POSTGRES_DB=project1
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "root"]
      interval: 5s
      retries: 5

  pgadmin:
    image: dpage/pgadmin4
    environment:
      - PGADMIN_DEFAULT_EMAIL=admin@admin.com
      - PGADMIN_DEFAULT_PASSWORD=root
    ports:
      - "8080:80"
    volumes:
      - pgadmin_data:/var/lib/pgadmin  # 添加持久化存储
    depends_on:
      - pgdatabase

volumes:
  postgres_data:
    name: airflow_postgres_data
  pgadmin_data:  # 添加 pgAdmin 数据卷
    name: airflow_pgadmin_data
  postgres-db-volume: