diff --git a/roles/s3/README.md b/roles/s3/README.md new file mode 100644 index 000000000..7bd7b2a23 --- /dev/null +++ b/roles/s3/README.md @@ -0,0 +1,105 @@ +# S3 Role + +This role sets up an S3-compatible object storage cluster using SeaweedFS, integrated with the OpenConext environment. + +## Overview + +The role implements a distributed SeaweedFS cluster with the following components: + +- Multiple master servers for high availability and coordination +- Multiple volume servers for distributed storage +- Filer server for file metadata and directory structure +- S3 API gateway for S3-compatible access + +All services are deployed as Docker containers and integrated with the existing loadbalancer network. + +## Requirements + +- Docker must be installed on the target machine (handled by the docker role dependency) +- Python with docker module for Ansible + +## Configuration + +### Main Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `s3_base_dir` | Base directory for S3 installation | `/opt/openconext/seaweedfs` | +| `s3_config_dir` | Directory for configuration files | `{{ s3_base_dir }}/config` | +| `s3_data_dir` | Directory for data storage | `{{ s3_base_dir }}/data` | +| `s3_access_key` | S3 access key for the admin user | `admin` | +| `s3_secret_key` | S3 secret key for the admin user | Generated random string | +| `s3_readonly_access_key` | S3 access key for read-only user | `readonly` | +| `s3_readonly_secret_key` | S3 secret key for read-only user | Generated random string | +| `s3_cors_origin` | CORS origin allowed for S3 API | `*` | +| `s3_filer_domain` | Domain name for the filer service | `filer.{{ base_domain }}` | +| `s3_api_domain` | Domain name for the S3 API service | `s3.{{ base_domain }}` | + +## Integration with OpenConext + +The S3 role integrates with the existing OpenConext deployment by: + +1. Using the same loadbalancer network for container networking +2. Using Traefik for routing requests to the appropriate containers +3. Following the same pattern for container management as other OpenConext roles +4. Sharing the base domain configuration for consistent URL patterns +| `s3_filer_cpu_limit` | CPU limit for filer servers | `1.0` | +| `s3_api_memory_limit` | Memory limit for API server | `512M` | +| `s3_api_cpu_limit` | CPU limit for API server | `0.5` | +| `s3_proxy_memory_limit` | Memory limit for Nginx proxy | `256M` | +| `s3_proxy_cpu_limit` | CPU limit for Nginx proxy | `0.2` | + +### S3 Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `s3_address_style` | S3 addressing style | `path-style` | +| `s3_signature_version` | S3 signature version | `v4` | +| `s3_disable_bucket_policies` | Disable bucket policies | `false` | +| `s3_enable_head_dir_object` | Enable HeadDirObject API | `false` | +| `s3_enable_cors` | Enable CORS support | `true` | +| `s3_include_etag` | Include ETag in responses | `true` | +| `s3_multipart_upload_limits_mib` | Maximum multipart uploads in memory | `10000` | +| `s3_max_body_size` | Maximum upload size in Nginx | `500M` | + +## Usage + +Include the role in your playbook: + +```yaml +- hosts: s3_servers + roles: + - role: s3 + vars: + s3_host_address: "s3.example.com" + s3_admin_key: "your-access-key" + s3_admin_secret: "your-secret-key" +``` + +## Accessing the S3 Service + +After deployment, the following services will be available: + +- S3 API endpoint: http://s3_host_address/ +- Master admin UI: http://s3_host_address/master/ +- Filer admin UI: http://s3_host_address/filer/ +- Volume admin UIs: http://s3_host_address/volume1/, http://s3_host_address/volume2/, etc. +- Management API: http://s3_host_address/api/ + +Admin UIs require authentication using the configured `s3_auth_user` and `s3_auth_password`. + +## Using with AWS S3 CLI + +Example configuration: + +```bash +aws configure set aws_access_key_id +aws configure set aws_secret_access_key +aws configure set default.region +aws configure set default.s3.addressing_style path +aws --endpoint-url=http://s3_host_address s3 ls +``` + +## License + +This role is based on the [SeaWeedFS-HA-Demo](https://github.com/HarryKodden/SeaWeedFS-HA-Demo) repository. diff --git a/roles/s3/defaults/main.yml b/roles/s3/defaults/main.yml new file mode 100644 index 000000000..b7001944c --- /dev/null +++ b/roles/s3/defaults/main.yml @@ -0,0 +1,20 @@ +--- +# Default variables for the S3 role + +# Base configuration +s3_base_dir: /opt/openconext/seaweedfs +s3_config_dir: "{{ s3_base_dir }}/config" +s3_data_dir: "{{ s3_base_dir }}/data" + +# S3 credentials - these can be overridden by values in the secrets file +s3_access_key: "{{ s3_access_key | default('admin') }}" +s3_secret_key: "{{ s3_secret_key | default(lookup('password', '/tmp/s3_secret_key chars=ascii_letters,digits length=40')) }}" +s3_readonly_access_key: "{{ s3_readonly_access_key | default('readonly') }}" +s3_readonly_secret_key: "{{ s3_readonly_secret_key | default(lookup('password', '/tmp/s3_readonly_secret_key chars=ascii_letters,digits length=40')) }}" + +# CORS configuration +s3_cors_origin: "*" + +# Domain configuration for S3 endpoints +s3_filer_domain: "filer.{{ base_domain }}" +s3_api_domain: "s3.{{ base_domain }}" diff --git a/roles/s3/handlers/main-new.yml b/roles/s3/handlers/main-new.yml new file mode 100644 index 000000000..a9ab2272f --- /dev/null +++ b/roles/s3/handlers/main-new.yml @@ -0,0 +1,56 @@ +--- +# Handlers for the S3 role + +- name: restart master1 + community.docker.docker_container: + name: master1 + restart: true + +- name: restart master2 + community.docker.docker_container: + name: master2 + restart: true + +- name: restart master3 + community.docker.docker_container: + name: master3 + restart: true + +- name: restart volume1 + community.docker.docker_container: + name: volume1 + restart: true + +- name: restart volume2 + community.docker.docker_container: + name: volume2 + restart: true + +- name: restart volume3 + community.docker.docker_container: + name: volume3 + restart: true + +- name: restart filer + community.docker.docker_container: + name: filer + restart: true + +- name: restart s3 + community.docker.docker_container: + name: s3 + restart: true + +- name: restart all s3 services + community.docker.docker_container: + name: "{{ item }}" + restart: true + with_items: + - master1 + - master2 + - master3 + - volume1 + - volume2 + - volume3 + - filer + - s3 diff --git a/roles/s3/handlers/main.yml b/roles/s3/handlers/main.yml new file mode 100644 index 000000000..ec265bf10 --- /dev/null +++ b/roles/s3/handlers/main.yml @@ -0,0 +1,56 @@ +--- +# Handlers for the S3 role + +- name: restart master1 + community.docker.docker_container: + name: master1 + restart: true + +- name: restart master2 + community.docker.docker_container: + name: master2 + restart: true + +- name: restart master3 + community.docker.docker_container: + name: master3 + restart: true + +- name: restart volume1 + community.docker.docker_container: + name: volume1 + restart: true + +- name: restart volume2 + community.docker.docker_container: + name: volume2 + restart: true + +- name: restart volume3 + community.docker.docker_container: + name: volume3 + restart: true + +- name: restart filer + community.docker.docker_container: + name: filer + restart: true + +- name: restart s3 + community.docker.docker_container: + name: s3 + restart: true + +- name: restart all s3 services + community.docker.docker_container: + name: "{{ item }}" + restart: true + with_items: + - master1 + - master2 + - master3 + - volume1 + - volume2 + - volume3 + - filer + - s3 diff --git a/roles/s3/meta/main.yml b/roles/s3/meta/main.yml new file mode 100644 index 000000000..867c1de2e --- /dev/null +++ b/roles/s3/meta/main.yml @@ -0,0 +1,32 @@ +--- +galaxy_info: + role_name: s3 + author: OpenConext + description: Deploys SeaweedFS S3 cluster + company: OpenConext + license: Apache-2.0 + min_ansible_version: "2.9" + + platforms: + - name: EL + versions: + - "7" + - "8" + - "9" + - name: Debian + versions: + - all + - name: Ubuntu + versions: + - all + + galaxy_tags: + - seaweedfs + - s3 + - storage + - filesystem + - distributed + - cloud + +dependencies: +- role: docker diff --git a/roles/s3/tasks/main.yml b/roles/s3/tasks/main.yml new file mode 100644 index 000000000..f6035db73 --- /dev/null +++ b/roles/s3/tasks/main.yml @@ -0,0 +1,237 @@ +--- +# S3 role using SeaweedFS +# Based on https://github.com/HarryKodden/SeaWeedFS-HA-Demo + +- name: Include OS-specific variables + include_vars: "{{ ansible_os_family }}.yml" + tags: + - s3 + - s3-install + +- name: Install required packages + ansible.builtin.package: + name: "{{ s3_packages }}" + state: present + tags: + - s3 + - s3-install + +- name: Create required directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + with_items: + - "/opt/openconext/seaweedfs" + - "/opt/openconext/seaweedfs/config" + - "/opt/openconext/seaweedfs/data/master1" + - "/opt/openconext/seaweedfs/data/master2" + - "/opt/openconext/seaweedfs/data/master3" + - "/opt/openconext/seaweedfs/data/volume1" + - "/opt/openconext/seaweedfs/data/volume2" + - "/opt/openconext/seaweedfs/data/volume3" + - "/opt/openconext/seaweedfs/data/filer" + tags: + - s3 + - s3-install + +- name: Create S3 JSON configuration + ansible.builtin.template: + src: s3.json.j2 + dest: "/opt/openconext/seaweedfs/config/s3.json" + owner: root + group: root + mode: "0644" + tags: + - s3 + - s3-config + +# Deploy master nodes +- name: Create and start master1 container + community.docker.docker_container: + name: master1 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "master -port=9333 -ip=master1 -mdir=/data -peers=master1:9333,master2:9333,master3:9333" + volumes: + - "/opt/openconext/seaweedfs/data/master1:/data" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://master1:9333/" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + tags: + - s3 + - s3-deploy + +- name: Create and start master2 container + community.docker.docker_container: + name: master2 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "master -port=9333 -ip=master2 -mdir=/data -peers=master1:9333,master2:9333,master3:9333" + volumes: + - "/opt/openconext/seaweedfs/data/master2:/data" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://master2:9333/" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + tags: + - s3 + - s3-deploy + +- name: Create and start master3 container + community.docker.docker_container: + name: master3 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "master -port=9333 -ip=master3 -mdir=/data -peers=master1:9333,master2:9333,master3:9333" + volumes: + - "/opt/openconext/seaweedfs/data/master3:/data" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://master3:9333/" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + tags: + - s3 + - s3-deploy + +# Deploy volume nodes +- name: Create and start volume1 container + community.docker.docker_container: + name: volume1 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "volume -port=8080 -ip=volume1 -mserver=master1:9333,master2:9333,master3:9333 -dir=/data -max=5 -rack=rack1" + volumes: + - "/opt/openconext/seaweedfs/data/volume1:/data" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://volume1:8080/status" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 5s + tags: + - s3 + - s3-deploy + +- name: Create and start volume2 container + community.docker.docker_container: + name: volume2 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "volume -port=8080 -ip=volume2 -mserver=master1:9333,master2:9333,master3:9333 -dir=/data -max=5 -rack=rack1" + volumes: + - "/opt/openconext/seaweedfs/data/volume2:/data" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://volume2:8080/status" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 5s + tags: + - s3 + - s3-deploy + +- name: Create and start volume3 container + community.docker.docker_container: + name: volume3 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "volume -port=8080 -ip=volume3 -mserver=master1:9333,master2:9333,master3:9333 -dir=/data -max=5 -rack=rack1" + volumes: + - "/opt/openconext/seaweedfs/data/volume3:/data" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://volume3:8080/status" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 5s + tags: + - s3 + - s3-deploy + +# Deploy filer +- name: Create and start filer container + community.docker.docker_container: + name: filer + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "filer -master=master1:9333,master2:9333,master3:9333 -ip=0.0.0.0 -port=8888 -dataCenter=dc1 -maxMB=1024" + volumes: + - "/opt/openconext/seaweedfs/data/filer:/data" + labels: + traefik.http.routers.filer.rule: "Host(`filer.{{ base_domain }}`)" + traefik.http.routers.filer.tls: "true" + traefik.enable: "true" + healthcheck: + test: [ "CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8888/" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 5s + tags: + - s3 + - s3-deploy + +# Deploy S3 API endpoint +- name: Create and start S3 container + community.docker.docker_container: + name: s3 + image: chrislusf/seaweedfs:latest + pull: true + restart_policy: "always" + state: started + networks: + - name: "loadbalancer" + command: "s3 -filer=filer:8888 -ip.bind=0.0.0.0 -port=8333 -config=/etc/seaweedfs/s3.json" + volumes: + - "/opt/openconext/seaweedfs/config/s3.json:/etc/seaweedfs/s3.json:ro" + labels: + traefik.http.routers.s3.rule: "Host(`s3.{{ base_domain }}`)" + traefik.http.routers.s3.tls: "true" + traefik.enable: "true" + healthcheck: + test: [ "CMD", "sh", "-c", "netstat -tlnp | grep :8333" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 10s + tags: + - s3 + - s3-deploy diff --git a/roles/s3/templates/s3.json.j2 b/roles/s3/templates/s3.json.j2 new file mode 100644 index 000000000..7c8d9ac82 --- /dev/null +++ b/roles/s3/templates/s3.json.j2 @@ -0,0 +1,30 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "{{ s3_access_key | default('admin') }}", + "secretKey": "{{ s3_secret_key | default('password') }}" + } + ], + "actions": ["Admin", "Read", "Write"] + }, + { + "name": "readonly", + "credentials": [ + { + "accessKey": "{{ s3_readonly_access_key | default('readonly') }}", + "secretKey": "{{ s3_readonly_secret_key | default('password') }}" + } + ], + "actions": ["Read"] + } + ], + "domains": [ + { + "name": "*", + "allowedOrigins": ["{{ s3_cors_origin | default('*') }}"] + } + ] +} diff --git a/roles/s3/vars/Debian.yml b/roles/s3/vars/Debian.yml new file mode 100644 index 000000000..8b2348718 --- /dev/null +++ b/roles/s3/vars/Debian.yml @@ -0,0 +1,16 @@ +--- +# Debian family specific variables for s3 role + +s3_packages: +- docker.io +- python3-docker + +s3_services: +- docker + +# System dependencies +seaweedfs_dependencies: +- wget +- curl +- jq +- python3 diff --git a/roles/s3/vars/RedHat.yml b/roles/s3/vars/RedHat.yml new file mode 100644 index 000000000..7e26a8c61 --- /dev/null +++ b/roles/s3/vars/RedHat.yml @@ -0,0 +1,16 @@ +--- +# RedHat family specific variables for s3 role + +s3_packages: +- docker +- python3-docker + +s3_services: +- docker + +# System dependencies +seaweedfs_dependencies: +- wget +- curl +- jq +- python3