From 4b0ea3f9d1318a43432b5e628e4f977a3aa97aa5 Mon Sep 17 00:00:00 2001 From: JPaja Date: Sun, 10 Aug 2025 17:26:32 +0200 Subject: [PATCH 1/3] Add: Build docker images Fix image names --- .github/workflows/build.yaml | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..62fb50a5 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,63 @@ +name: Build Docker images +on: + push: + branches: + - master + release: + types: [ published ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build and publish a Docker image for ${{ github.repository }}-api + uses: macbre/push-to-ghcr@master + with: + image_name: ${{ github.repository }}-api + github_token: ${{ secrets.GITHUB_TOKEN }} + dockerfile: services/api/Dockerfile-api + context: services/api + + - name: Build and publish a Docker image for ${{ github.repository }}-flagids + uses: macbre/push-to-ghcr@master + with: + image_name: ${{ github.repository }}-flagids + github_token: ${{ secrets.GITHUB_TOKEN }} + dockerfile: services/flagids/Dockerfile + context: services/flagids + + - name: Build and publish a Docker image for ${{ github.repository }}-assembler + uses: macbre/push-to-ghcr@master + with: + image_name: ${{ github.repository }}-assembler + github_token: ${{ secrets.GITHUB_TOKEN }} + dockerfile: services/go-importer/Dockerfile-assembler + context: services/go-importer + + - name: Build and publish a Docker image for ${{ github.repository }}-enricher + uses: macbre/push-to-ghcr@master + with: + image_name: ${{ github.repository }}-enricher + github_token: ${{ secrets.GITHUB_TOKEN }} + dockerfile: services/go-importer/Dockerfile-enricher + context: services/go-importer + + - name: Build and publish a Docker image for ${{ github.repository }}-timescale + uses: macbre/push-to-ghcr@master + with: + image_name: ${{ github.repository }}-timescale + github_token: ${{ secrets.GITHUB_TOKEN }} + dockerfile: services/timescale/Dockerfile + context: services/timescale + + - name: Build and publish a Docker image for ${{ github.repository }}-frontend + uses: macbre/push-to-ghcr@master + with: + image_name: ${{ github.repository }}-frontend + github_token: ${{ secrets.GITHUB_TOKEN }} + dockerfile: frontend/Dockerfile-frontend + context: frontend \ No newline at end of file From 28c67a2fb6c23ec0ad9d754b6a4979e3d71f1eea Mon Sep 17 00:00:00 2001 From: JPaja Date: Sun, 10 Aug 2025 18:11:26 +0200 Subject: [PATCH 2/3] Update docker-compose to pull public ghcr containers --- docker-compose-local.yml | 136 +++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 27 ++------ 2 files changed, 142 insertions(+), 21 deletions(-) create mode 100644 docker-compose-local.yml diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 00000000..19be2b57 --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,136 @@ +version: "3.5" +services: + timescale: + build: services/timescale + image: tulip-timescale:latest + restart: unless-stopped + volumes: + - timescale-data:/var/lib/postgresql/data + - ./services/schema/system.sql:/docker-entrypoint-initdb.d/100_system.sql:ro + - ./services/schema/functions.sql:/docker-entrypoint-initdb.d/101_functions.sql:ro + - ./services/schema/schema.sql:/docker-entrypoint-initdb.d/102_schema.sql:ro + networks: + - internal + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: tulip + POSTGRES_DB: tulip + # This does not need to be adjusted, unless you actually want to limit it + # Postgres uses shared memory for caching, and docker assigns just 64 MB by default + shm_size: '128g' + + frontend: + build: + context: frontend + dockerfile: Dockerfile-frontend + image: tulip-frontend:latest + restart: unless-stopped + ports: + - "3000:3000" + expose: + - 3000 + depends_on: + - timescale + - api + networks: + - internal + environment: + API_SERVER_ENDPOINT: http://api:5000/ + VIRTUAL_HOST: tulip.h4xx.eu + + api: + build: + context: services/api + dockerfile: Dockerfile-api + image: tulip-api:latest + restart: unless-stopped + depends_on: + - timescale + networks: + - internal + volumes: + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z + environment: + TIMESCALE: ${TIMESCALE} + TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} + FLAG_REGEX: ${FLAG_REGEX} + TICK_START: ${TICK_START} + TICK_LENGTH: ${TICK_LENGTH} + VM_IP: ${VM_IP} + + flagids: + restart: unless-stopped + build: + context: services/flagids + image: tulip-flagids:latest + depends_on: + - timescale + networks: + - internal + environment: + TIMESCALE: ${TIMESCALE} + TICK_START: ${TICK_START} + TICK_LENGTH: ${TICK_LENGTH} + FLAGID_SCRAPE: ${FLAGID_SCRAPE} + TEAM_ID: ${TEAM_ID} + FLAGID_ENDPOINT: ${FLAGID_ENDPOINT} + VISUALIZER_URL: ${VISUALIZER_URL} + DUMP_PCAPS: ${DUMP_PCAPS} + + assembler: + build: + context: services/go-importer + dockerfile: Dockerfile-assembler + image: tulip-assembler:latest + restart: unless-stopped + depends_on: + - timescale + networks: + - internal + volumes: + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z + # Command line flags most likely to fix a tulip issue: + # - -http-session-tracking: enable HTTP session tracking + # - -dir: directory to read traffic from + # - -skipchecksum: skip checksum validation + # - -flush-after: i.e. 2m Not needed in pcap rotation mode + # - -disable-converters: disable converters + # - -discard-extra-data: dont split large flow items, just discard them + command: "./assembler -http-session-tracking -skipchecksum -disable-converters -dir ${TRAFFIC_DIR_DOCKER}" + environment: + TIMESCALE: ${TIMESCALE} + FLAG_REGEX: ${FLAG_REGEX} + TICK_START: ${TICK_START} + TICK_LENGTH: ${TICK_LENGTH} + FLAGID_SCAN: ${FLAGID_SCAN} + FLAG_LIFETIME: ${FLAG_LIFETIME} + FLAG_VALIDATOR_TYPE: ${FLAG_VALIDATOR_TYPE} + FLAG_VALIDATOR_TEAM: ${FLAG_VALIDATOR_TEAM} + PCAP_OVER_IP: ${PCAP_OVER_IP} + DUMP_PCAPS: ${DUMP_PCAPS} + DUMP_PCAPS_INTERVAL: ${DUMP_PCAPS_INTERVAL} + DUMP_PCAPS_FILENAME: ${DUMP_PCAPS_FILENAME} + extra_hosts: + - "host.docker.internal:host-gateway" + + enricher: + build: + context: services/go-importer + dockerfile: Dockerfile-enricher + image: tulip-enricher:latest + restart: unless-stopped + depends_on: + - timescale + networks: + - internal + volumes: + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z + command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json" + environment: + TIMESCALE: ${TIMESCALE} + +volumes: + timescale-data: + +networks: + internal: diff --git a/docker-compose.yml b/docker-compose.yml index 19be2b57..5c95d9bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,7 @@ version: "3.5" services: timescale: - build: services/timescale - image: tulip-timescale:latest + image: ghcr.io/openattackdefensetools/tulip-timescale:latest restart: unless-stopped volumes: - timescale-data:/var/lib/postgresql/data @@ -20,10 +19,7 @@ services: shm_size: '128g' frontend: - build: - context: frontend - dockerfile: Dockerfile-frontend - image: tulip-frontend:latest + image: ghcr.io/openattackdefensetools/tulip-frontend:latest restart: unless-stopped ports: - "3000:3000" @@ -39,10 +35,7 @@ services: VIRTUAL_HOST: tulip.h4xx.eu api: - build: - context: services/api - dockerfile: Dockerfile-api - image: tulip-api:latest + image: ghcr.io/openattackdefensetools/tulip-api:latest restart: unless-stopped depends_on: - timescale @@ -60,9 +53,7 @@ services: flagids: restart: unless-stopped - build: - context: services/flagids - image: tulip-flagids:latest + image: ghcr.io/openattackdefensetools/tulip-flagids:latest depends_on: - timescale networks: @@ -78,10 +69,7 @@ services: DUMP_PCAPS: ${DUMP_PCAPS} assembler: - build: - context: services/go-importer - dockerfile: Dockerfile-assembler - image: tulip-assembler:latest + image: ghcr.io/openattackdefensetools/tulip-assembler:latest restart: unless-stopped depends_on: - timescale @@ -114,10 +102,7 @@ services: - "host.docker.internal:host-gateway" enricher: - build: - context: services/go-importer - dockerfile: Dockerfile-enricher - image: tulip-enricher:latest + image: ghcr.io/openattackdefensetools/tulip-enricher:latest restart: unless-stopped depends_on: - timescale From 49a664a582921fd4c6240f9f52a4dd8a93dff235 Mon Sep 17 00:00:00 2001 From: JPaja Date: Sun, 10 Aug 2025 19:21:01 +0200 Subject: [PATCH 3/3] Expose services to env variable --- .env.example | 7 +++++++ docker-compose-local.yml | 1 + docker-compose-test.yml | 1 + docker-compose.yml | 1 + services/api/configurations.py | 20 +++++++++++++++----- start.sh | 4 +++- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index cb488a64..3ca0e7f5 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ # Tulip config ############################## +SERVICES='[{"ip":"10.10.3.1","port":9876,"name":"Service 1"},{"ip":"10.10.3.1","port":5000,"name":"Service 2"}]' + # Timescale connection TIMESCALE="postgres://tulip@timescale:5432/tulip" @@ -66,6 +68,11 @@ DUMP_PCAPS_FILENAME="2006-01-02_15-04-05.pcap" FLAGID_SCRAPE= #FLAGID_SCRAPE=1 +# Enable local build +# Empty value = disabled +LOCAL_BUILD= +#LOCAL_BUILD=1 + # Enable flagid scanning - Tags flag ids in traffic # Empty value = disabled # Does nothing unless FLAGID_SCRAPE is set diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 19be2b57..5d8e05e6 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -51,6 +51,7 @@ services: volumes: - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z environment: + SERVICES: ${SERVICES} TIMESCALE: ${TIMESCALE} TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} FLAG_REGEX: ${FLAG_REGEX} diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 9b8bdef5..9a876833 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -37,6 +37,7 @@ services: volumes: - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro environment: + SERVICES: ${SERVICES} TULIP_MONGO: ${TULIP_MONGO} TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} FLAG_REGEX: ${FLAG_REGEX} diff --git a/docker-compose.yml b/docker-compose.yml index 5c95d9bc..7d6cc1c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,7 @@ services: volumes: - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z environment: + SERVICES: ${SERVICES} TIMESCALE: ${TIMESCALE} TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} FLAG_REGEX: ${FLAG_REGEX} diff --git a/services/api/configurations.py b/services/api/configurations.py index 12bb9a84..c7cf828d 100644 --- a/services/api/configurations.py +++ b/services/api/configurations.py @@ -23,6 +23,7 @@ # along with Flower. If not, see . import os +import json from pathlib import Path traffic_dir = Path(os.getenv("TULIP_TRAFFIC_DIR", "/traffic")) @@ -34,8 +35,11 @@ vm_ip = os.getenv("VM_IP", "10.10.3.1") visualizer_url = os.getenv("VISUALIZER_URL", "http://127.0.0.1:1337") -vm_ip_1 = "10.60.2.1" -helper = ''' +if os.getenv("SERVICES"): + services = json.loads(os.environ["SERVICES"]) +else: + vm_ip_1 = "10.60.2.1" + helper = ''' 10.61.5.1:1237 CyberUni 4 10.61.5.1:1236 CyberUni 3 10.61.5.1:1235 CyberUni 1 @@ -45,6 +49,12 @@ 10.62.5.1:5000 Trademark 10.63.5.1:1337 RPN ''' - -services = [{"ip": x.split(" ")[0].split(":")[0], "port": int(x.split(" ")[0].split(":")[1]), "name": " ".join(x.split(" ")[1:])} for x in helper.strip().split("\n")] -services += [{"ip": vm_ip_1, "port": -1, "name": "other"}] + services = [ + { + "ip": x.split(" ")[0].split(":")[0], + "port": int(x.split(" ")[0].split(":")[1]), + "name": " ".join(x.split(" ")[1:]) + } + for x in helper.strip().split("\n") + ] + services += [{"ip": vm_ip_1, "port": -1, "name": "other"}] diff --git a/start.sh b/start.sh index eb984adb..c70451c2 100755 --- a/start.sh +++ b/start.sh @@ -4,7 +4,9 @@ source .env if [ -n "$FLAGID_SCRAPE" ]; then docker-compose -f docker-compose-flagid.yml up; +elif [ -n "$LOCAL_BUILD" ]; then + docker-compose -f docker-compose-local.yml up else - docker-compose up + docker-compose up fi