Publish tulip docker images to GitHub packages

.env.example

@@ -2,6 +2,8 @@
 # Tulip config
 ##############################
 
+SERVICES='[{"ip":"10.10.3.1","port":9876,"name":"Service 1"},{"ip":"10.10.3.1","port":5000,"name":"Service 2"}]'
+
 # Timescale connection
 TIMESCALE="postgres://tulip@timescale:5432/tulip"
 
@@ -66,6 +68,11 @@ DUMP_PCAPS_FILENAME="2006-01-02_15-04-05.pcap"
 FLAGID_SCRAPE=
 #FLAGID_SCRAPE=1
 
+# Enable local build
+# Empty value = disabled
+LOCAL_BUILD=
+#LOCAL_BUILD=1
+
 # Enable flagid scanning - Tags flag ids in traffic
 # Empty value = disabled
 # Does nothing unless FLAGID_SCRAPE is set

.github/workflows/build.yaml

@@ -0,0 +1,63 @@
+name: Build Docker images
+on:
+  push:
+    branches:
+      - master
+  release:
+    types: [ published ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build and publish a Docker image for ${{ github.repository }}-api
+        uses: macbre/push-to-ghcr@master
+        with:
+          image_name: ${{ github.repository }}-api
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dockerfile: services/api/Dockerfile-api
+          context: services/api
+
+      - name: Build and publish a Docker image for ${{ github.repository }}-flagids
+        uses: macbre/push-to-ghcr@master
+        with:
+          image_name: ${{ github.repository }}-flagids
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dockerfile: services/flagids/Dockerfile
+          context: services/flagids
+
+      - name: Build and publish a Docker image for ${{ github.repository }}-assembler
+        uses: macbre/push-to-ghcr@master
+        with:
+          image_name: ${{ github.repository }}-assembler
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dockerfile: services/go-importer/Dockerfile-assembler
+          context: services/go-importer
+
+      - name: Build and publish a Docker image for ${{ github.repository }}-enricher
+        uses: macbre/push-to-ghcr@master
+        with:
+          image_name: ${{ github.repository }}-enricher
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dockerfile: services/go-importer/Dockerfile-enricher
+          context: services/go-importer
+
+      - name: Build and publish a Docker image for ${{ github.repository }}-timescale
+        uses: macbre/push-to-ghcr@master
+        with:
+          image_name: ${{ github.repository }}-timescale
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dockerfile: services/timescale/Dockerfile
+          context: services/timescale
+
+      - name: Build and publish a Docker image for ${{ github.repository }}-frontend
+        uses: macbre/push-to-ghcr@master
+        with:
+          image_name: ${{ github.repository }}-frontend
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          dockerfile: frontend/Dockerfile-frontend
+          context: frontend

docker-compose-local.yml

@@ -0,0 +1,137 @@
+version: "3.5"
+services:
+  timescale:
+    build: services/timescale
+    image: tulip-timescale:latest
+    restart: unless-stopped
+    volumes:
+    - timescale-data:/var/lib/postgresql/data
+    - ./services/schema/system.sql:/docker-entrypoint-initdb.d/100_system.sql:ro
+    - ./services/schema/functions.sql:/docker-entrypoint-initdb.d/101_functions.sql:ro
+    - ./services/schema/schema.sql:/docker-entrypoint-initdb.d/102_schema.sql:ro
+    networks:
+      - internal
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_USER: tulip
+      POSTGRES_DB: tulip
+    # This does not need to be adjusted, unless you actually want to limit it
+    # Postgres uses shared memory for caching, and docker assigns just 64 MB by default
+    shm_size: '128g'
+
+  frontend:
+    build:
+      context: frontend
+      dockerfile: Dockerfile-frontend
+    image: tulip-frontend:latest
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    expose:
+      - 3000
+    depends_on:
+      - timescale
+      - api
+    networks:
+      - internal
+    environment:
+      API_SERVER_ENDPOINT: http://api:5000/
+      VIRTUAL_HOST: tulip.h4xx.eu
+
+  api:
+    build:
+      context: services/api
+      dockerfile: Dockerfile-api
+    image: tulip-api:latest
+    restart: unless-stopped
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
+    environment:
+      SERVICES: ${SERVICES}
+      TIMESCALE: ${TIMESCALE}
+      TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
+      FLAG_REGEX: ${FLAG_REGEX}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      VM_IP: ${VM_IP}
+
+  flagids:
+    restart: unless-stopped
+    build:
+      context: services/flagids
+    image: tulip-flagids:latest
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    environment:
+      TIMESCALE: ${TIMESCALE}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCRAPE: ${FLAGID_SCRAPE}
+      TEAM_ID: ${TEAM_ID}
+      FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
+      VISUALIZER_URL: ${VISUALIZER_URL}
+      DUMP_PCAPS: ${DUMP_PCAPS}
+
+  assembler:
+    build:
+      context: services/go-importer
+      dockerfile: Dockerfile-assembler
+    image: tulip-assembler:latest
+    restart: unless-stopped
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
+    # Command line flags most likely to fix a tulip issue:
+    # - -http-session-tracking: enable HTTP session tracking
+    # - -dir: directory to read traffic from
+    # - -skipchecksum: skip checksum validation
+    # - -flush-after: i.e. 2m Not needed in pcap rotation mode
+    # - -disable-converters: disable converters
+    # - -discard-extra-data: dont split large flow items, just discard them
+    command: "./assembler -http-session-tracking -skipchecksum -disable-converters -dir ${TRAFFIC_DIR_DOCKER}"
+    environment:
+      TIMESCALE: ${TIMESCALE}
+      FLAG_REGEX: ${FLAG_REGEX}
+      TICK_START: ${TICK_START}
+      TICK_LENGTH: ${TICK_LENGTH}
+      FLAGID_SCAN: ${FLAGID_SCAN}
+      FLAG_LIFETIME: ${FLAG_LIFETIME}
+      FLAG_VALIDATOR_TYPE: ${FLAG_VALIDATOR_TYPE}
+      FLAG_VALIDATOR_TEAM: ${FLAG_VALIDATOR_TEAM}
+      PCAP_OVER_IP: ${PCAP_OVER_IP}
+      DUMP_PCAPS: ${DUMP_PCAPS}
+      DUMP_PCAPS_INTERVAL: ${DUMP_PCAPS_INTERVAL}
+      DUMP_PCAPS_FILENAME: ${DUMP_PCAPS_FILENAME}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  enricher:
+    build:
+      context: services/go-importer
+      dockerfile: Dockerfile-enricher
+    image: tulip-enricher:latest
+    restart: unless-stopped
+    depends_on:
+      - timescale
+    networks:
+      - internal
+    volumes:
+      - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
+    command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json"
+    environment:
+      TIMESCALE: ${TIMESCALE}
+
+volumes:
+  timescale-data:
+
+networks:
+  internal:

docker-compose-test.yml

@@ -37,6 +37,7 @@ services:
     volumes:
       - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
     environment:
+      SERVICES: ${SERVICES}
       TULIP_MONGO: ${TULIP_MONGO}
       TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
       FLAG_REGEX: ${FLAG_REGEX}

docker-compose.yml

@@ -1,8 +1,7 @@
 version: "3.5"
 services:
   timescale:
-    build: services/timescale
-    image: tulip-timescale:latest
+    image: ghcr.io/openattackdefensetools/tulip-timescale:latest
     restart: unless-stopped
     volumes:
     - timescale-data:/var/lib/postgresql/data
@@ -20,10 +19,7 @@ services:
     shm_size: '128g'
 
   frontend:
-    build:
-      context: frontend
-      dockerfile: Dockerfile-frontend
-    image: tulip-frontend:latest
+    image: ghcr.io/openattackdefensetools/tulip-frontend:latest
     restart: unless-stopped
     ports:
       - "3000:3000"
@@ -39,10 +35,7 @@ services:
       VIRTUAL_HOST: tulip.h4xx.eu
 
   api:
-    build:
-      context: services/api
-      dockerfile: Dockerfile-api
-    image: tulip-api:latest
+    image: ghcr.io/openattackdefensetools/tulip-api:latest
     restart: unless-stopped
     depends_on:
       - timescale
@@ -51,6 +44,7 @@ services:
     volumes:
       - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z
     environment:
+      SERVICES: ${SERVICES}
       TIMESCALE: ${TIMESCALE}
       TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
       FLAG_REGEX: ${FLAG_REGEX}
@@ -60,9 +54,7 @@ services:
 
   flagids:
     restart: unless-stopped
-    build:
-      context: services/flagids
-    image: tulip-flagids:latest
+    image: ghcr.io/openattackdefensetools/tulip-flagids:latest
     depends_on:
       - timescale
     networks:
@@ -78,10 +70,7 @@ services:
       DUMP_PCAPS: ${DUMP_PCAPS}
 
   assembler:
-    build:
-      context: services/go-importer
-      dockerfile: Dockerfile-assembler
-    image: tulip-assembler:latest
+    image: ghcr.io/openattackdefensetools/tulip-assembler:latest
     restart: unless-stopped
     depends_on:
       - timescale
@@ -114,10 +103,7 @@ services:
       - "host.docker.internal:host-gateway"
 
   enricher:
-    build:
-      context: services/go-importer
-      dockerfile: Dockerfile-enricher
-    image: tulip-enricher:latest
+    image: ghcr.io/openattackdefensetools/tulip-enricher:latest
     restart: unless-stopped
     depends_on:
       - timescale

services/api/configurations.py

@@ -23,6 +23,7 @@
 # along with Flower.  If not, see <https://www.gnu.org/licenses/>.
 
 import os
+import json
 from pathlib import Path
 
 traffic_dir = Path(os.getenv("TULIP_TRAFFIC_DIR", "/traffic"))
@@ -34,8 +35,11 @@
 vm_ip = os.getenv("VM_IP", "10.10.3.1")
 visualizer_url = os.getenv("VISUALIZER_URL", "http://127.0.0.1:1337")
 
-vm_ip_1 = "10.60.2.1"
-helper = '''
+if os.getenv("SERVICES"):
+    services = json.loads(os.environ["SERVICES"])
+else:
+    vm_ip_1 = "10.60.2.1"
+    helper = '''
 10.61.5.1:1237 CyberUni 4
 10.61.5.1:1236 CyberUni 3
 10.61.5.1:1235 CyberUni 1
@@ -45,6 +49,12 @@
 10.62.5.1:5000 Trademark
 10.63.5.1:1337 RPN
 '''
-
-services = [{"ip": x.split(" ")[0].split(":")[0], "port": int(x.split(" ")[0].split(":")[1]), "name": " ".join(x.split(" ")[1:])} for x in helper.strip().split("\n")]
-services += [{"ip": vm_ip_1, "port": -1, "name": "other"}]
+    services = [
+        {
+            "ip": x.split(" ")[0].split(":")[0],
+            "port": int(x.split(" ")[0].split(":")[1]),
+            "name": " ".join(x.split(" ")[1:])
+        }
+        for x in helper.strip().split("\n")
+    ]
+    services += [{"ip": vm_ip_1, "port": -1, "name": "other"}]

start.sh

@@ -4,7 +4,9 @@ source .env
 
 if [ -n "$FLAGID_SCRAPE" ]; then
   docker-compose -f docker-compose-flagid.yml up;
+elif [ -n "$LOCAL_BUILD" ]; then
+  docker-compose -f docker-compose-local.yml up
 else
-  docker-compose up 
+  docker-compose up
 fi