fix last details

eclipsevortex · eclipsevortex · commit c08eb401adc2 · 2025-04-20T13:03:28.000+01:00
diff --git a/.github/workflows/docker-workflow.yml b/.github/workflows/docker-workflow.yml
@@ -1,6 +1,7 @@
 name: Docker Build & Push All Services
 
 on:
+  delete:
   push:
     tags:
       - "v*.*.*"
@@ -20,22 +21,6 @@ jobs:
       - name: 🧠 Discover components and services
         id: set-matrix
         run: |
-          # Customize these lists as needed
-          EXCLUDED_COMPONENTS=(core validator)
-          EXCLUDED_SERVICES=()
-
-          should_exclude() {
-            local item=$1
-            shift
-            local list=("$@")
-            for ex in "${list[@]}"; do
-              if [[ "$item" == "$ex" ]]; then
-                return 0
-              fi
-            done
-            return 1
-          }
-
           mkdir -p .build/tmp_matrix
           echo '{ "include": [' > .build/tmp_matrix/matrix.json
           FIRST=true
@@ -44,32 +29,15 @@ jobs:
             [ -d "$comp" ] || continue
             comp_name=$(basename "$comp")
 
-            # 🔥 Skip excluded components
-            if should_exclude "$comp_name" "${EXCLUDED_COMPONENTS[@]}"; then
-              echo "⏭️ Skipping excluded component: $comp_name"
-              continue
-            fi
-
-            for service in "$comp"/*; do
-              [ -d "$service" ] || continue
-              service_name=$(basename "$service")
-
-              # 🔥 Skip excluded services
-              if should_exclude "$service_name" "${EXCLUDED_SERVICES[@]}"; then
-                echo "⏭️ Skipping excluded service: $service_name"
-                continue
+            # ✅ Include only if it has a pyproject or version.py
+            if [[ -f "$comp/pyproject.toml" || -f "$comp/version.py" ]]; then
+              if [ "$FIRST" = true ]; then
+                FIRST=false
+              else
+                echo "," >> .build/tmp_matrix/matrix.json
               fi
-
-              # ✅ Include only if it has a pyproject or version.py
-              if [[ -f "$service/pyproject.toml" || -f "$service/version.py" ]]; then
-                if [ "$FIRST" = true ]; then
-                  FIRST=false
-                else
-                  echo "," >> .build/tmp_matrix/matrix.json
-                fi
-                echo "  { \"component\": \"$comp_name\", \"service\": \"$service_name\" }" >> .build/tmp_matrix/matrix.json
-              fi
-            done
+              echo "  { \"component\": \"$comp_name\" }" >> .build/tmp_matrix/matrix.json
+            fi
           done
 
           echo "] }" >> .build/tmp_matrix/matrix.json
@@ -124,6 +92,7 @@ jobs:
           fi
 
   build:
+    if: github.event_name == 'push' || github.event_name == 'delete'
     needs: [discover, wheel-builder]
     runs-on: ubuntu-latest
     permissions:
@@ -166,30 +135,48 @@ jobs:
       - name: 🚀 Build and push version-tagged image (on tag push only)
         if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
         run: |
-          COMP=${{ matrix.component }}
-          SERVICE=${{ matrix.service }}
-          IMAGE="subvortex/subvortex-$COMP-$SERVICE"
+          COMP="${{ matrix.component }}"
+          REPO_NAME="subvortex-${COMP//_/-}" && echo "$COMP"
+          IMAGE="subvortex/$REPO_NAME"
           WHEEL_IMAGE="${{ needs.wheel-builder.outputs.tag }}"
-          RAW_VERSION_TAG="${{ steps.taginfo.outputs.version_tag }}"
-          VERSION_TAG="${RAW_VERSION_TAG#v}"
-          DOCKERFILE="subvortex/$COMP/$SERVICE/Dockerfile"
+          VERSION_TAG="${{ steps.taginfo.outputs.version_tag }}"
+          VERSION="${VERSION_TAG#v}"
+          DOCKERFILE="subvortex/$COMP/Dockerfile"
+
+           echo "🔍 Searching for component version... $COMP / $IMAGE"
+          COMPONENT_PATH="subvortex/$COMP"
+          if [ -f "$COMPONENT_PATH/pyproject.toml" ]; then
+            echo "✅ Found pyproject.toml"
+            COMPONENT_VERSION=$(grep -E '^version\s*=' "$COMPONENT_PATH/pyproject.toml" | head -1 | sed -E 's/version\s*=\s*"([^"]+)"/\1/')
+          elif [ -f "$COMPONENT_PATH/version.py" ]; then
+            echo "✅ Found version.py"
+            COMPONENT_VERSION=$(python -c "import ast; f=open('$COMPONENT_PATH/version.py'); print([n.value.s for n in ast.walk(ast.parse(f.read())) if isinstance(n, ast.Assign) and n.targets[0].id == '__version__'][0])")
+          else
+            echo "❌ No version file found for component"
+            exit 1
+          fi
+
+          echo "🧾 Final versions:"
+          echo "VERSION=$VERSION"
+          echo "COMPONENT_VERSION=$COMPONENT_VERSION"
 
-          echo "🚀 Building image $IMAGE:$VERSION_TAG"
+          echo "🚀 Building image $IMAGE:$VERSION"
 
           docker buildx build \
             --squash \
             --platform linux/amd64,linux/arm64 \
             --build-context wheelbuilder=docker-image://$WHEEL_IMAGE \
-            --build-arg VERSION=$VERSION_TAG \
-            --build-arg COMPONENT_VERSION=$VERSION_TAG \
-            --cache-from=type=gha,scope=wheels_${COMP}_${SERVICE}_${ARCH} \
-            --cache-to=type=gha,mode=max,scope=wheels_${COMP}_${SERVICE}_${ARCH} \
-            --tag $IMAGE:$VERSION_TAG \
+            --build-arg VERSION=$VERSION \
+            --build-arg COMPONENT_VERSION=$COMPONENT_VERSION \
+            --cache-from=type=gha,scope=wheels_${COMP}_${ARCH} \
+            --cache-to=type=gha,mode=max,scope=wheels_${COMP}_${ARCH} \
+            --tag $IMAGE:$VERSION \
             --file $DOCKERFILE \
             --push \
             .
 
   release:
+    if: github.event_name == 'release'
     needs: [discover]
     runs-on: ubuntu-latest
     permissions:
@@ -232,11 +219,11 @@ jobs:
       - name: 🚀 Retag and push floating tags (on release or prerelease)
         if: github.event_name == 'release' && github.event.action != 'deleted'
         run: |
-          COMP=${{ matrix.component }}
-          SERVICE=${{ matrix.service }}
-          IMAGE="subvortex/subvortex-$COMP-$SERVICE"
+          COMP="${{ matrix.component }}"
+          REPO_NAME="subvortex-${COMP//_/-}" && echo "$COMP"
+          IMAGE="subvortex/$REPO_NAME"
           RAW_VERSION_TAG="${{ steps.taginfo.outputs.version_tag }}"
-          VERSION_TAG="${RAW_VERSION_TAG#v}"
+          VERSION="${RAW_VERSION_TAG#v}"
           FLOATING_TAGS="${{ steps.taginfo.outputs.floating_tags }}"
           IS_PRERELEASE=${{ github.event.release.prerelease }}
           IS_DRAFT=${{ github.event.release.draft }}
@@ -249,8 +236,8 @@ jobs:
             exit 0
           fi
       
-          echo "🔍 Getting manifest for $IMAGE:$VERSION_TAG"
-          docker buildx imagetools inspect $IMAGE:$VERSION_TAG
+          echo "🔍 Getting manifest for $IMAGE:$VERSION"
+          docker buildx imagetools inspect $IMAGE:$VERSION
       
           for TAG in $FLOATING_TAGS; do
             # Skip "latest" for prereleases
@@ -259,27 +246,26 @@ jobs:
               continue
             fi
       
-            echo "🔁 Creating manifest for $IMAGE:$TAG from $IMAGE:$VERSION_TAG"
+            echo "🔁 Creating manifest for $IMAGE:$TAG from $IMAGE:$VERSION"
             docker buildx imagetools create \
               --tag $IMAGE:$TAG \
-              $IMAGE:$VERSION_TAG
+              $IMAGE:$VERSION
           done
 
       - name: 🧹 Remove floating tags (on release or prerelease delete)
         if: github.event_name == 'release' && github.event.action == 'deleted'
         run: |
-          COMP=${{ matrix.component }}
-          SERVICE=${{ matrix.service }}
-          IMAGE="subvortex/subvortex-$COMP-$SERVICE"
+          COMP="${{ matrix.component }}"
+          REPO_NAME="subvortex-${COMP//_/-}" && echo "$COMP"
+          IMAGE="subvortex/$REPO_NAME"
           RAW_VERSION_TAG="${{ github.event.release.tag_name }}"
-          VERSION_TAG="${RAW_VERSION_TAG#v}"
+          VERSION="${RAW_VERSION_TAG#v}"
           FLOATING_TAGS="${{ steps.taginfo.outputs.floating_tags }}"
           USERNAME="${{ secrets.DOCKER_USERNAME }}"
           PASSWORD="${{ secrets.DOCKER_PASSWORD }}"
-          REPO_NAME="subvortex-$COMP-$SERVICE"
 
-          echo "🗑️ Release deleted: $RAW_VERSION_TAG"
-          echo "🔍 Attempting to delete floating tags: $FLOATING_TAGS"
+          echo "🗑️ Release deleted: $VERSION"
+          echo "🔍 Handling floating tags: $FLOATING_TAGS"
 
           echo "🔐 Requesting Docker Hub JWT token..."
           TOKEN=$(curl -s -X POST https://hub.docker.com/v2/users/login/ \
@@ -291,18 +277,51 @@ jobs:
             exit 1
           fi
 
+          echo "📦 Fetching all tags from Docker Hub (excluding deleted tag: $VERSION)..."
+          ALL_TAGS=$(curl -s -H "Authorization: JWT $TOKEN" \
+            "https://hub.docker.com/v2/repositories/$USERNAME/$REPO_NAME/tags?page_size=100" | jq -r '.results[].name' | grep -v "^$VERSION$")
+
+          RELEASE_TAGS=$(echo "$ALL_TAGS" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' || true)
+          PRERELEASE_TAGS=$(echo "$ALL_TAGS" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-(alpha|rc)\.[0-9]+$' || true)
+
           for TAG in $FLOATING_TAGS; do
-            echo "❌ Deleting tag: $IMAGE:$TAG"
-            RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \
-              "https://hub.docker.com/v2/repositories/$USERNAME/$REPO_NAME/tags/$TAG/" \
-              -H "Authorization: JWT $TOKEN")
-
-            if [ "$RESPONSE" = "204" ]; then
-              echo "✅ Successfully deleted $IMAGE:$TAG"
-            elif [ "$RESPONSE" = "404" ]; then
-              echo "⚠️ Tag $TAG not found (already deleted or never pushed)"
+            echo "🔁 Handling floating tag: $TAG"
+
+            case "$TAG" in
+              dev)
+                TARGET=$(echo "$PRERELEASE_TAGS" | grep 'alpha' | sort -Vr | head -n1)
+                ;;
+              stage)
+                TARGET=$(echo "$PRERELEASE_TAGS" | grep 'rc' | sort -Vr | head -n1)
+                ;;
+              latest)
+                TARGET=$(echo "$RELEASE_TAGS" | sort -Vr | head -n1)
+                ;;
+              *)
+                echo "⚠️ Unknown floating tag: $TAG"
+                continue
+                ;;
+            esac
+
+            if [ -n "$TARGET" ]; then
+              echo "🔄 Re-pointing $TAG to $TARGET as multi-platform manifest"
+              echo "$PASSWORD" | docker login -u "$USERNAME" --password-stdin
+              docker buildx imagetools create \
+                --tag "$IMAGE:$TAG" \
+                "$IMAGE:$TARGET"
+              docker logout
             else
-              echo "❌ Failed to delete tag $TAG (HTTP $RESPONSE)"
+              echo "🗑️ No matching version for $TAG. Deleting..."
+              RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \
+                "https://hub.docker.com/v2/repositories/$USERNAME/$REPO_NAME/tags/$TAG/" \
+                -H "Authorization: JWT $TOKEN")
+              if [ "$RESPONSE" = "204" ]; then
+                echo "✅ Deleted $IMAGE:$TAG"
+              elif [ "$RESPONSE" = "404" ]; then
+                echo "⚠️ Tag $TAG not found"
+              else
+                echo "❌ Failed to delete $TAG (HTTP $RESPONSE)"
+              fi
             fi
           done
         
diff --git a/.gitignore b/.gitignore
@@ -171,4 +171,7 @@ ips_blocked.json
 
 firewall-events.json
 .DS_Store
-DS_Store
+DS_Store
+
+.build
+.secrets
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
@@ -0,0 +1,27 @@
+services:
+  auto_upgrader:
+    build:
+      context: ../../
+      dockerfile: ./subvortex/auto_upgrader/Dockerfile
+      additional_contexts:
+        wheelbuilder: docker-image://subvortex/subvortex-wheel-builder:latest
+    container_name: subvortex-auto-upgrader
+    restart: unless-stopped
+    env_file:
+      - ./subvortex/auto_upgrader/.env
+    volumes:
+      - /var/log:/app/logs
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+
+  watchtower:
+    image: containrrr/watchtower
+    container_name: subvortex-watchtower
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: --interval ${SUBVORTEX_CHECK_INTERVAL} --cleanup --label-enable
+    restart: unless-stopped
+    environment:
+      - WATCHTOWER_CLEANUP=true
+      - WATCHTOWER_LABEL_ENABLE=true
+      - WATCHTOWER_INCLUDE_RESTARTING=true
diff --git a/pyproject.toml b/pyproject.toml
@@ -48,6 +48,7 @@ include = ["subvortex", "subvortex.auto_upgrader"]
     "**/requirements.txt",
     "**/README.md",
     "**/env.template",
+    "**/docker-compose.yml",
 ]
 
 [tool.setuptools.exclude-package-data]
diff --git a/subvortex/auto_upgrader/Dockerfile b/subvortex/auto_upgrader/Dockerfile
@@ -27,6 +27,8 @@ LABEL auto_upgrader.version=$COMPONENT_VERSION
 
 # ---------- Environment ----------
 ENV PYTHONUNBUFFERED=1
+ARG SUBVORTEX_EXECUTION_ROLE=miner
+ENV SV_EXECUTION_ROLE=${SUBVORTEX_EXECUTION_ROLE}
 
 WORKDIR /app
 
@@ -42,13 +44,14 @@ RUN pip install --no-cache-dir --find-links=/tmp/wheels -r requirements.txt \
 # 📁 Copy source code
 COPY ./pyproject.toml ./pyproject.toml
 COPY ./subvortex/pyproject.toml ./subvortex/pyproject.toml
-COPY ./subvortex/auto_upgrader/version.py ./subvortex/auto_upgrader/version.py
-COPY ./subvortex/miner/auto_upgrader/src ./subvortex/auto_upgrader/src
-COPY ./subvortex/miner/auto_upgrader/entrypoint.sh ./subvortex/auto_upgrader/entrypoint.sh
-COPY ./subvortex/miner/auto_upgrader/pyproject.toml ./subvortex/auto_upgrader/pyproject.toml
+COPY ./subvortex/version.py ./subvortex/version.py
+COPY ./subvortex/auto_upgrader/src ./subvortex/auto_upgrader/src
+COPY ./subvortex/auto_upgrader/entrypoint.sh ./subvortex/auto_upgrader/entrypoint.sh
+COPY ./subvortex/auto_upgrader/pyproject.toml ./subvortex/auto_upgrader/pyproject.toml
 
 # 🧩 Install project (editable)
-RUN pip install -e . \
+RUN pip install "./subvortex/auto_upgrader[${SV_EXECUTION_ROLE}]" \
+    && pip install -e . \
     && rm -rf ~/.cache/pip
 
 ENTRYPOINT ["/bin/bash", "./subvortex/auto_upgrader/entrypoint.sh"]
diff --git a/subvortex/auto_upgrader/entrypoint.sh b/subvortex/auto_upgrader/entrypoint.sh
@@ -7,7 +7,9 @@ export $(grep -v '^#' .env | xargs)
 ARGS=()
 PREFIX="SUBVORTEX_"
 
-while IFS='=' read -r key value; do
+while IFS= read -r line; do
+  key="${line%%=*}"
+  value="${line#*=}"
   if [[ $key == ${PREFIX}* ]]; then
     key_suffix="${key#$PREFIX}"
     cli_key="--$(echo "$key_suffix" | tr '[:upper:]' '[:lower:]' | tr '_' '.')"
diff --git a/subvortex/core/Dockerfile.builder b/subvortex/core/Dockerfile.builder
@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1.4
+
+FROM python:3.11-slim AS builder
+
+WORKDIR /build
+
+# Install only necessary system libraries (minimal footprint)
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends \
+  build-essential \
+  curl \
+  git \
+  libssl-dev \
+  libffi-dev \
+  libpcap-dev \
+  libnfnetlink-dev \
+  libnetfilter-queue-dev \
+  python3-dev \
+  pkg-config && \
+  curl https://sh.rustup.rs -sSf | sh -s -- -y --profile=minimal && \
+  rm -rf /var/lib/apt/lists/*
+
+ENV CFLAGS="-I/usr/include/libnetfilter_queue"
+ENV PATH="/root/.cargo/bin:$PATH"
diff --git a/subvortex/pyproject.toml b/subvortex/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "subvortex"
-dynamic = ["version"]
+version = "0.0.1"
 description = "SubVortex Auto Upgrader"
 authors = [{ name = "Eclipse Vortex", email = "subvortex.bt@gmail.com" }]
 readme = "README.md"
@@ -29,9 +29,6 @@ classifiers = [
 [project.license]
 text = "MIT"
 
-[tool.setuptools.dynamic]
-version = { attr = "subvortex.version.__version__" }
-
 [tool.setuptools]
 include-package-data = true
 

Original file line number	Diff line number	Diff line change
`@@ -48,6 +48,7 @@ include = ["subvortex", "subvortex.auto_upgrader"]`
`48`	`48`	`"**/requirements.txt",`
`49`	`49`	`"**/README.md",`
`50`	`50`	`"**/env.template",`
	`51`	`+ "**/docker-compose.yml",`
`51`	`52`	`]`
`52`	`53`
`53`	`54`	`[tool.setuptools.exclude-package-data]`