diff --git a/.github/actions/setup-buildx/action.yml b/.github/actions/setup-buildx/action.yml new file mode 100644 index 00000000..6d5627e8 --- /dev/null +++ b/.github/actions/setup-buildx/action.yml @@ -0,0 +1,30 @@ +name: Setup Docker Buildx +description: > + Create a multi-arch Docker Buildx builder using remote BuildKit nodes. + The builder is automatically removed when the job finishes (cleanup is + enabled by default in docker/setup-buildx-action). + +inputs: + amd64-endpoint: + description: BuildKit endpoint for linux/amd64 + default: tcp://buildkit-amd64.buildkit:1234 + arm64-endpoint: + description: BuildKit endpoint for linux/arm64 + default: tcp://buildkit-arm64.buildkit:1234 + name: + description: Builder instance name + default: navigator + +runs: + using: composite + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + name: ${{ inputs.name }} + driver: remote + endpoint: ${{ inputs.amd64-endpoint }} + platforms: linux/amd64 + append: | + - endpoint: ${{ inputs.arm64-endpoint }} + platforms: linux/arm64 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000..48b93cfa --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,56 @@ +name: Checks + +on: + push: + branches: [main] + pull_request: + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +permissions: + contents: read + packages: read + +jobs: + rust: + name: Rust + runs-on: build-amd64 + container: + image: ghcr.io/nvidia/nv-agent-env/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Format + run: mise run rust:format:check + + - name: Lint + run: mise run rust:lint + + - name: Test + run: mise run test:rust + + python: + name: Python + runs-on: build-amd64 + container: + image: ghcr.io/nvidia/nv-agent-env/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: uv sync --frozen + + - name: Lint + run: mise run python:lint + + - name: Test + run: mise run test:python diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml new file mode 100644 index 00000000..8410fdd5 --- /dev/null +++ b/.github/workflows/ci-image.yml @@ -0,0 +1,49 @@ +name: CI Image + +on: + push: + branches: [main] + paths: + - 'deploy/docker/Dockerfile.ci' + - 'mise.toml' + - 'build/**' + - '.github/workflows/ci-image.yml' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + CI_IMAGE: ghcr.io/nvidia/nv-agent-env/ci + +permissions: + contents: read + packages: write + +jobs: + build-ci-image: + name: Build + runs-on: build-amd64 + steps: + - uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + + - name: Build and push CI image + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --cache-from type=registry,ref=${{ env.CI_IMAGE }}:buildcache \ + --cache-to type=registry,ref=${{ env.CI_IMAGE }}:buildcache,mode=max \ + --build-arg MISE_GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + --push \ + -t ${{ env.CI_IMAGE }}:${{ github.sha }} \ + -t ${{ env.CI_IMAGE }}:latest \ + -f deploy/docker/Dockerfile.ci \ + . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e45063e2..7168f457 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -154,12 +154,12 @@ mise run test:e2e:sandbox # Sandbox Python e2e tests ```bash # Rust -mise run fmt # Format code -mise run fmt:check # Check formatting -mise run clippy # Run Clippy lints +mise run rust:format # Format code +mise run rust:format:check # Check formatting +mise run rust:lint # Lint with Clippy # Python -mise run python:fmt # Format with ruff +mise run python:format # Format with ruff mise run python:lint # Lint with ruff mise run python:typecheck # Type check with ty diff --git a/build/ci.toml b/build/ci.toml index fb90abc8..d8feace5 100644 --- a/build/ci.toml +++ b/build/ci.toml @@ -1,8 +1,8 @@ # CI, pre-commit, and sandbox runner tasks [lint] -description = "Run all linters (fmt, clippy, ruff)" -depends = ["fmt:check", "clippy", "python:lint"] +description = "Run all linters and format checks" +depends = ["rust:format:check", "rust:lint", "python:lint"] [all] description = "Build, lint, and test everything" diff --git a/build/python.toml b/build/python.toml index bdcf4930..be308759 100644 --- a/build/python.toml +++ b/build/python.toml @@ -112,7 +112,7 @@ depends = ["python:proto"] env = { UV_NO_SYNC = "1" } run = "uv run ruff check {{vars.python_paths}}" -["python:fmt"] +["python:format"] description = "Format Python code with ruff" run = "uv run ruff format {{vars.python_paths}}" diff --git a/build/rust.toml b/build/rust.toml index b5b568e5..f3771f6b 100644 --- a/build/rust.toml +++ b/build/rust.toml @@ -12,15 +12,15 @@ run = "cargo build --workspace --release" description = "Check all Rust crates for errors" run = "cargo check --workspace" -[clippy] -description = "Run Clippy lints" +["rust:lint"] +description = "Lint Rust code with Clippy" run = "cargo clippy --workspace --all-targets" -[fmt] +["rust:format"] description = "Format Rust code" run = "cargo fmt --all" -["fmt:check"] +["rust:format:check"] description = "Check Rust formatting" run = "cargo fmt --all -- --check" diff --git a/deploy/docker/Dockerfile.ci b/deploy/docker/Dockerfile.ci index 6dacbdd5..0afb4a48 100644 --- a/deploy/docker/Dockerfile.ci +++ b/deploy/docker/Dockerfile.ci @@ -7,15 +7,11 @@ FROM ubuntu:24.04 ARG DOCKER_VERSION=27.5.1 ARG BUILDX_VERSION=v0.21.1 ARG TARGETARCH -ARG KUBECTL_VERSION=v1.35.1 -ARG HELM_VERSION=v4.1.1 -ARG PROTOC_VERSION=29.6 -ARG SCCACHE_VERSION=v0.14.0 ENV DEBIAN_FRONTEND=noninteractive ENV MISE_DATA_DIR=/opt/mise ENV MISE_CACHE_DIR=/opt/mise/cache -ENV PATH="/opt/mise/shims:/root/.local/bin:$PATH" +ENV PATH="/opt/mise/shims:/root/.cargo/bin:/root/.local/bin:$PATH" # Install system dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -28,7 +24,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ python3-venv \ cmake \ - protobuf-compiler \ socat \ unzip \ xz-utils \ @@ -58,50 +53,18 @@ RUN case "$TARGETARCH" in \ && /tmp/aws/install \ && rm -rf /tmp/aws /tmp/awscliv2.zip -# Install kubectl, helm, and protoc without GitHub API lookups -RUN case "$TARGETARCH" in \ - amd64) karch=amd64; helm_arch=amd64; protoc_arch=x86_64 ;; \ - arm64) karch=arm64; helm_arch=arm64; protoc_arch=aarch_64 ;; \ - *) echo "Unsupported TARGETARCH: $TARGETARCH"; exit 1 ;; \ - esac \ - && curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${karch}/kubectl" -o /usr/local/bin/kubectl \ - && chmod +x /usr/local/bin/kubectl \ - && curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${helm_arch}.tar.gz" -o /tmp/helm.tgz \ - && tar -xzf /tmp/helm.tgz -C /tmp \ - && mv "/tmp/linux-${helm_arch}/helm" /usr/local/bin/helm \ - && chmod +x /usr/local/bin/helm \ - && rm -rf /tmp/helm.tgz "/tmp/linux-${helm_arch}" \ - && curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip" -o /tmp/protoc.zip \ - && unzip -q /tmp/protoc.zip -d /tmp/protoc \ - && mv /tmp/protoc/bin/protoc /usr/local/bin/protoc \ - && chmod +x /usr/local/bin/protoc \ - && rm -rf /tmp/protoc /tmp/protoc.zip - -# Install sccache directly on amd64 (mise/aqua plugin is arch-limited) -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - curl -fsSL "https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" -o /tmp/sccache.tgz \ - && tar -xzf /tmp/sccache.tgz -C /tmp \ - && mv "/tmp/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache \ - && chmod +x /usr/local/bin/sccache \ - && rm -rf /tmp/sccache.tgz "/tmp/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl"; \ - else \ - echo "Skipping sccache install on $TARGETARCH"; \ - fi - # Install mise RUN curl https://mise.run | sh -# Copy mise.toml and build task includes, then install core tools +# Copy mise.toml and build task includes, then install all tools via mise COPY mise.toml /opt/mise/mise.toml COPY build/ /opt/mise/build/ WORKDIR /opt/mise +ARG MISE_GITHUB_TOKEN RUN mise trust /opt/mise/mise.toml && \ - env -u RUSTC_WRAPPER mise install python rust 'cargo:cargo-edit' && \ + env -u RUSTC_WRAPPER mise install && \ /root/.cargo/bin/rustup component remove rust-docs || true && \ rm -rf /root/.rustup/toolchains/*/share/doc /root/.rustup/toolchains/*/share/man -# Install uv directly to avoid GitHub API rate limits in CI image builds -RUN curl -LsSf https://astral.sh/uv/0.10.2/install.sh | sh - # Set working directory for CI jobs WORKDIR /builds