|
| 1 | +name: UI E2E tests with model inference cache |
| 2 | + |
| 3 | +on: |
| 4 | + workflow_call: |
| 5 | + secrets: |
| 6 | + S3_ACCESS_KEY_ID: |
| 7 | + required: true |
| 8 | + S3_SECRET_ACCESS_KEY: |
| 9 | + required: true |
| 10 | + OPENAI_API_KEY: |
| 11 | + required: false |
| 12 | + FIREWORKS_ACCOUNT_ID: |
| 13 | + required: false |
| 14 | + FIREWORKS_API_KEY: |
| 15 | + required: false |
| 16 | + inputs: |
| 17 | + regen_cache: |
| 18 | + required: true |
| 19 | + type: boolean |
| 20 | + |
| 21 | +jobs: |
| 22 | + ui-tests-e2e: |
| 23 | + runs-on: namespace-profile-tensorzero-8x16 |
| 24 | + # We currently only run this job when we have secrets available, since we need to use an S3 object_store |
| 25 | + # In the future, we might want to fix this so that it can run in PR CI for external (forked) PRs |
| 26 | + # For now, it just runs in the merge queue and on prs from the main repo |
| 27 | + if: ${{ (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') || github.event_name == 'merge_group' }} |
| 28 | + |
| 29 | + steps: |
| 30 | + # TODO - investigate why using the Namespace checkout action causes |
| 31 | + # 'tensorzero_core::built_info::GIT_COMMIT_HASH_SHORT' to be `None` |
| 32 | + - name: Check out the repo |
| 33 | + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 |
| 34 | + |
| 35 | + - name: Setup Node |
| 36 | + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 |
| 37 | + with: |
| 38 | + node-version: "22.9.0" |
| 39 | + |
| 40 | + - name: Setup `pnpm` |
| 41 | + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda |
| 42 | + |
| 43 | + - name: Install `pnpm` dependencies |
| 44 | + run: pnpm install --frozen-lockfile |
| 45 | + |
| 46 | + - name: Setup Playwright |
| 47 | + run: pnpm --filter=tensorzero-ui exec playwright install --with-deps chromium |
| 48 | + |
| 49 | + - name: Cache `s3-fixtures` |
| 50 | + uses: namespacelabs/nscloud-cache-action@2f50e7d0f70475e6f59a55ba0f05eec9108e77cc |
| 51 | + with: |
| 52 | + path: | |
| 53 | + ./ui/fixtures/s3-fixtures |
| 54 | +
|
| 55 | + # When we're regenerating the model inference cache, don't download the container images |
| 56 | + # The images will get built by `regenerate-model-inference-cache.sh` |
| 57 | + - name: Download container images |
| 58 | + if: inputs.regen_cache == false |
| 59 | + uses: namespace-actions/download-artifact@5c070f7d7ebdc47682b04aa736c76e46ff5f6e1e |
| 60 | + with: |
| 61 | + pattern: build-*-container |
| 62 | + merge-multiple: true |
| 63 | + |
| 64 | + - name: Load `gateway` and `ui` containers |
| 65 | + if: inputs.regen_cache == false |
| 66 | + run: | |
| 67 | + docker load < gateway-container.tar |
| 68 | + docker load < ui-container.tar |
| 69 | +
|
| 70 | + # This allows us to use 'no-build' on subsequent steps |
| 71 | + - name: Build needed docker images |
| 72 | + working-directory: ui |
| 73 | + run: | |
| 74 | + docker compose -f fixtures/docker-compose.e2e.yml -f fixtures/docker-compose.ui.yml build fixtures mock-inference-provider |
| 75 | +
|
| 76 | + - name: Set common fixture environment variables |
| 77 | + working-directory: ui |
| 78 | + run: | |
| 79 | + # Environment variables shared by the gateway and ui containers |
| 80 | + echo "TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@clickhouse:8123/tensorzero_ui_fixtures" >> fixtures/.env |
| 81 | + echo "TENSORZERO_GATEWAY_URL=http://gateway:3000" >> fixtures/.env |
| 82 | + echo "TENSORZERO_GATEWAY_TAG=sha-${{ github.sha }}" >> fixtures/.env |
| 83 | + echo "TENSORZERO_UI_TAG=sha-${{ github.sha }}" >> fixtures/.env |
| 84 | + # We need these set in the ui container, so that we construct the correct optimizer config |
| 85 | + # to pass to 'experimentalLaunchOptimizationWorkflow' |
| 86 | + echo "FIREWORKS_BASE_URL=http://mock-inference-provider:3030/fireworks/" >> fixtures/.env |
| 87 | + echo "OPENAI_BASE_URL=http://mock-inference-provider:3030/openai/" >> fixtures/.env |
| 88 | + echo "FIREWORKS_ACCOUNT_ID=fake_fireworks_account" >> fixtures/.env |
| 89 | + echo "VITE_TENSORZERO_FORCE_CACHE_ON=1" >> fixtures/.env |
| 90 | +
|
| 91 | + - name: Regenerate model inference cache |
| 92 | + working-directory: ui |
| 93 | + if: inputs.regen_cache == true |
| 94 | + run: | |
| 95 | + echo "FIREWORKS_ACCOUNT_ID=${{ secrets.FIREWORKS_ACCOUNT_ID }}" >> fixtures/.env-gateway |
| 96 | + echo "FIREWORKS_API_KEY=${{ secrets.FIREWORKS_API_KEY }}" >> fixtures/.env-gateway |
| 97 | + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> fixtures/.env-gateway |
| 98 | + echo "S3_ACCESS_KEY_ID=${{ secrets.S3_ACCESS_KEY_ID }}" >> fixtures/.env-gateway |
| 99 | + echo "S3_SECRET_ACCESS_KEY=${{ secrets.S3_SECRET_ACCESS_KEY }}" >> fixtures/.env-gateway |
| 100 | + ./fixtures/regenerate-model-inference-cache.sh |
| 101 | + # Take down and destroy the containers, so that our subsequent test run starts with a fresh ClickHouse |
| 102 | + # server, and inserts our regenerated fixtures from scratch |
| 103 | + docker compose -f ./fixtures/docker-compose.e2e.yml -f ./fixtures/docker-compose.ui.yml down |
| 104 | + docker compose -f ./fixtures/docker-compose.e2e.yml -f ./fixtures/docker-compose.ui.yml rm -f |
| 105 | +
|
| 106 | +
|
| 107 | + - name: Upload new model inference cache |
| 108 | + if: inputs.regen_cache == true |
| 109 | + uses: namespace-actions/upload-artifact@9a78c62e083914789d908952f9773e42744b9f68 |
| 110 | + with: |
| 111 | + name: model-inference-cache |
| 112 | + path: | |
| 113 | + ui/fixtures/model_inference_cache_e2e.jsonl |
| 114 | +
|
| 115 | + - name: Start dependency Docker containers and apply fixtures |
| 116 | + working-directory: ui |
| 117 | + run: | |
| 118 | + # Environment variables only used by the gateway container |
| 119 | + # We deliberately leave these unset when starting the UI container, to ensure |
| 120 | + # that it doesn't depend on them being set |
| 121 | + # When the secrets are not passed in to this workflow |
| 122 | + # (which happens when we test our existing cache file in PR CI), |
| 123 | + # we set them to dummy values |
| 124 | + echo "FIREWORKS_ACCOUNT_ID=${{ secrets.FIREWORKS_ACCOUNT_ID || 'not_used' }}" >> fixtures/.env-gateway |
| 125 | + echo "FIREWORKS_API_KEY=${{ secrets.FIREWORKS_API_KEY || 'not_used' }}" >> fixtures/.env-gateway |
| 126 | + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY || 'not_used' }}" >> fixtures/.env-gateway |
| 127 | + echo "S3_ACCESS_KEY_ID=${{ secrets.S3_ACCESS_KEY_ID }}" >> fixtures/.env-gateway |
| 128 | + echo "S3_SECRET_ACCESS_KEY=${{ secrets.S3_SECRET_ACCESS_KEY }}" >> fixtures/.env-gateway |
| 129 | + docker compose -f fixtures/docker-compose.e2e.yml up --no-build -d |
| 130 | + docker compose -f fixtures/docker-compose.e2e.yml wait fixtures |
| 131 | +
|
| 132 | + - name: Start UI Docker container |
| 133 | + working-directory: ui |
| 134 | + run: | |
| 135 | + docker compose -f fixtures/docker-compose.ui.yml up --no-build -d --wait |
| 136 | +
|
| 137 | + - name: Run UI E2E tests |
| 138 | + id: e2e_tests |
| 139 | + env: |
| 140 | + TENSORZERO_CI: 1 |
| 141 | + run: pnpm ui:test:e2e --grep-invert "@credentials" |
| 142 | + |
| 143 | + - name: Print Docker Compose logs |
| 144 | + if: always() |
| 145 | + working-directory: ui |
| 146 | + run: docker compose -f fixtures/docker-compose.e2e.yml -f fixtures/docker-compose.ui.yml logs -t |
| 147 | + |
| 148 | + - name: Print container health checks |
| 149 | + if: always() |
| 150 | + working-directory: ui |
| 151 | + run: docker inspect --format "{{json .State.Health }}" $(docker compose -f fixtures/docker-compose.e2e.yml -f fixtures/docker-compose.ui.yml ps -q ui) | jq |
| 152 | + |
| 153 | + - name: Make sure the current commit short hash is in the Docker Compose gateway logs |
| 154 | + if: always() |
| 155 | + working-directory: ui |
| 156 | + run: | |
| 157 | + SHORT_HASH=$(echo "${{ github.sha }}" | cut -c1-7) |
| 158 | + docker compose -f fixtures/docker-compose.e2e.yml logs gateway | grep "(commit: ${SHORT_HASH})" || { |
| 159 | + echo "ERROR: Commit hash ${SHORT_HASH} not found in gateway logs" |
| 160 | + exit 1 |
| 161 | + } |
| 162 | +
|
| 163 | + - name: Print ClickHouse error logs |
| 164 | + if: always() |
| 165 | + run: docker exec fixtures-clickhouse-1 cat /var/log/clickhouse-server/clickhouse-server.err.log |
| 166 | + |
| 167 | + - name: Print ClickHouse trace logs |
| 168 | + if: always() |
| 169 | + run: docker exec fixtures-clickhouse-1 cat /var/log/clickhouse-server/clickhouse-server.log |
| 170 | + |
| 171 | + - name: Upload Playwright artifacts |
| 172 | + if: failure() |
| 173 | + uses: namespace-actions/upload-artifact@9a78c62e083914789d908952f9773e42744b9f68 |
| 174 | + with: |
| 175 | + name: playwright-report |
| 176 | + path: | |
| 177 | + ui/playwright-report/ |
| 178 | + ui/test-results/ |
| 179 | + retention-days: 7 |
0 commit comments