Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion test/sanity/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@ COPY --chown=testuser packages/opencode/script/postinstall.mjs /home/testuser/.a
COPY --chown=testuser packages/opencode/package.json /home/testuser/.altimate-install/package.json
COPY --chown=testuser .opencode/skills/ /home/testuser/.altimate-install/skills/

# Install altimate-core native binding (required at runtime)
# Install altimate-core native binding (required at runtime).
# NOTE: Database drivers (pg, snowflake-sdk, mysql2, etc.) are NOT installed here
# because a real `npm install -g altimate-code` doesn't install them either —
# they're optionalDependencies in the workspace @altimateai/drivers package which
# is never published. This is issue #295. Driver resolvability tests are expected
# to FAIL here, proving the bug exists for real users.
RUN cd /home/testuser/.altimate-install && \
echo '{"dependencies":{"@altimateai/altimate-core":"latest"}}' > package.json && \
bun install && \
Expand Down
16 changes: 10 additions & 6 deletions test/sanity/ci-local.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,25 +56,25 @@ if [ "$MODE" = "--full" ] || [ "$MODE" = "full" ]; then
echo "=== Full CI (Docker) ==="

# Driver E2E with Docker containers
run_step "Docker Services Up" docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" up -d postgres mysql mssql redshift
run_step "Docker Services Up" docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" up -d postgres mysql mssql redshift mongodb

echo " Waiting for services to be healthy..."
HEALTHY=0
for _wait in $(seq 1 30); do
HEALTHY=$(docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" ps --format json 2>/dev/null | grep -c '"healthy"' || echo "0")
if [ "$HEALTHY" -ge 4 ]; then break; fi
if [ "$HEALTHY" -ge 5 ]; then break; fi
sleep 2
done

if [ "$HEALTHY" -lt 4 ]; then
echo " >>> Docker Services: FAILED ($HEALTHY/4 healthy after 60s)"
if [ "$HEALTHY" -lt 5 ]; then
echo " >>> Docker Services: FAILED ($HEALTHY/5 healthy after 60s)"
EXIT_CODE=1
else
echo " >>> Docker Services: $HEALTHY/4 healthy"
echo " >>> Docker Services: $HEALTHY/5 healthy"
fi

# Skip driver tests if services aren't healthy
if [ "$HEALTHY" -lt 4 ]; then
if [ "$HEALTHY" -lt 5 ]; then
echo " SKIP: Driver E2E tests (services not healthy)"
else

Expand All @@ -88,6 +88,10 @@ if [ "$MODE" = "--full" ] || [ "$MODE" = "full" ]; then
TEST_REDSHIFT_HOST=127.0.0.1 TEST_REDSHIFT_PORT=15439 TEST_REDSHIFT_PASSWORD=testpass123 \
bun test test/altimate/drivers-docker-e2e.test.ts --timeout 30000"

run_step "Driver E2E (mongodb)" bash -c "cd $REPO_ROOT/packages/opencode && \
TEST_MONGODB_HOST=127.0.0.1 TEST_MONGODB_PORT=17017 \
bun test test/altimate/drivers-mongodb-e2e.test.ts --timeout 30000"

# Full sanity suite in Docker
run_step "Sanity Suite (Docker)" docker compose -f "$REPO_ROOT/test/sanity/docker-compose.yml" \
up --build --abort-on-container-exit --exit-code-from sanity
Expand Down
19 changes: 14 additions & 5 deletions test/sanity/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ services:
- TEST_REDSHIFT_HOST=redshift
- TEST_REDSHIFT_PORT=5432
- TEST_REDSHIFT_PASSWORD=testpass123
- TEST_MONGODB_HOST=mongodb
- TEST_MONGODB_PORT=27017
depends_on:
postgres:
condition: service_healthy
mysql:
condition: service_healthy
mssql:
condition: service_healthy
redshift:
mongodb:
condition: service_healthy
# mysql, mssql, redshift are for driver E2E tests (ci-local.sh --full),
# not the sanity shell scripts. Don't block sanity on them.

postgres:
image: postgres:16-alpine
Expand Down Expand Up @@ -74,3 +74,12 @@ services:
test: pg_isready
interval: 5s
retries: 10

mongodb:
image: mongo:7
ports:
- "17017:27017"
healthcheck:
test: mongosh --eval "db.adminCommand('ping')" --quiet
interval: 5s
retries: 10
110 changes: 89 additions & 21 deletions test/sanity/phases/resilience.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ echo '{}' > package.json
git add -A && git commit -q -m "init"

# 1. SQLite DB created after first run
echo " [1/7] SQLite DB creation..."
echo " [1/10] SQLite DB creation..."
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
altimate_run "db-create" "say hello" || true
# Find the DB — could be opencode.db, opencode-latest.db, or opencode-{channel}.db
Expand All @@ -42,7 +42,7 @@ else
fi

# 2. WAL mode enabled
echo " [2/7] WAL mode..."
echo " [2/10] WAL mode..."
if [ -n "$DB_PATH" ] && [ -f "$DB_PATH" ] && command -v sqlite3 >/dev/null 2>&1; then
WAL_MODE=$(sqlite3 "$DB_PATH" "PRAGMA journal_mode;" 2>/dev/null || echo "unknown")
assert_eq "$WAL_MODE" "wal" "WAL mode enabled"
Expand All @@ -51,7 +51,7 @@ else
fi

# 3. Session persisted
echo " [3/7] Session persistence..."
echo " [3/10] Session persistence..."
if [ -n "$DB_PATH" ] && [ -f "$DB_PATH" ] && command -v sqlite3 >/dev/null 2>&1; then
SESSION_COUNT=$(sqlite3 "$DB_PATH" "SELECT count(*) FROM session;" 2>/dev/null || echo "0")
assert_ge "$SESSION_COUNT" 1 "session persisted (got $SESSION_COUNT)"
Expand All @@ -60,7 +60,7 @@ else
fi

# 4. Session continue (DB survives restart)
echo " [4/7] Session continue..."
echo " [4/10] Session continue..."
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
altimate_run "continue" --continue "what was my last message" || true
assert_not_contains "$(get_output continue)" "TIMEOUT" "session continue works"
Expand All @@ -69,7 +69,7 @@ else
fi

# 5. Compaction doesn't crash (best-effort — seed if fixture available)
echo " [5/7] Compaction resilience..."
echo " [5/10] Compaction resilience..."
if [ -n "$DB_PATH" ] && [ -f "$SCRIPT_DIR/fixtures/compaction-session.sql" ] && command -v sqlite3 >/dev/null 2>&1; then
sqlite3 "$DB_PATH" < "$SCRIPT_DIR/fixtures/compaction-session.sql" 2>/dev/null || true
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
Expand All @@ -90,20 +90,8 @@ else
skip_test "Compaction resilience" "fixture or sqlite3 not available"
fi

# 6. Graceful on missing provider key
echo " [6/7] Missing API key handling..."
SAVED_KEY="${ANTHROPIC_API_KEY:-}"
unset ANTHROPIC_API_KEY
OUTPUT=$(timeout 10 altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
# Should get a clean error, not an unhandled exception / stack trace
assert_not_contains "$OUTPUT" "TypeError" "no TypeError on missing key"
assert_not_contains "$OUTPUT" "Cannot read properties" "no unhandled error on missing key"
if [ -n "$SAVED_KEY" ]; then
export ANTHROPIC_API_KEY="$SAVED_KEY"
fi

# 7. Config backwards compatibility
echo " [7/7] Config backwards compat..."
# 6. Config backwards compatibility
echo " [6/10] Config backwards compat..."
CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/altimate-code"
mkdir -p "$CONFIG_DIR"
if [ -f "$SCRIPT_DIR/fixtures/old-config.json" ]; then
Expand All @@ -119,8 +107,8 @@ else
skip_test "Config backwards compat" "old-config.json fixture not found"
fi

# 8. Broken config graceful handling
echo " [8/8] Broken config handling..."
# 7. Broken config graceful handling
echo " [7/10] Broken config handling..."
if [ -f "$SCRIPT_DIR/fixtures/broken-config.json" ]; then
cp "$SCRIPT_DIR/fixtures/broken-config.json" "$CONFIG_DIR/opencode.json"
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
Expand All @@ -135,6 +123,86 @@ else
skip_test "Broken config handling" "broken-config.json fixture not found"
fi

# 8. Yolo deny enforcement — deny rules block even with --yolo (#372, #377)
echo " [8/10] Yolo deny enforcement..."
if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
# Create a config with an explicit deny rule
# App reads from $XDG_CONFIG_HOME/altimate-code/opencode.jsonc
DENY_CONFIG_DIR=$(mktemp -d /tmp/sanity-deny-config-XXXXXX)
DENY_MARKER="$DENY_CONFIG_DIR/deny-marker"
mkdir -p "$DENY_CONFIG_DIR/altimate-code"
cat > "$DENY_CONFIG_DIR/altimate-code/opencode.jsonc" <<DENYEOF
{
"permission": {
"bash": {
"*": "allow",
"touch ${DENY_MARKER}": "deny",
"touch /tmp/sanity-deny-*": "deny"
}
}
}
DENYEOF
# Use an observable side-effect: deny "touch" on a marker file.
# If deny enforcement works, the marker file will NOT be created.
DENY_OUTPUT=$(XDG_CONFIG_HOME="$DENY_CONFIG_DIR" timeout 30 altimate run --max-turns 2 --yolo --format json \
"run this exact bash command: touch $DENY_MARKER" 2>&1 || true)
# Primary check: the marker file must not exist (deny blocked execution)
if [ -f "$DENY_MARKER" ]; then
echo " FAIL: yolo mode bypassed deny rule — denied command was executed"
FAIL_COUNT=$((FAIL_COUNT + 1))
elif echo "$DENY_OUTPUT" | grep -qi "denied\|blocked\|BLOCKED by deny rule\|not allowed"; then
echo " PASS: yolo deny rule explicitly blocked command"
PASS_COUNT=$((PASS_COUNT + 1))
elif [ -z "$DENY_OUTPUT" ]; then
echo " FAIL: no output from deny enforcement test"
FAIL_COUNT=$((FAIL_COUNT + 1))
else
# Model may have refused on its own — marker absent so still safe
echo " PASS: yolo deny rule (command not executed, marker absent)"
PASS_COUNT=$((PASS_COUNT + 1))
fi
rm -rf "$DENY_CONFIG_DIR"
else
skip_test "Yolo deny enforcement" "no ANTHROPIC_API_KEY"
fi

# 9. Missing API key handling (no unhandled exceptions)
echo " [9/10] Missing API key handling..."
SAVED_KEY="${ANTHROPIC_API_KEY:-}"
unset ANTHROPIC_API_KEY
OUTPUT=$(timeout 10 altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
assert_not_contains "$OUTPUT" "TypeError" "no TypeError on missing key"
assert_not_contains "$OUTPUT" "Cannot read properties" "no unhandled error on missing key"
if [ -n "$SAVED_KEY" ]; then
export ANTHROPIC_API_KEY="$SAVED_KEY"
fi

# 10. No internet — graceful error, not blank screen (#181)
echo " [10/10] No internet graceful handling..."
# Block all outbound HTTPS via multiple methods for reliability:
# - Set both lowercase and uppercase proxy vars to unreachable TEST-NET-1
# - Clear NO_PROXY to prevent bypass
# - Use unshare --net if available (network namespace isolation — most reliable)
if command -v unshare >/dev/null 2>&1; then
NO_NET_OUTPUT=$(timeout 15 unshare --net altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
else
NO_NET_OUTPUT=$(timeout 15 env \
https_proxy=http://192.0.2.1:1 http_proxy=http://192.0.2.1:1 \
HTTPS_PROXY=http://192.0.2.1:1 HTTP_PROXY=http://192.0.2.1:1 \
ALL_PROXY=http://192.0.2.1:1 NO_PROXY="" \
altimate run --max-turns 1 --yolo "hello" 2>&1 || true)
fi
assert_not_contains "$NO_NET_OUTPUT" "TypeError" "no TypeError without internet"
assert_not_contains "$NO_NET_OUTPUT" "Cannot read properties" "no unhandled error without internet"
# Should get some kind of connection/auth error, not a blank hang
if [ -z "$NO_NET_OUTPUT" ]; then
echo " FAIL: no output at all without internet (blank screen)"
FAIL_COUNT=$((FAIL_COUNT + 1))
else
echo " PASS: produced output without internet ($(echo "$NO_NET_OUTPUT" | wc -l) lines)"
PASS_COUNT=$((PASS_COUNT + 1))
fi

# Cleanup
rm -rf "$WORKDIR"

Expand Down
Loading
Loading