BuildKite CI (#40)

* BuildKite CI

Author: Mudit

.buildkite/premerge.definition.yaml

@@ -0,0 +1,11 @@
+agent_queue_id: trigger-pipelines
+description: C Example Project premerge pipeline
+github:
+  branch_configuration: []
+  default_branch: master
+  pull_request_branch_filter_configuration: []
+teams:
+- name: Everyone
+  permission: BUILD_AND_READ
+- name: spatialos-eng/worker
+  permission: MANAGE_BUILD_AND_READ

.buildkite/premerge.steps.yaml

@@ -0,0 +1,54 @@
+common: &common
+  timeout_in_minutes: 60
+  retry:
+    automatic:
+        # These are designed to trap and retry failures because agent lost connection.
+        # Agent exits with -1 in this case.
+      - exit_status: -1
+        limit: 3
+      - exit_status: 255
+        limit: 3
+      - exit_status: 128
+        limit: 3
+
+linux_agent: &linux_agent
+  agents:
+    - "agent_count=1"
+    - "capable_of_building=worker-sdk"
+    - "environment=production"
+    - "permission_set=builder"
+    - "platform=linux"
+    - "scaler_version=2"
+    - "machine_type=half"
+    - "queue=${CI_LINUX_BUILDER_QUEUE:-v4-20-03-23-110204-bk9814-9f960b76}"
+  <<: *common
+
+macos_agent: &macos_agent
+  agents:
+    - "queue=${CI_MACOS_BUILDER_QUEUE:-macos_worker-sdk_17b57b101ca8ff76}"
+    - "environment=production"
+  <<: *common
+
+windows_agent: &windows_agent
+  agents:
+    - "agent_count=1"
+    - "capable_of_building=worker-sdk"
+    - "environment=production"
+    - "machine_type=half"
+    - "permission_set=builder"
+    - "platform=windows"
+    - "scaler_version=2"
+    - "queue=${CI_WINDOWS_BUILDER_QUEUE:-v4-20-03-25-183029-bk9912-014014bb}"
+  <<: *common
+
+steps:
+  - label: "build-linux"
+    command: "ci/build.sh"
+    <<: *linux_agent
+  - label: "build-macos"
+    command: "ci/build.sh"
+    <<: *macos_agent
+  - label: "build-windows"
+    command: "bash -c ci/build.sh"
+    <<: *windows_agent
+

.toolsharerc

@@ -0,0 +1,5 @@
+pinned_tool:
+  - tool: "imp-ci"
+    version: "20200206.175502.ed41e47ac7"
+  - tool: "imp-tool"
+    version: "20200217.173900.11796cb28c"

ci/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:18.04
+
+RUN apt-get update \
+    && apt-get install -y \
+        cmake \
+        g++ \
+        gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG USER_ID
+ARG GROUP_ID
+
+RUN addgroup --gid $GROUP_ID user \
+    && adduser --disabled-password --uid $USER_ID --gid $GROUP_ID user
+USER user

ci/build.sh

@@ -4,12 +4,85 @@ set -e -u -x -o pipefail
 
 cd "$(dirname "$0")/.."
 
-PLATFORM=""
+if [[ -z ${BUILDKITE+x} ]]; then
+  echo "This script is intended to run in CI only."
+  exit 1
+fi
+
+SPATIALOS_TOOLBELT_VERSION="latest"
+
+TOOLBELT_PLATFORM=""
+SPATIAL_PLATFORM=""
+SPATIAL_BINARY=""
+
+# Setup platform specific environment variables.
 if [[ "$(uname -s)" == "Linux" ]]; then
-  PLATFORM="linux"
+  TOOLBELT_PLATFORM="linux"
+  SPATIAL_PLATFORM="linux"
+  SPATIAL_BINARY="spatial"
 elif [[ "$(uname -s)" == "Darwin" ]]; then
-  PLATFORM="macos"
+  TOOLBELT_PLATFORM="mac"
+  SPATIAL_PLATFORM="macos"
+  SPATIAL_BINARY="spatial"
+
+  imp-tool-bootstrap subscribe \
+    --use_gcs_oidc_auth=false \
+    --gcs_credentials_type=auto \
+    --tools=imp-tool
+else
+  TOOLBELT_PLATFORM="win"
+  SPATIAL_PLATFORM="windows"
+  SPATIAL_BINARY="spatial.exe"
+fi
+
+# Create temporary directories for the spatial binary and authentication token.
+export AUTH_DIR="$(mktemp -d)"
+export TOOLS_DIR="$(mktemp -d)"
+
+# Retrieve the authentication token.
+if [[ "$SPATIAL_PLATFORM" == "macos" ]]; then
+  imp-vault read-key \
+      --environment=production \
+      --field=token \
+      --key=secret/sync.v1/dev-workflow/production-buildkite/buildkite-agents/spatialos-service-account/ci/improbable/worker-blank-projects-ci-token \
+      --vault_role=continuous-integration-production-improbable-iam \
+      --write_to="${AUTH_DIR}/oauth2_refresh_token"
+else
+  imp-ci secrets read \
+      --environment=production \
+      --buildkite-org=improbable \
+      --secret-type=spatialos-service-account \
+      --secret-name=worker-blank-projects-ci-token \
+      --field=token \
+      --write-to="${AUTH_DIR}/oauth2_refresh_token"
+fi
+
+# Retrieve the spatial binary.
+curl -Ls -o "${TOOLS_DIR}/${SPATIAL_BINARY}.tmp" "https://console.improbable.io/toolbelt/download/${SPATIALOS_TOOLBELT_VERSION}/${TOOLBELT_PLATFORM}" || exit 1
+chmod +x "${TOOLS_DIR}/${SPATIAL_BINARY}.tmp"
+mv "${TOOLS_DIR}/${SPATIAL_BINARY}.tmp" "${TOOLS_DIR}/${SPATIAL_BINARY}"
+
+# Build
+if [[ "$SPATIAL_PLATFORM" == "linux" ]]; then
+  docker build \
+    -t c_exampe_project_image \
+    -f ci/Dockerfile \
+    --build-arg USER_ID=$(id -u) \
+    --build-arg GROUP_ID=$(id -g) \
+    .
+  docker run \
+    --rm \
+    --volume "${TOOLS_DIR}":/build/tools \
+    --env TOOLS_DIR=/build/tools \
+    --volume "${AUTH_DIR}":/build/auth \
+    --env AUTH_DIR=/build/auth \
+    --volume $(pwd):/code \
+    --workdir /code \
+    c_exampe_project_image \
+    /bin/bash -c 'export PATH="/code/ci:${PATH}"; \
+      spatial build --target '"$SPATIAL_PLATFORM"
 else
-  PLATFORM="windows"
+  # Add the spatial wrapper script to the path.
+  export PATH="$(pwd)/ci":"$PATH"
+  spatial build --target $SPATIAL_PLATFORM
 fi
-spatial build --target $PLATFORM --log_level=debug

ci/spatial

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+"${TOOLS_DIR}"/spatial --log_level=debug --oauth2_client_cli_token_directory="${AUTH_DIR}" "$@"

ci/spatial.cmd

@@ -0,0 +1 @@
+bash "%~dp0spatial" %*