diff --git a/README.md b/README.md index db64112ce9..e7b81b01f5 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [additional\_github\_apps](#input\_additional\_github\_apps) | Additional GitHub Apps for distributing API rate limit usage. Each must be installed on the same repos/orgs as the primary app. |
list(object({
key_base64 = optional(string)
key_base64_ssm = optional(object({ arn = string, name = string }))
id = optional(string)
id_ssm = optional(object({ arn = string, name = string }))
installation_id = optional(string)
installation_id_ssm = optional(object({ arn = string, name = string }))
})) | `[]` | no |
| [ami](#input\_ami) | AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place.object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter_arn = optional(string, null)
kms_key_arn = optional(string, null)
}) | `null` | no |
| [ami\_housekeeper\_cleanup\_config](#input\_ami\_housekeeper\_cleanup\_config) | Configuration for AMI cleanup.object({
amiFilters = optional(list(object({
Name = string
Values = list(string)
})),
[{
Name : "state",
Values : ["available"],
},
{
Name : "image-type",
Values : ["machine"],
}]
)
dryRun = optional(bool, false)
launchTemplateNames = optional(list(string))
maxItems = optional(number)
minimumDaysOld = optional(number, 30)
ssmParameterNames = optional(list(string))
}) | `{}` | no |
| [ami\_housekeeper\_lambda\_s3\_key](#input\_ami\_housekeeper\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
diff --git a/examples/multi-runner/README.md b/examples/multi-runner/README.md
index 8f14b48503..e7609a8c77 100644
--- a/examples/multi-runner/README.md
+++ b/examples/multi-runner/README.md
@@ -16,6 +16,10 @@ For exact match, all the labels defined in the workflow should be present in the
For the list of provided runner configurations, there will be a single webhook and only a single GitHub App to receive the notifications for all types of workflow triggers.
+## Multiple GitHub Apps (rate limit distribution)
+
+This example also shows how to optionally configure multiple GitHub Apps via the `additional_github_apps` variable. When configured, the control-plane lambdas (scale-up, scale-down, pool, job-retry) randomly select an app for each GitHub API call, spreading the rate limit usage across all apps. Only the primary app needs a webhook URL configured in GitHub.
+
## Lambda distribution
Per combination of OS and architecture a lambda distribution syncer will be created. For this example there will be three instances (windows X64, linux X64, linux ARM).
diff --git a/examples/multi-runner/main.tf b/examples/multi-runner/main.tf
index 13df82a0bb..de92264430 100644
--- a/examples/multi-runner/main.tf
+++ b/examples/multi-runner/main.tf
@@ -117,6 +117,17 @@ module "runners" {
webhook_secret = random_id.random.hex
}
+ # Uncomment to distribute GitHub API rate limit usage across multiple GitHub Apps.
+ # Each additional app must be installed on the same repos/orgs as the primary app.
+ # The control-plane lambdas will randomly select an app for each API call.
+ # additional_github_apps = [
+ # {
+ # key_base64 = var.additional_github_app_0.key_base64
+ # id = var.additional_github_app_0.id
+ # installation_id = var.additional_github_app_0.installation_id # optional, avoids an API call
+ # },
+ # ]
+
# Deploy webhook using the EventBridge
eventbridge = {
enable = true
diff --git a/lambdas/functions/control-plane/src/github/auth.test.ts b/lambdas/functions/control-plane/src/github/auth.test.ts
index 80b4314182..64c5925194 100644
--- a/lambdas/functions/control-plane/src/github/auth.test.ts
+++ b/lambdas/functions/control-plane/src/github/auth.test.ts
@@ -5,7 +5,7 @@ import { RequestInterface, RequestParameters } from '@octokit/types';
import { getParameter } from '@aws-github-runner/aws-ssm-util';
import * as nock from 'nock';
-import { createGithubAppAuth, createOctokitClient } from './auth';
+import { createGithubAppAuth, createOctokitClient, getStoredInstallationId, resetAppCredentialsCache } from './auth';
import { describe, it, expect, beforeEach, vi } from 'vitest';
type MockProxylist(object({
key_base64 = optional(string)
key_base64_ssm = optional(object({ arn = string, name = string }))
id = optional(string)
id_ssm = optional(object({ arn = string, name = string }))
installation_id = optional(string)
installation_id_ssm = optional(object({ arn = string, name = string }))
})) | `[]` | no |
| [ami\_housekeeper\_cleanup\_config](#input\_ami\_housekeeper\_cleanup\_config) | Configuration for AMI cleanup. | object({
maxItems = optional(number)
minimumDaysOld = optional(number)
amiFilters = optional(list(object({
Name = string
Values = list(string)
})))
launchTemplateNames = optional(list(string))
ssmParameterNames = optional(list(string))
dryRun = optional(bool)
}) | `{}` | no |
| [ami\_housekeeper\_lambda\_memory\_size](#input\_ami\_housekeeper\_lambda\_memory\_size) | Memory size limit in MB of the lambda. | `number` | `256` | no |
| [ami\_housekeeper\_lambda\_s3\_key](#input\_ami\_housekeeper\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
diff --git a/modules/multi-runner/main.tf b/modules/multi-runner/main.tf
index 905cc7f793..5ef97c659d 100644
--- a/modules/multi-runner/main.tf
+++ b/modules/multi-runner/main.tf
@@ -3,9 +3,22 @@ locals {
"ghr:environment" = var.prefix
})
+ _primary_app_id = coalesce(var.github_app.id_ssm, module.ssm.parameters.github_app_id)
+ _primary_app_key_base64 = coalesce(var.github_app.key_base64_ssm, module.ssm.parameters.github_app_key_base64)
+
github_app_parameters = {
- id = coalesce(var.github_app.id_ssm, module.ssm.parameters.github_app_id)
- key_base64 = coalesce(var.github_app.key_base64_ssm, module.ssm.parameters.github_app_key_base64)
+ id = concat(
+ [local._primary_app_id],
+ [for p in module.ssm.additional_app_parameters : p.id]
+ )
+ key_base64 = concat(
+ [local._primary_app_key_base64],
+ [for p in module.ssm.additional_app_parameters : p.key_base64]
+ )
+ installation_id = concat(
+ [null],
+ [for p in module.ssm.additional_app_parameters : p.installation_id]
+ )
webhook_secret = coalesce(var.github_app.webhook_secret_ssm, module.ssm.parameters.github_app_webhook_secret)
}
diff --git a/modules/multi-runner/outputs.tf b/modules/multi-runner/outputs.tf
index 7ce7171faf..b22278accf 100644
--- a/modules/multi-runner/outputs.tf
+++ b/modules/multi-runner/outputs.tf
@@ -45,11 +45,20 @@ output "webhook" {
}
output "ssm_parameters" {
- value = { for k, v in local.github_app_parameters : k => {
- name = v.name
- arn = v.arn
- }
- }
+ value = merge(
+ { for idx, v in local.github_app_parameters.id : "github_app_id_${idx}" => {
+ name = v.name
+ arn = v.arn
+ } },
+ { for idx, v in local.github_app_parameters.key_base64 : "github_app_key_base64_${idx}" => {
+ name = v.name
+ arn = v.arn
+ } },
+ { "github_app_webhook_secret" = {
+ name = local.github_app_parameters.webhook_secret.name
+ arn = local.github_app_parameters.webhook_secret.arn
+ } },
+ )
}
output "instance_termination_watcher" {
diff --git a/modules/multi-runner/ssm.tf b/modules/multi-runner/ssm.tf
index 6a3a234e6f..3e4b740fdd 100644
--- a/modules/multi-runner/ssm.tf
+++ b/modules/multi-runner/ssm.tf
@@ -1,7 +1,8 @@
module "ssm" {
- source = "../ssm"
- kms_key_arn = var.kms_key_arn
- path_prefix = "${local.ssm_root_path}/${var.ssm_paths.app}"
- github_app = var.github_app
- tags = local.tags
+ source = "../ssm"
+ kms_key_arn = var.kms_key_arn
+ path_prefix = "${local.ssm_root_path}/${var.ssm_paths.app}"
+ github_app = var.github_app
+ additional_github_apps = var.additional_github_apps
+ tags = local.tags
}
diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf
index faf9c946c4..a3d8663358 100644
--- a/modules/multi-runner/variables.tf
+++ b/modules/multi-runner/variables.tf
@@ -36,6 +36,38 @@ variable "github_app" {
}
+variable "additional_github_apps" {
+ description = <<-EOF
+ Additional GitHub Apps for round-robin API rate limit distribution.
+
+ The primary app (var.github_app) is always included and is the one whose
+ webhook secret is used for incoming webhook signature validation. Only the
+ primary app needs a webhook configured in GitHub.
+
+ Additional apps listed here are used exclusively by the control-plane
+ lambdas (scale-up, scale-down, pool, job-retry) which randomly select an
+ app for each GitHub API call. Each additional app must be installed on the
+ same repositories/organizations as the primary app.
+ EOF
+ type = list(object({
+ key_base64 = optional(string)
+ key_base64_ssm = optional(object({ arn = string, name = string }))
+ id = optional(string)
+ id_ssm = optional(object({ arn = string, name = string }))
+ installation_id = optional(string)
+ installation_id_ssm = optional(object({ arn = string, name = string }))
+ }))
+ default = []
+ validation {
+ condition = alltrue([
+ for app in var.additional_github_apps :
+ (app.key_base64 != null || app.key_base64_ssm != null) &&
+ (app.id != null || app.id_ssm != null)
+ ])
+ error_message = "Each additional GitHub app must provide either key_base64 or key_base64_ssm, and either id or id_ssm."
+ }
+}
+
variable "prefix" {
description = "The prefix used for naming resources"
type = string
diff --git a/modules/runners/README.md b/modules/runners/README.md
index 231e542fa6..3db9c4954c 100644
--- a/modules/runners/README.md
+++ b/modules/runners/README.md
@@ -162,7 +162,7 @@ yarn run dist
| [enable\_userdata](#input\_enable\_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no |
| [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no |
| [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB..However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com\| | `string` | `null` | no |
-| [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | object({
key_base64 = map(string)
id = map(string)
}) | n/a | yes |
+| [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters.object({
key_base64 = list(map(string))
id = list(map(string))
installation_id = list(object({ name = string, arn = string }))
}) | n/a | yes |
| [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})) | `[]` | no |
| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no |
| [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no |
diff --git a/modules/runners/job-retry/main.tf b/modules/runners/job-retry/main.tf
index eba478b214..d5455951d0 100644
--- a/modules/runners/job-retry/main.tf
+++ b/modules/runners/job-retry/main.tf
@@ -3,14 +3,15 @@ locals {
name = "job-retry"
environment_variables = {
- ENABLE_ORGANIZATION_RUNNERS = var.config.enable_organization_runners
- ENABLE_METRIC_JOB_RETRY = var.config.metrics.enable && var.config.metrics.metric.enable_job_retry
- ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.metrics.enable && var.config.metrics.metric.enable_github_app_rate_limit
- GHES_URL = var.config.ghes_url
- USER_AGENT = var.config.user_agent
- JOB_QUEUE_SCALE_UP_URL = var.config.sqs_build_queue.url
- PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name
- PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name
+ ENABLE_ORGANIZATION_RUNNERS = var.config.enable_organization_runners
+ ENABLE_METRIC_JOB_RETRY = var.config.metrics.enable && var.config.metrics.metric.enable_job_retry
+ ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.metrics.enable && var.config.metrics.metric.enable_github_app_rate_limit
+ GHES_URL = var.config.ghes_url
+ USER_AGENT = var.config.user_agent
+ JOB_QUEUE_SCALE_UP_URL = var.config.sqs_build_queue.url
+ PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github_app_parameters.id : p.name])
+ PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github_app_parameters.key_base64 : p.name])
+ PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github_app_parameters.installation_id : p != null ? p.name : ""])
}
config = merge(var.config, {
@@ -62,11 +63,14 @@ resource "aws_iam_role_policy" "job_retry" {
name = "job_retry-policy"
role = module.job_retry.lambda.role.name
policy = templatefile("${path.module}/policies/lambda.json", {
- kms_key_arn = var.config.kms_key_arn != null ? var.config.kms_key_arn : ""
- sqs_build_queue_arn = var.config.sqs_build_queue.arn
- sqs_job_retry_queue_arn = aws_sqs_queue.job_retry_check_queue.arn
- github_app_id_arn = var.config.github_app_parameters.id.arn
- github_app_key_base64_arn = var.config.github_app_parameters.key_base64.arn
+ kms_key_arn = var.config.kms_key_arn != null ? var.config.kms_key_arn : ""
+ sqs_build_queue_arn = var.config.sqs_build_queue.arn
+ sqs_job_retry_queue_arn = aws_sqs_queue.job_retry_check_queue.arn
+ github_app_parameter_arns = jsonencode(concat(
+ [for p in var.config.github_app_parameters.id : p.arn],
+ [for p in var.config.github_app_parameters.key_base64 : p.arn],
+ [for p in var.config.github_app_parameters.installation_id : p.arn if p != null],
+ ))
})
}
diff --git a/modules/runners/job-retry/policies/lambda.json b/modules/runners/job-retry/policies/lambda.json
index 591ec04790..6bc8c1c5c7 100644
--- a/modules/runners/job-retry/policies/lambda.json
+++ b/modules/runners/job-retry/policies/lambda.json
@@ -6,10 +6,7 @@
"Action": [
"ssm:GetParameter"
],
- "Resource": [
- "${github_app_key_base64_arn}",
- "${github_app_id_arn}"
- ]
+ "Resource": ${github_app_parameter_arns}
},
{
"Effect": "Allow",
diff --git a/modules/runners/job-retry/variables.tf b/modules/runners/job-retry/variables.tf
index 7ccfdf63b3..cb010d7552 100644
--- a/modules/runners/job-retry/variables.tf
+++ b/modules/runners/job-retry/variables.tf
@@ -44,8 +44,9 @@ variable "config" {
ghes_url = optional(string, null)
user_agent = optional(string, null)
github_app_parameters = object({
- key_base64 = map(string)
- id = map(string)
+ key_base64 = list(map(string))
+ id = list(map(string))
+ installation_id = list(object({ name = string, arn = string }))
})
kms_key_arn = optional(string, null)
lambda_event_source_mapping_batch_size = optional(number, 10)
diff --git a/modules/runners/policies/lambda-scale-down.json b/modules/runners/policies/lambda-scale-down.json
index d35be746b7..9ff62aba55 100644
--- a/modules/runners/policies/lambda-scale-down.json
+++ b/modules/runners/policies/lambda-scale-down.json
@@ -48,10 +48,7 @@
"Action": [
"ssm:GetParameter"
],
- "Resource": [
- "${github_app_key_base64_arn}",
- "${github_app_id_arn}"
- ]
+ "Resource": ${github_app_parameter_arns}
%{ if kms_key_arn != "" ~}
},
{
diff --git a/modules/runners/policies/lambda-scale-up.json b/modules/runners/policies/lambda-scale-up.json
index 3b16e710d5..245548c39b 100644
--- a/modules/runners/policies/lambda-scale-up.json
+++ b/modules/runners/policies/lambda-scale-up.json
@@ -32,12 +32,7 @@
"Action": [
"ssm:GetParameter"
],
- "Resource": [
- "${github_app_key_base64_arn}",
- "${github_app_id_arn}",
- "${ssm_config_path}/*",
- "${ssm_ami_id_parameter_arn}"
- ]
+ "Resource": ${github_app_parameter_arns}
},
{
"Effect": "Allow",
diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf
index ced73825d4..202389a846 100644
--- a/modules/runners/pool/main.tf
+++ b/modules/runners/pool/main.tf
@@ -17,38 +17,39 @@ resource "aws_lambda_function" "pool" {
environment {
variables = {
- AMI_ID_SSM_PARAMETER_NAME = var.config.ami_id_ssm_parameter_name
- DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate
- ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral
- ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config
- ENVIRONMENT = var.config.prefix
- GHES_URL = var.config.ghes.url
- USER_AGENT = var.config.user_agent
- INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy
- INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price
- INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type
- INSTANCE_TYPES = join(",", var.config.instance_types)
- LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name
- LOG_LEVEL = var.config.lambda.log_level
- NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1
- PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name
- PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name
- POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false"
- RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes
- RUNNER_LABELS = lower(join(",", var.config.runner.labels))
- RUNNER_GROUP_NAME = var.config.runner.group_name
- RUNNER_NAME_PREFIX = var.config.runner.name_prefix
- RUNNER_OWNER = var.config.runner.pool_owner
- SSM_TOKEN_PATH = var.config.ssm_token_path
- SSM_CONFIG_PATH = var.config.ssm_config_path
- SUBNET_IDS = join(",", var.config.subnet_ids)
- POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool"
- POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
- POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
- POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
- ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.runner.enable_on_demand_failover_for_errors)
- SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags
- SCALE_ERRORS = jsonencode(var.config.runner.scale_errors)
+ AMI_ID_SSM_PARAMETER_NAME = var.config.ami_id_ssm_parameter_name
+ DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate
+ ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral
+ ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config
+ ENVIRONMENT = var.config.prefix
+ GHES_URL = var.config.ghes.url
+ USER_AGENT = var.config.user_agent
+ INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy
+ INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price
+ INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type
+ INSTANCE_TYPES = join(",", var.config.instance_types)
+ LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name
+ LOG_LEVEL = var.config.lambda.log_level
+ NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1
+ PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github_app_parameters.id : p.name])
+ PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github_app_parameters.key_base64 : p.name])
+ PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github_app_parameters.installation_id : p != null ? p.name : ""])
+ POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false"
+ RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes
+ RUNNER_LABELS = lower(join(",", var.config.runner.labels))
+ RUNNER_GROUP_NAME = var.config.runner.group_name
+ RUNNER_NAME_PREFIX = var.config.runner.name_prefix
+ RUNNER_OWNER = var.config.runner.pool_owner
+ SSM_TOKEN_PATH = var.config.ssm_token_path
+ SSM_CONFIG_PATH = var.config.ssm_config_path
+ SUBNET_IDS = join(",", var.config.subnet_ids)
+ POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool"
+ POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
+ POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
+ POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
+ ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.runner.enable_on_demand_failover_for_errors)
+ SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags
+ SCALE_ERRORS = jsonencode(var.config.runner.scale_errors)
}
}
@@ -89,11 +90,14 @@ resource "aws_iam_role_policy" "pool" {
policy = templatefile("${path.module}/policies/lambda-pool.json", {
arn_ssm_parameters_path_config = var.config.arn_ssm_parameters_path_config
arn_runner_instance_role = var.config.runner.role.arn
- github_app_id_arn = var.config.github_app_parameters.id.arn
- github_app_key_base64_arn = var.config.github_app_parameters.key_base64.arn
- kms_key_arn = var.config.kms_key_arn
- ami_kms_key_arn = var.config.ami_kms_key_arn
- ssm_ami_id_parameter_arn = var.config.ami_id_ssm_parameter_arn
+ github_app_parameter_arns = jsonencode(concat(
+ [for p in var.config.github_app_parameters.id : p.arn],
+ [for p in var.config.github_app_parameters.key_base64 : p.arn],
+ [for p in var.config.github_app_parameters.installation_id : p.arn if p != null],
+ ))
+ kms_key_arn = var.config.kms_key_arn
+ ami_kms_key_arn = var.config.ami_kms_key_arn
+ ssm_ami_id_parameter_arn = var.config.ami_id_ssm_parameter_arn
})
}
diff --git a/modules/runners/pool/policies/lambda-pool.json b/modules/runners/pool/policies/lambda-pool.json
index b0360a825c..8910a3972f 100644
--- a/modules/runners/pool/policies/lambda-pool.json
+++ b/modules/runners/pool/policies/lambda-pool.json
@@ -53,10 +53,7 @@
"Action": [
"ssm:GetParameter"
],
- "Resource": [
- "${github_app_key_base64_arn}",
- "${github_app_id_arn}"
- ]
+ "Resource": ${github_app_parameter_arns}
%{ if kms_key_arn != "" ~}
},
{
diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf
index d005f3479e..33834ff4bd 100644
--- a/modules/runners/pool/variables.tf
+++ b/modules/runners/pool/variables.tf
@@ -24,8 +24,9 @@ variable "config" {
ssl_verify = string
})
github_app_parameters = object({
- key_base64 = map(string)
- id = map(string)
+ key_base64 = list(map(string))
+ id = list(map(string))
+ installation_id = list(object({ name = string, arn = string }))
})
subnet_ids = list(string)
runner = object({
diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf
index a36f3b0532..2bbece0d02 100644
--- a/modules/runners/scale-down.tf
+++ b/modules/runners/scale-down.tf
@@ -22,23 +22,24 @@ resource "aws_lambda_function" "scale_down" {
environment {
variables = {
- ENVIRONMENT = var.prefix
- ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit
- GHES_URL = var.ghes_url
- USER_AGENT = var.user_agent
- LOG_LEVEL = var.log_level
- MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os])
- NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1
- PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name
- PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name
- POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false"
- RUNNER_BOOT_TIME_IN_MINUTES = var.runner_boot_time_in_minutes
- SCALE_DOWN_CONFIG = jsonencode(var.idle_config)
- POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-down"
- POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace
- POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
- POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
- POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
+ ENVIRONMENT = var.prefix
+ ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit
+ GHES_URL = var.ghes_url
+ USER_AGENT = var.user_agent
+ LOG_LEVEL = var.log_level
+ MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os])
+ NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1
+ PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.github_app_parameters.id : p.name])
+ PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.github_app_parameters.key_base64 : p.name])
+ PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.github_app_parameters.installation_id : p != null ? p.name : ""])
+ POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false"
+ RUNNER_BOOT_TIME_IN_MINUTES = var.runner_boot_time_in_minutes
+ SCALE_DOWN_CONFIG = jsonencode(var.idle_config)
+ POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-down"
+ POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace
+ POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
+ POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
+ POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
}
}
@@ -96,10 +97,13 @@ resource "aws_iam_role_policy" "scale_down" {
name = "scale-down-policy"
role = aws_iam_role.scale_down.name
policy = templatefile("${path.module}/policies/lambda-scale-down.json", {
- environment = var.prefix
- github_app_id_arn = var.github_app_parameters.id.arn
- github_app_key_base64_arn = var.github_app_parameters.key_base64.arn
- kms_key_arn = local.kms_key_arn
+ environment = var.prefix
+ github_app_parameter_arns = jsonencode(concat(
+ [for p in var.github_app_parameters.id : p.arn],
+ [for p in var.github_app_parameters.key_base64 : p.arn],
+ [for p in var.github_app_parameters.installation_id : p.arn if p != null],
+ ))
+ kms_key_arn = local.kms_key_arn
})
}
diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf
index 73bf4b6df6..6130326da1 100644
--- a/modules/runners/scale-up.tf
+++ b/modules/runners/scale-up.tf
@@ -25,43 +25,44 @@ resource "aws_lambda_function" "scale_up" {
architectures = [var.lambda_architecture]
environment {
variables = {
- AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name
- DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate
- ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners
- ENABLE_JIT_CONFIG = var.enable_jit_config
- ENABLE_JOB_QUEUED_CHECK = local.enable_job_queued_check
- ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit
- ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners
- ENVIRONMENT = var.prefix
- GHES_URL = var.ghes_url
- USER_AGENT = var.user_agent
- INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy
- INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price
- INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type
- INSTANCE_TYPES = join(",", var.instance_types)
- LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name
- LOG_LEVEL = var.log_level
- MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os])
- NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1
- PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name
- PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name
- POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false"
- POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace
- POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
- POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
- POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
- RUNNER_LABELS = lower(join(",", var.runner_labels))
- RUNNER_GROUP_NAME = var.runner_group_name
- RUNNER_NAME_PREFIX = var.runner_name_prefix
- RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count
- POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up"
- SSM_TOKEN_PATH = local.token_path
- SSM_CONFIG_PATH = "${var.ssm_paths.root}/${var.ssm_paths.config}"
- SSM_PARAMETER_STORE_TAGS = local.parameter_store_tags
- SUBNET_IDS = join(",", var.subnet_ids)
- ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.enable_on_demand_failover_for_errors)
- SCALE_ERRORS = jsonencode(var.scale_errors)
- JOB_RETRY_CONFIG = jsonencode(local.job_retry_config)
+ AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name
+ DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate
+ ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners
+ ENABLE_JIT_CONFIG = var.enable_jit_config
+ ENABLE_JOB_QUEUED_CHECK = local.enable_job_queued_check
+ ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit
+ ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners
+ ENVIRONMENT = var.prefix
+ GHES_URL = var.ghes_url
+ USER_AGENT = var.user_agent
+ INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy
+ INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price
+ INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type
+ INSTANCE_TYPES = join(",", var.instance_types)
+ LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name
+ LOG_LEVEL = var.log_level
+ MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os])
+ NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1
+ PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.github_app_parameters.id : p.name])
+ PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.github_app_parameters.key_base64 : p.name])
+ PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.github_app_parameters.installation_id : p != null ? p.name : ""])
+ POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false"
+ POWERTOOLS_METRICS_NAMESPACE = var.metrics.namespace
+ POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
+ POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
+ POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
+ RUNNER_LABELS = lower(join(",", var.runner_labels))
+ RUNNER_GROUP_NAME = var.runner_group_name
+ RUNNER_NAME_PREFIX = var.runner_name_prefix
+ RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count
+ POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up"
+ SSM_TOKEN_PATH = local.token_path
+ SSM_CONFIG_PATH = "${var.ssm_paths.root}/${var.ssm_paths.config}"
+ SSM_PARAMETER_STORE_TAGS = local.parameter_store_tags
+ SUBNET_IDS = join(",", var.subnet_ids)
+ ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.enable_on_demand_failover_for_errors)
+ SCALE_ERRORS = jsonencode(var.scale_errors)
+ JOB_RETRY_CONFIG = jsonencode(local.job_retry_config)
}
}
@@ -117,14 +118,17 @@ resource "aws_iam_role_policy" "scale_up" {
name = "scale-up-policy"
role = aws_iam_role.scale_up.name
policy = templatefile("${path.module}/policies/lambda-scale-up.json", {
- arn_runner_instance_role = aws_iam_role.runner.arn
- sqs_arn = var.sqs_build_queue.arn
- github_app_id_arn = var.github_app_parameters.id.arn
- github_app_key_base64_arn = var.github_app_parameters.key_base64.arn
- ssm_config_path = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm_paths.root}/${var.ssm_paths.config}"
- kms_key_arn = local.kms_key_arn
- ami_kms_key_arn = local.ami_kms_key_arn
- ssm_ami_id_parameter_arn = local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : var.ami.id_ssm_parameter_arn
+ arn_runner_instance_role = aws_iam_role.runner.arn
+ sqs_arn = var.sqs_build_queue.arn
+ github_app_parameter_arns = jsonencode(concat(
+ [for p in var.github_app_parameters.id : p.arn],
+ [for p in var.github_app_parameters.key_base64 : p.arn],
+ [for p in var.github_app_parameters.installation_id : p.arn if p != null],
+ ["arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm_paths.root}/${var.ssm_paths.config}/*"]
+ ))
+ kms_key_arn = local.kms_key_arn
+ ami_kms_key_arn = local.ami_kms_key_arn
+ ssm_ami_id_parameter_arn = local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : var.ami.id_ssm_parameter_arn
})
}
diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf
index db58a86b42..925fb6b233 100644
--- a/modules/runners/variables.tf
+++ b/modules/runners/variables.tf
@@ -192,10 +192,23 @@ variable "enable_organization_runners" {
}
variable "github_app_parameters" {
- description = "Parameter Store for GitHub App Parameters."
+ description = <<-EOF
+ Parameter Store for GitHub App Parameters.
+
+ Supports multiple GitHub Apps for round-robin API rate limit distribution.
+ Each list element corresponds to one GitHub App and is a map containing
+ `name` and `arn` keys referencing SSM parameters. The first element is the
+ primary app (the one whose webhook secret is used for incoming webhook
+ validation). All apps must be installed on the same repositories/organizations.
+
+ The control-plane lambdas (scale-up, scale-down, pool, job-retry) randomly
+ select an app from the list for each GitHub API call, distributing rate
+ limit consumption across all configured apps.
+ EOF
type = object({
- key_base64 = map(string)
- id = map(string)
+ key_base64 = list(map(string))
+ id = list(map(string))
+ installation_id = list(object({ name = string, arn = string }))
})
}
diff --git a/modules/ssm/outputs.tf b/modules/ssm/outputs.tf
index 4017f6ab3d..462c7265be 100644
--- a/modules/ssm/outputs.tf
+++ b/modules/ssm/outputs.tf
@@ -14,3 +14,22 @@ output "parameters" {
}
}
}
+
+output "additional_app_parameters" {
+ value = [
+ for idx, app in var.additional_github_apps : {
+ id = {
+ name = app.id_ssm != null ? app.id_ssm.name : aws_ssm_parameter.additional_github_app_id[idx].name
+ arn = app.id_ssm != null ? app.id_ssm.arn : aws_ssm_parameter.additional_github_app_id[idx].arn
+ }
+ key_base64 = {
+ name = app.key_base64_ssm != null ? app.key_base64_ssm.name : aws_ssm_parameter.additional_github_app_key_base64[idx].name
+ arn = app.key_base64_ssm != null ? app.key_base64_ssm.arn : aws_ssm_parameter.additional_github_app_key_base64[idx].arn
+ }
+ installation_id = app.installation_id != null || app.installation_id_ssm != null ? {
+ name = app.installation_id_ssm != null ? app.installation_id_ssm.name : aws_ssm_parameter.additional_github_app_installation_id[idx].name
+ arn = app.installation_id_ssm != null ? app.installation_id_ssm.arn : aws_ssm_parameter.additional_github_app_installation_id[idx].arn
+ } : null
+ }
+ ]
+}
diff --git a/modules/ssm/ssm.tf b/modules/ssm/ssm.tf
index 3f13333e68..f7002e3f1d 100644
--- a/modules/ssm/ssm.tf
+++ b/modules/ssm/ssm.tf
@@ -24,3 +24,30 @@ resource "aws_ssm_parameter" "github_app_webhook_secret" {
key_id = local.kms_key_arn
tags = var.tags
}
+
+resource "aws_ssm_parameter" "additional_github_app_id" {
+ for_each = { for idx, app in var.additional_github_apps : idx => app if app.id_ssm == null }
+ name = "${var.path_prefix}/additional_github_app_${each.key}_id"
+ type = "SecureString"
+ value = each.value.id
+ key_id = local.kms_key_arn
+ tags = var.tags
+}
+
+resource "aws_ssm_parameter" "additional_github_app_key_base64" {
+ for_each = { for idx, app in var.additional_github_apps : idx => app if app.key_base64_ssm == null }
+ name = "${var.path_prefix}/additional_github_app_${each.key}_key_base64"
+ type = "SecureString"
+ value = each.value.key_base64
+ key_id = local.kms_key_arn
+ tags = var.tags
+}
+
+resource "aws_ssm_parameter" "additional_github_app_installation_id" {
+ for_each = { for idx, app in var.additional_github_apps : idx => app if app.installation_id_ssm == null && nonsensitive(app.installation_id != null) }
+ name = "${var.path_prefix}/additional_github_app_${each.key}_installation_id"
+ type = "SecureString"
+ value = each.value.installation_id
+ key_id = local.kms_key_arn
+ tags = var.tags
+}
diff --git a/modules/ssm/variables.tf b/modules/ssm/variables.tf
index 1eb796aea7..d7387ecc30 100644
--- a/modules/ssm/variables.tf
+++ b/modules/ssm/variables.tf
@@ -45,6 +45,19 @@ variable "kms_key_arn" {
default = null
}
+variable "additional_github_apps" {
+ description = "Additional GitHub Apps for distributing API rate limit usage."
+ type = list(object({
+ key_base64 = optional(string)
+ key_base64_ssm = optional(object({ arn = string, name = string }))
+ id = optional(string)
+ id_ssm = optional(object({ arn = string, name = string }))
+ installation_id = optional(string)
+ installation_id_ssm = optional(object({ arn = string, name = string }))
+ }))
+ default = []
+}
+
variable "tags" {
description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment."
type = map(string)
diff --git a/variables.tf b/variables.tf
index 90769578c0..ae8b372f1c 100644
--- a/variables.tf
+++ b/variables.tf
@@ -67,6 +67,27 @@ variable "github_app" {
}
}
+variable "additional_github_apps" {
+ description = "Additional GitHub Apps for distributing API rate limit usage. Each must be installed on the same repos/orgs as the primary app."
+ type = list(object({
+ key_base64 = optional(string)
+ key_base64_ssm = optional(object({ arn = string, name = string }))
+ id = optional(string)
+ id_ssm = optional(object({ arn = string, name = string }))
+ installation_id = optional(string)
+ installation_id_ssm = optional(object({ arn = string, name = string }))
+ }))
+ default = []
+ validation {
+ condition = alltrue([
+ for app in var.additional_github_apps :
+ (app.key_base64 != null || app.key_base64_ssm != null) &&
+ (app.id != null || app.id_ssm != null)
+ ])
+ error_message = "Each additional GitHub app must provide either key_base64 or key_base64_ssm, and either id or id_ssm."
+ }
+}
+
variable "scale_down_schedule_expression" {
description = "Scheduler expression to check every x for scale down."
type = string