From e0f55183f12b0be2094a3bd0bc9ed88d3076be7b Mon Sep 17 00:00:00 2001 From: Samantha Date: Mon, 16 Jun 2025 17:18:57 -0400 Subject: [PATCH 1/8] grpc: Enable client-side health_v1 health checking --- grpc/client.go | 22 +++++++++++++++++++--- grpc/server.go | 17 +++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/grpc/client.go b/grpc/client.go index b7773cdf006..87ff82f7995 100644 --- a/grpc/client.go +++ b/grpc/client.go @@ -14,11 +14,13 @@ import ( "github.com/letsencrypt/boulder/cmd" bcreds "github.com/letsencrypt/boulder/grpc/creds" - // 'grpc/health' is imported for its init function, which causes clients to - // rely on the Health Service for load-balancing. // 'grpc/internal/resolver/dns' is imported for its init function, which // registers the SRV resolver. "google.golang.org/grpc/balancer/roundrobin" + + // 'grpc/health' is imported for its init function, which causes clients to + // rely on the Health Service for load-balancing as long as a + // "healthCheckConfig" is specified in the gRPC service config. _ "google.golang.org/grpc/health" _ "github.com/letsencrypt/boulder/grpc/internal/resolver/dns" @@ -61,7 +63,21 @@ func ClientSetup(c *cmd.GRPCClientConfig, tlsConfig *tls.Config, statsRegistry p creds := bcreds.NewClientCredentials(tlsConfig.RootCAs, tlsConfig.Certificates, hostOverride) return grpc.NewClient( target, - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, roundrobin.Name)), + grpc.WithDefaultServiceConfig( + fmt.Sprintf( + // By setting the service name to an empty string in + // healthCheckConfig, we're instructing the gRPC client to query + // the overall health status of each server. The grpc-go health + // server, as constructed by health.NewServer(), unconditionally + // sets the overall service (e.g. "") status to SERVING. If a + // specific service name were set, the server would need to + // explicitly transition that service to SERVING; otherwise, + // clients would receive a NOT_FOUND status and the connection + // would be marked as unhealthy (TRANSIENT_FAILURE). + `{"healthCheckConfig": {"serviceName": ""},"loadBalancingConfig": [{"%s":{}}]}`, + roundrobin.Name, + ), + ), grpc.WithTransportCredentials(creds), grpc.WithChainUnaryInterceptor(unaryInterceptors...), grpc.WithChainStreamInterceptor(streamInterceptors...), diff --git a/grpc/server.go b/grpc/server.go index 4029c77d270..e1daf24610a 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -129,6 +129,15 @@ func (sb *serverBuilder) Build(tlsConfig *tls.Config, statsRegistry prometheus.R } } + // Ensure that the health service has the same ClientNames as the other + // services, so that health checks can be performed by clients which are + // allowed to connect to the server. + healthService := sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName] + for as := range acceptedSANs { + healthService.ClientNames = append(healthService.ClientNames, as) + } + sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName] = healthService + creds, err := bcreds.NewServerCredentials(tlsConfig, acceptedSANs) if err != nil { return nil, err @@ -224,8 +233,12 @@ func (sb *serverBuilder) Build(tlsConfig *tls.Config, statsRegistry prometheus.R // initLongRunningCheck initializes a goroutine which will periodically check // the health of the provided service and update the health server accordingly. +// +// TODO(#8255): Remove the service parameter and instead rely on transitioning +// the overall health of the server (e.g. "") instead of individual services. func (sb *serverBuilder) initLongRunningCheck(shutdownCtx context.Context, service string, checkImpl func(context.Context) error) { // Set the initial health status for the service. + sb.healthSrv.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) sb.healthSrv.SetServingStatus(service, healthpb.HealthCheckResponse_NOT_SERVING) // check is a helper function that checks the health of the service and, if @@ -249,10 +262,14 @@ func (sb *serverBuilder) initLongRunningCheck(shutdownCtx context.Context, servi } if next != healthpb.HealthCheckResponse_SERVING { + sb.logger.Errf("transitioning health of %q from %q to %q, due to: %s", sb.cfg.Address, last, next, err) sb.logger.Errf("transitioning health of %q from %q to %q, due to: %s", service, last, next, err) } else { + sb.logger.Infof("transitioning health of %q from %q to %q", sb.cfg.Address, last, next) sb.logger.Infof("transitioning health of %q from %q to %q", service, last, next) + } + sb.healthSrv.SetServingStatus("", next) sb.healthSrv.SetServingStatus(service, next) return next } From 8c3ea9202a2f3fe4c49a638781c932998a4b89ef Mon Sep 17 00:00:00 2001 From: Samantha Date: Mon, 16 Jun 2025 18:14:57 -0400 Subject: [PATCH 2/8] Fix log lines and test --- grpc/server.go | 4 ++-- grpc/server_test.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/grpc/server.go b/grpc/server.go index e1daf24610a..8af60dc0868 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -262,10 +262,10 @@ func (sb *serverBuilder) initLongRunningCheck(shutdownCtx context.Context, servi } if next != healthpb.HealthCheckResponse_SERVING { - sb.logger.Errf("transitioning health of %q from %q to %q, due to: %s", sb.cfg.Address, last, next, err) + sb.logger.Errf("transitioning overall health from %q to %q, due to: %s", last, next, err) sb.logger.Errf("transitioning health of %q from %q to %q, due to: %s", service, last, next, err) } else { - sb.logger.Infof("transitioning health of %q from %q to %q", sb.cfg.Address, last, next) + sb.logger.Infof("transitioning overall health from %q to %q", last, next) sb.logger.Infof("transitioning health of %q from %q to %q", service, last, next) } diff --git a/grpc/server_test.go b/grpc/server_test.go index 7553e24c759..16c2e86a4ec 100644 --- a/grpc/server_test.go +++ b/grpc/server_test.go @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/health" ) -func Test_serverBuilder_initLongRunningCheck(t *testing.T) { +func TestServerBuilderInitLongRunningCheck(t *testing.T) { t.Parallel() hs := health.NewServer() mockLogger := blog.NewMock() @@ -41,8 +41,8 @@ func Test_serverBuilder_initLongRunningCheck(t *testing.T) { // - ~100ms 3rd check failed, SERVING to NOT_SERVING serving := mockLogger.GetAllMatching(".*\"NOT_SERVING\" to \"SERVING\"") notServing := mockLogger.GetAllMatching((".*\"SERVING\" to \"NOT_SERVING\"")) - test.Assert(t, len(serving) == 1, "expected one serving log line") - test.Assert(t, len(notServing) == 1, "expected one not serving log line") + test.Assert(t, len(serving) == 2, "expected two serving log lines") + test.Assert(t, len(notServing) == 2, "expected two not serving log lines") mockLogger.Clear() @@ -67,6 +67,6 @@ func Test_serverBuilder_initLongRunningCheck(t *testing.T) { // - ~100ms 3rd check passed, NOT_SERVING to SERVING serving = mockLogger.GetAllMatching(".*\"NOT_SERVING\" to \"SERVING\"") notServing = mockLogger.GetAllMatching((".*\"SERVING\" to \"NOT_SERVING\"")) - test.Assert(t, len(serving) == 2, "expected two serving log lines") - test.Assert(t, len(notServing) == 1, "expected one not serving log line") + test.Assert(t, len(serving) == 4, "expected four serving log lines") + test.Assert(t, len(notServing) == 2, "expected two not serving log lines") } From 7f42517994da2cb794143edf94ec7b077ca9b1a2 Mon Sep 17 00:00:00 2001 From: Samantha Date: Mon, 16 Jun 2025 18:15:48 -0400 Subject: [PATCH 3/8] Remove empty line --- grpc/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/grpc/server.go b/grpc/server.go index 8af60dc0868..c8ac08eb178 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -267,7 +267,6 @@ func (sb *serverBuilder) initLongRunningCheck(shutdownCtx context.Context, servi } else { sb.logger.Infof("transitioning overall health from %q to %q", last, next) sb.logger.Infof("transitioning health of %q from %q to %q", service, last, next) - } sb.healthSrv.SetServingStatus("", next) sb.healthSrv.SetServingStatus(service, next) From 4ea0d57b4aeefcf7b009024ded6d2c221574a723 Mon Sep 17 00:00:00 2001 From: Samantha Date: Tue, 17 Jun 2025 10:42:28 -0400 Subject: [PATCH 4/8] Add consul healthchecking for all gRPC endpoints --- test/config-next/akamai-purger.json | 3 +- test/config-next/ca.json | 3 +- test/config-next/crl-storer.json | 3 +- test/config-next/email-exporter.json | 3 +- test/config-next/nonce-a.json | 3 +- test/config-next/nonce-b.json | 3 +- test/config-next/publisher.json | 3 +- test/config-next/ra.json | 3 +- test/config-next/remoteva-a.json | 3 +- test/config-next/remoteva-b.json | 3 +- test/config-next/remoteva-c.json | 3 +- test/config-next/va.json | 3 +- test/config/akamai-purger.json | 3 +- test/config/ca.json | 3 +- test/config/crl-storer.json | 3 +- test/config/email-exporter.json | 3 +- test/config/nonce-a.json | 3 +- test/config/nonce-b.json | 3 +- test/config/publisher.json | 3 +- test/config/ra.json | 3 +- test/config/remoteva-a.json | 3 +- test/config/remoteva-b.json | 3 +- test/config/remoteva-c.json | 3 +- test/config/va.json | 3 +- test/consul/config.hcl | 265 ++++++++++++++++-- .../akamai-purger-queue-drain-config.json | 3 +- test/startservers.py | 6 +- 27 files changed, 295 insertions(+), 51 deletions(-) diff --git a/test/config-next/akamai-purger.json b/test/config-next/akamai-purger.json index 538ddac76b5..a40635b3eaa 100644 --- a/test/config-next/akamai-purger.json +++ b/test/config-next/akamai-purger.json @@ -26,7 +26,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/ca.json b/test/config-next/ca.json index e72b9df94f7..0d8c11ecd7c 100644 --- a/test/config-next/ca.json +++ b/test/config-next/ca.json @@ -26,7 +26,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/crl-storer.json b/test/config-next/crl-storer.json index 0934bcef071..14186f8b094 100644 --- a/test/config-next/crl-storer.json +++ b/test/config-next/crl-storer.json @@ -15,7 +15,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/email-exporter.json b/test/config-next/email-exporter.json index 5652e0c1c38..05a0e4e5be4 100644 --- a/test/config-next/email-exporter.json +++ b/test/config-next/email-exporter.json @@ -12,7 +12,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/nonce-a.json b/test/config-next/nonce-a.json index d14b44063f2..6aec8f8ad53 100644 --- a/test/config-next/nonce-a.json +++ b/test/config-next/nonce-a.json @@ -22,7 +22,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/nonce-b.json b/test/config-next/nonce-b.json index d14b44063f2..6aec8f8ad53 100644 --- a/test/config-next/nonce-b.json +++ b/test/config-next/nonce-b.json @@ -22,7 +22,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/publisher.json b/test/config-next/publisher.json index 3d0a0fb7e4e..7eb6aacaaee 100644 --- a/test/config-next/publisher.json +++ b/test/config-next/publisher.json @@ -30,7 +30,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/ra.json b/test/config-next/ra.json index 7229bae422f..332733429d9 100644 --- a/test/config-next/ra.json +++ b/test/config-next/ra.json @@ -151,7 +151,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/remoteva-a.json b/test/config-next/remoteva-a.json index d1c30b16df8..9070512f29a 100644 --- a/test/config-next/remoteva-a.json +++ b/test/config-next/remoteva-a.json @@ -29,7 +29,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/remoteva-b.json b/test/config-next/remoteva-b.json index 427dfa1f5cd..0eabbdb2247 100644 --- a/test/config-next/remoteva-b.json +++ b/test/config-next/remoteva-b.json @@ -29,7 +29,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/remoteva-c.json b/test/config-next/remoteva-c.json index 5fe5551e5f1..4ed1ce8f4b6 100644 --- a/test/config-next/remoteva-c.json +++ b/test/config-next/remoteva-c.json @@ -29,7 +29,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/va.json b/test/config-next/va.json index 8293ec3813f..a74d7062409 100644 --- a/test/config-next/va.json +++ b/test/config-next/va.json @@ -31,7 +31,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/akamai-purger.json b/test/config/akamai-purger.json index 3b2fe51b7a7..07b75a6e035 100644 --- a/test/config/akamai-purger.json +++ b/test/config/akamai-purger.json @@ -24,7 +24,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/ca.json b/test/config/ca.json index e9a866ee6aa..7fef3814443 100644 --- a/test/config/ca.json +++ b/test/config/ca.json @@ -27,7 +27,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/crl-storer.json b/test/config/crl-storer.json index 3ab267b0f64..20ff4e7cb8d 100644 --- a/test/config/crl-storer.json +++ b/test/config/crl-storer.json @@ -17,7 +17,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/email-exporter.json b/test/config/email-exporter.json index 8505cc4535e..a4b3939725f 100644 --- a/test/config/email-exporter.json +++ b/test/config/email-exporter.json @@ -12,7 +12,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/nonce-a.json b/test/config/nonce-a.json index e549c30ba1e..5da95ce1cbf 100644 --- a/test/config/nonce-a.json +++ b/test/config/nonce-a.json @@ -20,7 +20,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/nonce-b.json b/test/config/nonce-b.json index e549c30ba1e..5da95ce1cbf 100644 --- a/test/config/nonce-b.json +++ b/test/config/nonce-b.json @@ -20,7 +20,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/publisher.json b/test/config/publisher.json index 1909a6f601b..836ea81807a 100644 --- a/test/config/publisher.json +++ b/test/config/publisher.json @@ -30,7 +30,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/ra.json b/test/config/ra.json index 613c5e1a111..40a4ed550c5 100644 --- a/test/config/ra.json +++ b/test/config/ra.json @@ -150,7 +150,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/remoteva-a.json b/test/config/remoteva-a.json index 2ace42df439..4e4636ac7a6 100644 --- a/test/config/remoteva-a.json +++ b/test/config/remoteva-a.json @@ -33,7 +33,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/remoteva-b.json b/test/config/remoteva-b.json index 171b8534ad9..a7b73f1c56b 100644 --- a/test/config/remoteva-b.json +++ b/test/config/remoteva-b.json @@ -33,7 +33,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/remoteva-c.json b/test/config/remoteva-c.json index 22c168b662c..8a40c8ef566 100644 --- a/test/config/remoteva-c.json +++ b/test/config/remoteva-c.json @@ -33,7 +33,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/va.json b/test/config/va.json index 1172ad9de7b..2dbf08a105d 100644 --- a/test/config/va.json +++ b/test/config/va.json @@ -32,7 +32,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/consul/config.hcl b/test/consul/config.hcl index a296e154966..096e86735eb 100644 --- a/test/consul/config.hcl +++ b/test/consul/config.hcl @@ -31,6 +31,18 @@ services { address = "10.77.77.77" port = 9399 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "akamai-purger-a-grpc" + name = "akamai-purger-a-grpc" + grpc = "10.77.77.77:9399" + grpc_use_tls = true + tls_server_name = "akamai-purger.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -39,6 +51,18 @@ services { address = "10.77.77.77" port = 9603 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "email-exporter-a-grpc" + name = "email-exporter-a-grpc" + grpc = "10.77.77.77:9603" + grpc_use_tls = true + tls_server_name = "email-exporter.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -59,6 +83,18 @@ services { address = "10.77.77.77" port = 9393 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ca-a-grpc" + name = "ca-a-grpc" + grpc = "10.77.77.77:9393" + grpc_use_tls = true + tls_server_name = "ca.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -67,6 +103,18 @@ services { address = "10.77.77.77" port = 9493 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ca-b-grpc" + name = "ca-b-grpc" + grpc = "10.77.77.77:9493" + grpc_use_tls = true + tls_server_name = "ca.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -75,6 +123,18 @@ services { address = "10.77.77.77" port = 9309 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "crl-storer-a-grpc" + name = "crl-storer-a-grpc" + grpc = "10.77.77.77:9309" + grpc_use_tls = true + tls_server_name = "crl-storer.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -118,6 +178,18 @@ services { address = "10.77.77.77" port = 9301 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "nonce-taro-a-grpc" + name = "nonce-taro-a-grpc" + grpc = "10.77.77.77:9301" + grpc_use_tls = true + tls_server_name = "nonce.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -126,6 +198,18 @@ services { address = "10.77.77.77" port = 9501 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "nonce-taro-b-grpc" + name = "nonce-taro-b-grpc" + grpc = "10.77.77.77:9501" + grpc_use_tls = true + tls_server_name = "nonce.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -134,6 +218,18 @@ services { address = "10.77.77.77" port = 9401 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "nonce-zinc-grpc" + name = "nonce-zinc-grpc" + grpc = "10.77.77.77:9401" + grpc_use_tls = true + tls_server_name = "nonce.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -142,6 +238,18 @@ services { address = "10.77.77.77" port = 9391 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "publisher-a-grpc" + name = "publisher-a-grpc" + grpc = "10.77.77.77:9391" + grpc_use_tls = true + tls_server_name = "publisher.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -150,6 +258,18 @@ services { address = "10.77.77.77" port = 9491 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "publisher-b-grpc" + name = "publisher-b-grpc" + grpc = "10.77.77.77:9491" + grpc_use_tls = true + tls_server_name = "publisher.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -158,6 +278,18 @@ services { address = "10.77.77.77" port = 9594 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-sct-provider-a-grpc" + name = "ra-sct-provider-a-grpc" + grpc = "10.77.77.77:9594" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -166,6 +298,18 @@ services { address = "10.77.77.77" port = 9694 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-sct-provider-b-grpc" + name = "ra-sct-provider-b-grpc" + grpc = "10.77.77.77:9694" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -174,6 +318,18 @@ services { address = "10.77.77.77" port = 9394 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-a-grpc" + name = "ra-a-grpc" + grpc = "10.77.77.77:9394" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -182,6 +338,18 @@ services { address = "10.77.77.77" port = 9494 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-b-grpc" + name = "ra-b-grpc" + grpc = "10.77.77.77:9494" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -190,6 +358,18 @@ services { address = "10.77.77.77" port = 9397 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "rva1-a-grpc" + name = "rva1-a-grpc" + grpc = "10.77.77.77:9397" + grpc_use_tls = true + tls_server_name = "rva.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -198,6 +378,18 @@ services { address = "10.77.77.77" port = 9498 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "rva1-b-grpc" + name = "rva1-b-grpc" + grpc = "10.77.77.77:9498" + grpc_use_tls = true + tls_server_name = "rva.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -206,23 +398,18 @@ services { address = "10.77.77.77" port = 9499 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. -} - -# TODO(#5294) Remove rva2-a/b in favor of rva1-a/b -services { - id = "rva2-a" - name = "rva2" - address = "10.77.77.77" - port = 9897 - tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. -} - -services { - id = "rva2-b" - name = "rva2" - address = "10.77.77.77" - port = 9998 - tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "rva1-c-grpc" + name = "rva1-c-grpc" + grpc = "10.77.77.77:9499" + grpc_use_tls = true + tls_server_name = "rva.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -239,7 +426,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-a-grpc-sa" @@ -248,7 +436,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-a-grpc-saro" @@ -257,7 +446,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" } ] } @@ -276,7 +466,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-b-grpc-sa" @@ -285,7 +476,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-b-grpc-saro" @@ -294,7 +486,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" } ] } @@ -305,6 +498,18 @@ services { address = "10.77.77.77" port = 9392 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "va-a-grpc" + name = "va-a-grpc" + grpc = "10.77.77.77:9392" + grpc_use_tls = true + tls_server_name = "va.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -313,6 +518,18 @@ services { address = "10.77.77.77" port = 9492 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "va-b-grpc" + name = "va-b-grpc" + grpc = "10.77.77.77:9492" + grpc_use_tls = true + tls_server_name = "va.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -351,7 +568,7 @@ services { name = "case1a-failing" http = "http://localhost:12345" // invalid url method = "GET" - interval = "2s" + interval = "500ms" } ] } diff --git a/test/integration/testdata/akamai-purger-queue-drain-config.json b/test/integration/testdata/akamai-purger-queue-drain-config.json index 0a09d857e1b..1f84ad64f05 100644 --- a/test/integration/testdata/akamai-purger-queue-drain-config.json +++ b/test/integration/testdata/akamai-purger-queue-drain-config.json @@ -28,7 +28,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/startservers.py b/test/startservers.py index 4f4b508bab5..8ece1f6bd19 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -219,9 +219,11 @@ def start(fakeclock): signal.signal(signal.SIGINT, lambda _, __: stop()) # Check that we can resolve the service names before we try to start any - # services. This prevents a confusing error (timed out health check). + # services. This prevents a confusing error (timed out health check). We use + # the boulder.service.consul name because it has no health check, and thus + # will be served by the DNS server even if it's non-responsive. try: - socket.getaddrinfo('publisher.service.consul', None) + socket.getaddrinfo('boulder.service.consul', None) except Exception as e: print("Error querying DNS. Is consul running? `docker compose ps bconsul`. %s" % (e)) return False From 831440c34b0671e6bbd85718307eef6cde32c86c Mon Sep 17 00:00:00 2001 From: Samantha Date: Tue, 17 Jun 2025 16:01:14 -0400 Subject: [PATCH 5/8] Order clientNames in JSON --- test/config-next/akamai-purger.json | 4 ++-- test/config-next/ca.json | 4 ++-- test/config-next/crl-storer.json | 4 ++-- test/config-next/email-exporter.json | 4 ++-- test/config-next/nonce-a.json | 4 ++-- test/config-next/nonce-b.json | 4 ++-- test/config-next/publisher.json | 4 ++-- test/config-next/ra.json | 4 ++-- test/config-next/remoteva-a.json | 4 ++-- test/config-next/remoteva-b.json | 4 ++-- test/config-next/remoteva-c.json | 4 ++-- test/config-next/sa.json | 4 ++-- test/config-next/va.json | 4 ++-- test/config/akamai-purger.json | 4 ++-- test/config/ca.json | 4 ++-- test/config/crl-storer.json | 4 ++-- test/config/email-exporter.json | 4 ++-- test/config/nonce-a.json | 4 ++-- test/config/nonce-b.json | 4 ++-- test/config/publisher.json | 4 ++-- test/config/ra.json | 4 ++-- test/config/remoteva-a.json | 4 ++-- test/config/remoteva-b.json | 4 ++-- test/config/remoteva-c.json | 4 ++-- test/config/sa.json | 4 ++-- test/config/va.json | 4 ++-- .../testdata/akamai-purger-queue-drain-config.json | 4 ++-- 27 files changed, 54 insertions(+), 54 deletions(-) diff --git a/test/config-next/akamai-purger.json b/test/config-next/akamai-purger.json index a40635b3eaa..27088ad86a6 100644 --- a/test/config-next/akamai-purger.json +++ b/test/config-next/akamai-purger.json @@ -26,8 +26,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/ca.json b/test/config-next/ca.json index 0d8c11ecd7c..86462457777 100644 --- a/test/config-next/ca.json +++ b/test/config-next/ca.json @@ -26,8 +26,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/crl-storer.json b/test/config-next/crl-storer.json index 14186f8b094..736b5b2093c 100644 --- a/test/config-next/crl-storer.json +++ b/test/config-next/crl-storer.json @@ -15,8 +15,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/email-exporter.json b/test/config-next/email-exporter.json index 05a0e4e5be4..af4447fc1ff 100644 --- a/test/config-next/email-exporter.json +++ b/test/config-next/email-exporter.json @@ -12,8 +12,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/nonce-a.json b/test/config-next/nonce-a.json index 6aec8f8ad53..29db005dbab 100644 --- a/test/config-next/nonce-a.json +++ b/test/config-next/nonce-a.json @@ -22,8 +22,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/nonce-b.json b/test/config-next/nonce-b.json index 6aec8f8ad53..29db005dbab 100644 --- a/test/config-next/nonce-b.json +++ b/test/config-next/nonce-b.json @@ -22,8 +22,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/publisher.json b/test/config-next/publisher.json index 7eb6aacaaee..27d6a4942ad 100644 --- a/test/config-next/publisher.json +++ b/test/config-next/publisher.json @@ -30,8 +30,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/ra.json b/test/config-next/ra.json index 332733429d9..ad52c360a9c 100644 --- a/test/config-next/ra.json +++ b/test/config-next/ra.json @@ -151,8 +151,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/remoteva-a.json b/test/config-next/remoteva-a.json index 9070512f29a..0fbc7e5c3e8 100644 --- a/test/config-next/remoteva-a.json +++ b/test/config-next/remoteva-a.json @@ -29,8 +29,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/remoteva-b.json b/test/config-next/remoteva-b.json index 0eabbdb2247..087d9c1509d 100644 --- a/test/config-next/remoteva-b.json +++ b/test/config-next/remoteva-b.json @@ -29,8 +29,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/remoteva-c.json b/test/config-next/remoteva-c.json index 4ed1ce8f4b6..fa5f230a030 100644 --- a/test/config-next/remoteva-c.json +++ b/test/config-next/remoteva-c.json @@ -29,8 +29,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/sa.json b/test/config-next/sa.json index 1b9ff4687d8..b5c31039782 100644 --- a/test/config-next/sa.json +++ b/test/config-next/sa.json @@ -40,8 +40,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/va.json b/test/config-next/va.json index a74d7062409..d8c1ff72b12 100644 --- a/test/config-next/va.json +++ b/test/config-next/va.json @@ -31,8 +31,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/akamai-purger.json b/test/config/akamai-purger.json index 07b75a6e035..d906f77e564 100644 --- a/test/config/akamai-purger.json +++ b/test/config/akamai-purger.json @@ -24,8 +24,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/ca.json b/test/config/ca.json index 7fef3814443..2aafa52a11e 100644 --- a/test/config/ca.json +++ b/test/config/ca.json @@ -27,8 +27,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/crl-storer.json b/test/config/crl-storer.json index 20ff4e7cb8d..88ca5b65526 100644 --- a/test/config/crl-storer.json +++ b/test/config/crl-storer.json @@ -17,8 +17,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/email-exporter.json b/test/config/email-exporter.json index a4b3939725f..92fac3d402f 100644 --- a/test/config/email-exporter.json +++ b/test/config/email-exporter.json @@ -12,8 +12,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/nonce-a.json b/test/config/nonce-a.json index 5da95ce1cbf..b9c4bbb1790 100644 --- a/test/config/nonce-a.json +++ b/test/config/nonce-a.json @@ -20,8 +20,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/nonce-b.json b/test/config/nonce-b.json index 5da95ce1cbf..b9c4bbb1790 100644 --- a/test/config/nonce-b.json +++ b/test/config/nonce-b.json @@ -20,8 +20,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/publisher.json b/test/config/publisher.json index 836ea81807a..fdc2380cd02 100644 --- a/test/config/publisher.json +++ b/test/config/publisher.json @@ -30,8 +30,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/ra.json b/test/config/ra.json index 40a4ed550c5..16be04ccece 100644 --- a/test/config/ra.json +++ b/test/config/ra.json @@ -150,8 +150,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/remoteva-a.json b/test/config/remoteva-a.json index 4e4636ac7a6..f375b3f40e7 100644 --- a/test/config/remoteva-a.json +++ b/test/config/remoteva-a.json @@ -33,8 +33,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/remoteva-b.json b/test/config/remoteva-b.json index a7b73f1c56b..2d9200794d1 100644 --- a/test/config/remoteva-b.json +++ b/test/config/remoteva-b.json @@ -33,8 +33,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/remoteva-c.json b/test/config/remoteva-c.json index 8a40c8ef566..27a38e3a5c1 100644 --- a/test/config/remoteva-c.json +++ b/test/config/remoteva-c.json @@ -33,8 +33,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/sa.json b/test/config/sa.json index ec46b82dfe6..4a174ee714c 100644 --- a/test/config/sa.json +++ b/test/config/sa.json @@ -42,8 +42,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/va.json b/test/config/va.json index 2dbf08a105d..fdd784abc3e 100644 --- a/test/config/va.json +++ b/test/config/va.json @@ -32,8 +32,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/integration/testdata/akamai-purger-queue-drain-config.json b/test/integration/testdata/akamai-purger-queue-drain-config.json index 1f84ad64f05..e1e02f5a7db 100644 --- a/test/integration/testdata/akamai-purger-queue-drain-config.json +++ b/test/integration/testdata/akamai-purger-queue-drain-config.json @@ -28,8 +28,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } From 61bb5d5f45f7600d1e10429bb0d3494276e6997f Mon Sep 17 00:00:00 2001 From: Samantha Date: Tue, 17 Jun 2025 16:08:40 -0400 Subject: [PATCH 6/8] Construct a deduped slice and directly modify ClientNames --- cmd/config.go | 10 +++++----- grpc/server.go | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index b2c64622313..13842fdf9b2 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -465,7 +465,7 @@ type GRPCServerConfig struct { // These service names must match the service names advertised by gRPC itself, // which are identical to the names set in our gRPC .proto files prefixed by // the package names set in those files (e.g. "ca.CertificateAuthority"). - Services map[string]GRPCServiceConfig `json:"services" validate:"required,dive,required"` + Services map[string]*GRPCServiceConfig `json:"services" validate:"required,dive,required"` // MaxConnectionAge specifies how long a connection may live before the server sends a GoAway to the // client. Because gRPC connections re-resolve DNS after a connection close, // this controls how long it takes before a client learns about changes to its @@ -476,10 +476,10 @@ type GRPCServerConfig struct { // GRPCServiceConfig contains the information needed to configure a gRPC service. type GRPCServiceConfig struct { - // PerServiceClientNames is a map of gRPC service names to client certificate - // SANs. The upstream listening server will reject connections from clients - // which do not appear in this list, and the server interceptor will reject - // RPC calls for this service from clients which are not listed here. + // ClientNames is the list of accepted gRPC client certificate SANs. + // Connections from clients not in this list will be rejected by the + // upstream listener, and RPCs from unlisted clients will be denied by the + // server interceptor. ClientNames []string `json:"clientNames" validate:"min=1,dive,hostname,required"` } diff --git a/grpc/server.go b/grpc/server.go index c8ac08eb178..2fb09f7f09c 100644 --- a/grpc/server.go +++ b/grpc/server.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net" + "slices" "strings" "time" @@ -123,20 +124,20 @@ func (sb *serverBuilder) Build(tlsConfig *tls.Config, statsRegistry prometheus.R // This is the names which are allowlisted at the server level, plus the union // of all names which are allowlisted for any individual service. acceptedSANs := make(map[string]struct{}) + var acceptedSANsSlice []string for _, service := range sb.cfg.Services { for _, name := range service.ClientNames { acceptedSANs[name] = struct{}{} + if !slices.Contains(acceptedSANsSlice, name) { + acceptedSANsSlice = append(acceptedSANsSlice, name) + } } } // Ensure that the health service has the same ClientNames as the other // services, so that health checks can be performed by clients which are // allowed to connect to the server. - healthService := sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName] - for as := range acceptedSANs { - healthService.ClientNames = append(healthService.ClientNames, as) - } - sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName] = healthService + sb.cfg.Services[healthpb.Health_ServiceDesc.ServiceName].ClientNames = acceptedSANsSlice creds, err := bcreds.NewServerCredentials(tlsConfig, acceptedSANs) if err != nil { From 1d0efc3a5b22ea97039b5a099d871ed92d6a92c4 Mon Sep 17 00:00:00 2001 From: Samantha Date: Tue, 17 Jun 2025 16:37:12 -0400 Subject: [PATCH 7/8] Reverting Consul changes for now, it's too flaky in CI --- test/config-next/akamai-purger.json | 1 - test/config-next/ca.json | 1 - test/config-next/crl-storer.json | 1 - test/config-next/email-exporter.json | 1 - test/config-next/nonce-a.json | 1 - test/config-next/nonce-b.json | 1 - test/config-next/publisher.json | 1 - test/config-next/ra.json | 1 - test/config-next/remoteva-a.json | 1 - test/config-next/remoteva-b.json | 1 - test/config-next/remoteva-c.json | 1 - test/config-next/sa.json | 4 +- test/config-next/va.json | 1 - test/config/akamai-purger.json | 1 - test/config/ca.json | 1 - test/config/crl-storer.json | 1 - test/config/email-exporter.json | 1 - test/config/nonce-a.json | 1 - test/config/nonce-b.json | 1 - test/config/publisher.json | 1 - test/config/ra.json | 1 - test/config/remoteva-a.json | 1 - test/config/remoteva-b.json | 1 - test/config/remoteva-c.json | 1 - test/config/sa.json | 4 +- test/config/va.json | 1 - test/consul/config.hcl | 265 ++---------------- .../akamai-purger-queue-drain-config.json | 1 - test/startservers.py | 6 +- 29 files changed, 30 insertions(+), 274 deletions(-) diff --git a/test/config-next/akamai-purger.json b/test/config-next/akamai-purger.json index 27088ad86a6..538ddac76b5 100644 --- a/test/config-next/akamai-purger.json +++ b/test/config-next/akamai-purger.json @@ -26,7 +26,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/ca.json b/test/config-next/ca.json index 86462457777..e72b9df94f7 100644 --- a/test/config-next/ca.json +++ b/test/config-next/ca.json @@ -26,7 +26,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/crl-storer.json b/test/config-next/crl-storer.json index 736b5b2093c..0934bcef071 100644 --- a/test/config-next/crl-storer.json +++ b/test/config-next/crl-storer.json @@ -15,7 +15,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/email-exporter.json b/test/config-next/email-exporter.json index af4447fc1ff..5652e0c1c38 100644 --- a/test/config-next/email-exporter.json +++ b/test/config-next/email-exporter.json @@ -12,7 +12,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/nonce-a.json b/test/config-next/nonce-a.json index 29db005dbab..d14b44063f2 100644 --- a/test/config-next/nonce-a.json +++ b/test/config-next/nonce-a.json @@ -22,7 +22,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/nonce-b.json b/test/config-next/nonce-b.json index 29db005dbab..d14b44063f2 100644 --- a/test/config-next/nonce-b.json +++ b/test/config-next/nonce-b.json @@ -22,7 +22,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/publisher.json b/test/config-next/publisher.json index 27d6a4942ad..3d0a0fb7e4e 100644 --- a/test/config-next/publisher.json +++ b/test/config-next/publisher.json @@ -30,7 +30,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/ra.json b/test/config-next/ra.json index ad52c360a9c..7229bae422f 100644 --- a/test/config-next/ra.json +++ b/test/config-next/ra.json @@ -151,7 +151,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/remoteva-a.json b/test/config-next/remoteva-a.json index 0fbc7e5c3e8..d1c30b16df8 100644 --- a/test/config-next/remoteva-a.json +++ b/test/config-next/remoteva-a.json @@ -29,7 +29,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/remoteva-b.json b/test/config-next/remoteva-b.json index 087d9c1509d..427dfa1f5cd 100644 --- a/test/config-next/remoteva-b.json +++ b/test/config-next/remoteva-b.json @@ -29,7 +29,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/remoteva-c.json b/test/config-next/remoteva-c.json index fa5f230a030..5fe5551e5f1 100644 --- a/test/config-next/remoteva-c.json +++ b/test/config-next/remoteva-c.json @@ -29,7 +29,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/sa.json b/test/config-next/sa.json index b5c31039782..1b9ff4687d8 100644 --- a/test/config-next/sa.json +++ b/test/config-next/sa.json @@ -40,8 +40,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config-next/va.json b/test/config-next/va.json index d8c1ff72b12..8293ec3813f 100644 --- a/test/config-next/va.json +++ b/test/config-next/va.json @@ -31,7 +31,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/akamai-purger.json b/test/config/akamai-purger.json index d906f77e564..3b2fe51b7a7 100644 --- a/test/config/akamai-purger.json +++ b/test/config/akamai-purger.json @@ -24,7 +24,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/ca.json b/test/config/ca.json index 2aafa52a11e..e9a866ee6aa 100644 --- a/test/config/ca.json +++ b/test/config/ca.json @@ -27,7 +27,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/crl-storer.json b/test/config/crl-storer.json index 88ca5b65526..3ab267b0f64 100644 --- a/test/config/crl-storer.json +++ b/test/config/crl-storer.json @@ -17,7 +17,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/email-exporter.json b/test/config/email-exporter.json index 92fac3d402f..8505cc4535e 100644 --- a/test/config/email-exporter.json +++ b/test/config/email-exporter.json @@ -12,7 +12,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/nonce-a.json b/test/config/nonce-a.json index b9c4bbb1790..e549c30ba1e 100644 --- a/test/config/nonce-a.json +++ b/test/config/nonce-a.json @@ -20,7 +20,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/nonce-b.json b/test/config/nonce-b.json index b9c4bbb1790..e549c30ba1e 100644 --- a/test/config/nonce-b.json +++ b/test/config/nonce-b.json @@ -20,7 +20,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/publisher.json b/test/config/publisher.json index fdc2380cd02..1909a6f601b 100644 --- a/test/config/publisher.json +++ b/test/config/publisher.json @@ -30,7 +30,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/ra.json b/test/config/ra.json index 16be04ccece..613c5e1a111 100644 --- a/test/config/ra.json +++ b/test/config/ra.json @@ -150,7 +150,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/remoteva-a.json b/test/config/remoteva-a.json index f375b3f40e7..2ace42df439 100644 --- a/test/config/remoteva-a.json +++ b/test/config/remoteva-a.json @@ -33,7 +33,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/remoteva-b.json b/test/config/remoteva-b.json index 2d9200794d1..171b8534ad9 100644 --- a/test/config/remoteva-b.json +++ b/test/config/remoteva-b.json @@ -33,7 +33,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/remoteva-c.json b/test/config/remoteva-c.json index 27a38e3a5c1..22c168b662c 100644 --- a/test/config/remoteva-c.json +++ b/test/config/remoteva-c.json @@ -33,7 +33,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/sa.json b/test/config/sa.json index 4a174ee714c..ec46b82dfe6 100644 --- a/test/config/sa.json +++ b/test/config/sa.json @@ -42,8 +42,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", - "health-checker.boulder" + "health-checker.boulder", + "consul.boulder" ] } } diff --git a/test/config/va.json b/test/config/va.json index fdd784abc3e..1172ad9de7b 100644 --- a/test/config/va.json +++ b/test/config/va.json @@ -32,7 +32,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/consul/config.hcl b/test/consul/config.hcl index 096e86735eb..a296e154966 100644 --- a/test/consul/config.hcl +++ b/test/consul/config.hcl @@ -31,18 +31,6 @@ services { address = "10.77.77.77" port = 9399 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "akamai-purger-a-grpc" - name = "akamai-purger-a-grpc" - grpc = "10.77.77.77:9399" - grpc_use_tls = true - tls_server_name = "akamai-purger.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -51,18 +39,6 @@ services { address = "10.77.77.77" port = 9603 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "email-exporter-a-grpc" - name = "email-exporter-a-grpc" - grpc = "10.77.77.77:9603" - grpc_use_tls = true - tls_server_name = "email-exporter.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -83,18 +59,6 @@ services { address = "10.77.77.77" port = 9393 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "ca-a-grpc" - name = "ca-a-grpc" - grpc = "10.77.77.77:9393" - grpc_use_tls = true - tls_server_name = "ca.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -103,18 +67,6 @@ services { address = "10.77.77.77" port = 9493 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "ca-b-grpc" - name = "ca-b-grpc" - grpc = "10.77.77.77:9493" - grpc_use_tls = true - tls_server_name = "ca.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -123,18 +75,6 @@ services { address = "10.77.77.77" port = 9309 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "crl-storer-a-grpc" - name = "crl-storer-a-grpc" - grpc = "10.77.77.77:9309" - grpc_use_tls = true - tls_server_name = "crl-storer.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -178,18 +118,6 @@ services { address = "10.77.77.77" port = 9301 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "nonce-taro-a-grpc" - name = "nonce-taro-a-grpc" - grpc = "10.77.77.77:9301" - grpc_use_tls = true - tls_server_name = "nonce.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -198,18 +126,6 @@ services { address = "10.77.77.77" port = 9501 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "nonce-taro-b-grpc" - name = "nonce-taro-b-grpc" - grpc = "10.77.77.77:9501" - grpc_use_tls = true - tls_server_name = "nonce.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -218,18 +134,6 @@ services { address = "10.77.77.77" port = 9401 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "nonce-zinc-grpc" - name = "nonce-zinc-grpc" - grpc = "10.77.77.77:9401" - grpc_use_tls = true - tls_server_name = "nonce.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -238,18 +142,6 @@ services { address = "10.77.77.77" port = 9391 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "publisher-a-grpc" - name = "publisher-a-grpc" - grpc = "10.77.77.77:9391" - grpc_use_tls = true - tls_server_name = "publisher.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -258,18 +150,6 @@ services { address = "10.77.77.77" port = 9491 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "publisher-b-grpc" - name = "publisher-b-grpc" - grpc = "10.77.77.77:9491" - grpc_use_tls = true - tls_server_name = "publisher.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -278,18 +158,6 @@ services { address = "10.77.77.77" port = 9594 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "ra-sct-provider-a-grpc" - name = "ra-sct-provider-a-grpc" - grpc = "10.77.77.77:9594" - grpc_use_tls = true - tls_server_name = "ra.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -298,18 +166,6 @@ services { address = "10.77.77.77" port = 9694 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "ra-sct-provider-b-grpc" - name = "ra-sct-provider-b-grpc" - grpc = "10.77.77.77:9694" - grpc_use_tls = true - tls_server_name = "ra.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -318,18 +174,6 @@ services { address = "10.77.77.77" port = 9394 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "ra-a-grpc" - name = "ra-a-grpc" - grpc = "10.77.77.77:9394" - grpc_use_tls = true - tls_server_name = "ra.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -338,18 +182,6 @@ services { address = "10.77.77.77" port = 9494 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "ra-b-grpc" - name = "ra-b-grpc" - grpc = "10.77.77.77:9494" - grpc_use_tls = true - tls_server_name = "ra.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -358,18 +190,6 @@ services { address = "10.77.77.77" port = 9397 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "rva1-a-grpc" - name = "rva1-a-grpc" - grpc = "10.77.77.77:9397" - grpc_use_tls = true - tls_server_name = "rva.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -378,18 +198,6 @@ services { address = "10.77.77.77" port = 9498 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "rva1-b-grpc" - name = "rva1-b-grpc" - grpc = "10.77.77.77:9498" - grpc_use_tls = true - tls_server_name = "rva.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -398,18 +206,23 @@ services { address = "10.77.77.77" port = 9499 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "rva1-c-grpc" - name = "rva1-c-grpc" - grpc = "10.77.77.77:9499" - grpc_use_tls = true - tls_server_name = "rva.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] +} + +# TODO(#5294) Remove rva2-a/b in favor of rva1-a/b +services { + id = "rva2-a" + name = "rva2" + address = "10.77.77.77" + port = 9897 + tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. +} + +services { + id = "rva2-b" + name = "rva2" + address = "10.77.77.77" + port = 9998 + tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. } services { @@ -426,8 +239,7 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "1s" - timeout = "500ms" + interval = "2s" }, { id = "sa-a-grpc-sa" @@ -436,8 +248,7 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "1s" - timeout = "500ms" + interval = "2s" }, { id = "sa-a-grpc-saro" @@ -446,8 +257,7 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "1s" - timeout = "500ms" + interval = "2s" } ] } @@ -466,8 +276,7 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "1s" - timeout = "500ms" + interval = "2s" }, { id = "sa-b-grpc-sa" @@ -476,8 +285,7 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "1s" - timeout = "500ms" + interval = "2s" }, { id = "sa-b-grpc-saro" @@ -486,8 +294,7 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "1s" - timeout = "500ms" + interval = "2s" } ] } @@ -498,18 +305,6 @@ services { address = "10.77.77.77" port = 9392 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "va-a-grpc" - name = "va-a-grpc" - grpc = "10.77.77.77:9392" - grpc_use_tls = true - tls_server_name = "va.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -518,18 +313,6 @@ services { address = "10.77.77.77" port = 9492 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. - checks = [ - { - id = "va-b-grpc" - name = "va-b-grpc" - grpc = "10.77.77.77:9492" - grpc_use_tls = true - tls_server_name = "va.boulder" - tls_skip_verify = false - interval = "1s" - timeout = "500ms" - } - ] } services { @@ -568,7 +351,7 @@ services { name = "case1a-failing" http = "http://localhost:12345" // invalid url method = "GET" - interval = "500ms" + interval = "2s" } ] } diff --git a/test/integration/testdata/akamai-purger-queue-drain-config.json b/test/integration/testdata/akamai-purger-queue-drain-config.json index e1e02f5a7db..0a09d857e1b 100644 --- a/test/integration/testdata/akamai-purger-queue-drain-config.json +++ b/test/integration/testdata/akamai-purger-queue-drain-config.json @@ -28,7 +28,6 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "consul.boulder", "health-checker.boulder" ] } diff --git a/test/startservers.py b/test/startservers.py index 8ece1f6bd19..4f4b508bab5 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -219,11 +219,9 @@ def start(fakeclock): signal.signal(signal.SIGINT, lambda _, __: stop()) # Check that we can resolve the service names before we try to start any - # services. This prevents a confusing error (timed out health check). We use - # the boulder.service.consul name because it has no health check, and thus - # will be served by the DNS server even if it's non-responsive. + # services. This prevents a confusing error (timed out health check). try: - socket.getaddrinfo('boulder.service.consul', None) + socket.getaddrinfo('publisher.service.consul', None) except Exception as e: print("Error querying DNS. Is consul running? `docker compose ps bconsul`. %s" % (e)) return False From b593665daf62427ccf2be5eecd21eec476a2bb64 Mon Sep 17 00:00:00 2001 From: Samantha Date: Tue, 17 Jun 2025 16:41:15 -0400 Subject: [PATCH 8/8] consul: Enable health checking for all gRPC endpoints --- test/config-next/akamai-purger.json | 1 + test/config-next/ca.json | 1 + test/config-next/crl-storer.json | 1 + test/config-next/email-exporter.json | 1 + test/config-next/nonce-a.json | 1 + test/config-next/nonce-b.json | 1 + test/config-next/publisher.json | 1 + test/config-next/ra.json | 1 + test/config-next/remoteva-a.json | 1 + test/config-next/remoteva-b.json | 1 + test/config-next/remoteva-c.json | 1 + test/config-next/sa.json | 4 +- test/config-next/va.json | 1 + test/config/akamai-purger.json | 1 + test/config/ca.json | 1 + test/config/crl-storer.json | 1 + test/config/email-exporter.json | 1 + test/config/nonce-a.json | 1 + test/config/nonce-b.json | 1 + test/config/publisher.json | 1 + test/config/ra.json | 1 + test/config/remoteva-a.json | 1 + test/config/remoteva-b.json | 1 + test/config/remoteva-c.json | 1 + test/config/sa.json | 4 +- test/config/va.json | 1 + test/consul/config.hcl | 265 ++++++++++++++++-- .../akamai-purger-queue-drain-config.json | 1 + test/startservers.py | 6 +- 29 files changed, 274 insertions(+), 30 deletions(-) diff --git a/test/config-next/akamai-purger.json b/test/config-next/akamai-purger.json index 538ddac76b5..27088ad86a6 100644 --- a/test/config-next/akamai-purger.json +++ b/test/config-next/akamai-purger.json @@ -26,6 +26,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/ca.json b/test/config-next/ca.json index e72b9df94f7..86462457777 100644 --- a/test/config-next/ca.json +++ b/test/config-next/ca.json @@ -26,6 +26,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/crl-storer.json b/test/config-next/crl-storer.json index 0934bcef071..736b5b2093c 100644 --- a/test/config-next/crl-storer.json +++ b/test/config-next/crl-storer.json @@ -15,6 +15,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/email-exporter.json b/test/config-next/email-exporter.json index 5652e0c1c38..af4447fc1ff 100644 --- a/test/config-next/email-exporter.json +++ b/test/config-next/email-exporter.json @@ -12,6 +12,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/nonce-a.json b/test/config-next/nonce-a.json index d14b44063f2..29db005dbab 100644 --- a/test/config-next/nonce-a.json +++ b/test/config-next/nonce-a.json @@ -22,6 +22,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/nonce-b.json b/test/config-next/nonce-b.json index d14b44063f2..29db005dbab 100644 --- a/test/config-next/nonce-b.json +++ b/test/config-next/nonce-b.json @@ -22,6 +22,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/publisher.json b/test/config-next/publisher.json index 3d0a0fb7e4e..27d6a4942ad 100644 --- a/test/config-next/publisher.json +++ b/test/config-next/publisher.json @@ -30,6 +30,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/ra.json b/test/config-next/ra.json index 7229bae422f..ad52c360a9c 100644 --- a/test/config-next/ra.json +++ b/test/config-next/ra.json @@ -151,6 +151,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/remoteva-a.json b/test/config-next/remoteva-a.json index d1c30b16df8..0fbc7e5c3e8 100644 --- a/test/config-next/remoteva-a.json +++ b/test/config-next/remoteva-a.json @@ -29,6 +29,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/remoteva-b.json b/test/config-next/remoteva-b.json index 427dfa1f5cd..087d9c1509d 100644 --- a/test/config-next/remoteva-b.json +++ b/test/config-next/remoteva-b.json @@ -29,6 +29,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/remoteva-c.json b/test/config-next/remoteva-c.json index 5fe5551e5f1..fa5f230a030 100644 --- a/test/config-next/remoteva-c.json +++ b/test/config-next/remoteva-c.json @@ -29,6 +29,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config-next/sa.json b/test/config-next/sa.json index 1b9ff4687d8..b5c31039782 100644 --- a/test/config-next/sa.json +++ b/test/config-next/sa.json @@ -40,8 +40,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config-next/va.json b/test/config-next/va.json index 8293ec3813f..d8c1ff72b12 100644 --- a/test/config-next/va.json +++ b/test/config-next/va.json @@ -31,6 +31,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/akamai-purger.json b/test/config/akamai-purger.json index 3b2fe51b7a7..d906f77e564 100644 --- a/test/config/akamai-purger.json +++ b/test/config/akamai-purger.json @@ -24,6 +24,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/ca.json b/test/config/ca.json index e9a866ee6aa..2aafa52a11e 100644 --- a/test/config/ca.json +++ b/test/config/ca.json @@ -27,6 +27,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/crl-storer.json b/test/config/crl-storer.json index 3ab267b0f64..88ca5b65526 100644 --- a/test/config/crl-storer.json +++ b/test/config/crl-storer.json @@ -17,6 +17,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/email-exporter.json b/test/config/email-exporter.json index 8505cc4535e..92fac3d402f 100644 --- a/test/config/email-exporter.json +++ b/test/config/email-exporter.json @@ -12,6 +12,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/nonce-a.json b/test/config/nonce-a.json index e549c30ba1e..b9c4bbb1790 100644 --- a/test/config/nonce-a.json +++ b/test/config/nonce-a.json @@ -20,6 +20,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/nonce-b.json b/test/config/nonce-b.json index e549c30ba1e..b9c4bbb1790 100644 --- a/test/config/nonce-b.json +++ b/test/config/nonce-b.json @@ -20,6 +20,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/publisher.json b/test/config/publisher.json index 1909a6f601b..fdc2380cd02 100644 --- a/test/config/publisher.json +++ b/test/config/publisher.json @@ -30,6 +30,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/ra.json b/test/config/ra.json index 613c5e1a111..16be04ccece 100644 --- a/test/config/ra.json +++ b/test/config/ra.json @@ -150,6 +150,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/remoteva-a.json b/test/config/remoteva-a.json index 2ace42df439..f375b3f40e7 100644 --- a/test/config/remoteva-a.json +++ b/test/config/remoteva-a.json @@ -33,6 +33,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/remoteva-b.json b/test/config/remoteva-b.json index 171b8534ad9..2d9200794d1 100644 --- a/test/config/remoteva-b.json +++ b/test/config/remoteva-b.json @@ -33,6 +33,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/remoteva-c.json b/test/config/remoteva-c.json index 22c168b662c..27a38e3a5c1 100644 --- a/test/config/remoteva-c.json +++ b/test/config/remoteva-c.json @@ -33,6 +33,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/config/sa.json b/test/config/sa.json index ec46b82dfe6..4a174ee714c 100644 --- a/test/config/sa.json +++ b/test/config/sa.json @@ -42,8 +42,8 @@ }, "grpc.health.v1.Health": { "clientNames": [ - "health-checker.boulder", - "consul.boulder" + "consul.boulder", + "health-checker.boulder" ] } } diff --git a/test/config/va.json b/test/config/va.json index 1172ad9de7b..fdd784abc3e 100644 --- a/test/config/va.json +++ b/test/config/va.json @@ -32,6 +32,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/consul/config.hcl b/test/consul/config.hcl index a296e154966..096e86735eb 100644 --- a/test/consul/config.hcl +++ b/test/consul/config.hcl @@ -31,6 +31,18 @@ services { address = "10.77.77.77" port = 9399 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "akamai-purger-a-grpc" + name = "akamai-purger-a-grpc" + grpc = "10.77.77.77:9399" + grpc_use_tls = true + tls_server_name = "akamai-purger.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -39,6 +51,18 @@ services { address = "10.77.77.77" port = 9603 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "email-exporter-a-grpc" + name = "email-exporter-a-grpc" + grpc = "10.77.77.77:9603" + grpc_use_tls = true + tls_server_name = "email-exporter.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -59,6 +83,18 @@ services { address = "10.77.77.77" port = 9393 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ca-a-grpc" + name = "ca-a-grpc" + grpc = "10.77.77.77:9393" + grpc_use_tls = true + tls_server_name = "ca.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -67,6 +103,18 @@ services { address = "10.77.77.77" port = 9493 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ca-b-grpc" + name = "ca-b-grpc" + grpc = "10.77.77.77:9493" + grpc_use_tls = true + tls_server_name = "ca.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -75,6 +123,18 @@ services { address = "10.77.77.77" port = 9309 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "crl-storer-a-grpc" + name = "crl-storer-a-grpc" + grpc = "10.77.77.77:9309" + grpc_use_tls = true + tls_server_name = "crl-storer.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -118,6 +178,18 @@ services { address = "10.77.77.77" port = 9301 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "nonce-taro-a-grpc" + name = "nonce-taro-a-grpc" + grpc = "10.77.77.77:9301" + grpc_use_tls = true + tls_server_name = "nonce.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -126,6 +198,18 @@ services { address = "10.77.77.77" port = 9501 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "nonce-taro-b-grpc" + name = "nonce-taro-b-grpc" + grpc = "10.77.77.77:9501" + grpc_use_tls = true + tls_server_name = "nonce.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -134,6 +218,18 @@ services { address = "10.77.77.77" port = 9401 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "nonce-zinc-grpc" + name = "nonce-zinc-grpc" + grpc = "10.77.77.77:9401" + grpc_use_tls = true + tls_server_name = "nonce.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -142,6 +238,18 @@ services { address = "10.77.77.77" port = 9391 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "publisher-a-grpc" + name = "publisher-a-grpc" + grpc = "10.77.77.77:9391" + grpc_use_tls = true + tls_server_name = "publisher.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -150,6 +258,18 @@ services { address = "10.77.77.77" port = 9491 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "publisher-b-grpc" + name = "publisher-b-grpc" + grpc = "10.77.77.77:9491" + grpc_use_tls = true + tls_server_name = "publisher.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -158,6 +278,18 @@ services { address = "10.77.77.77" port = 9594 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-sct-provider-a-grpc" + name = "ra-sct-provider-a-grpc" + grpc = "10.77.77.77:9594" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -166,6 +298,18 @@ services { address = "10.77.77.77" port = 9694 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-sct-provider-b-grpc" + name = "ra-sct-provider-b-grpc" + grpc = "10.77.77.77:9694" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -174,6 +318,18 @@ services { address = "10.77.77.77" port = 9394 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-a-grpc" + name = "ra-a-grpc" + grpc = "10.77.77.77:9394" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -182,6 +338,18 @@ services { address = "10.77.77.77" port = 9494 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "ra-b-grpc" + name = "ra-b-grpc" + grpc = "10.77.77.77:9494" + grpc_use_tls = true + tls_server_name = "ra.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -190,6 +358,18 @@ services { address = "10.77.77.77" port = 9397 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "rva1-a-grpc" + name = "rva1-a-grpc" + grpc = "10.77.77.77:9397" + grpc_use_tls = true + tls_server_name = "rva.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -198,6 +378,18 @@ services { address = "10.77.77.77" port = 9498 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "rva1-b-grpc" + name = "rva1-b-grpc" + grpc = "10.77.77.77:9498" + grpc_use_tls = true + tls_server_name = "rva.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -206,23 +398,18 @@ services { address = "10.77.77.77" port = 9499 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. -} - -# TODO(#5294) Remove rva2-a/b in favor of rva1-a/b -services { - id = "rva2-a" - name = "rva2" - address = "10.77.77.77" - port = 9897 - tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. -} - -services { - id = "rva2-b" - name = "rva2" - address = "10.77.77.77" - port = 9998 - tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "rva1-c-grpc" + name = "rva1-c-grpc" + grpc = "10.77.77.77:9499" + grpc_use_tls = true + tls_server_name = "rva.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -239,7 +426,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-a-grpc-sa" @@ -248,7 +436,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-a-grpc-saro" @@ -257,7 +446,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" } ] } @@ -276,7 +466,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-b-grpc-sa" @@ -285,7 +476,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" }, { id = "sa-b-grpc-saro" @@ -294,7 +486,8 @@ services { grpc_use_tls = true tls_server_name = "sa.boulder" tls_skip_verify = false - interval = "2s" + interval = "1s" + timeout = "500ms" } ] } @@ -305,6 +498,18 @@ services { address = "10.77.77.77" port = 9392 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "va-a-grpc" + name = "va-a-grpc" + grpc = "10.77.77.77:9392" + grpc_use_tls = true + tls_server_name = "va.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -313,6 +518,18 @@ services { address = "10.77.77.77" port = 9492 tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution. + checks = [ + { + id = "va-b-grpc" + name = "va-b-grpc" + grpc = "10.77.77.77:9492" + grpc_use_tls = true + tls_server_name = "va.boulder" + tls_skip_verify = false + interval = "1s" + timeout = "500ms" + } + ] } services { @@ -351,7 +568,7 @@ services { name = "case1a-failing" http = "http://localhost:12345" // invalid url method = "GET" - interval = "2s" + interval = "500ms" } ] } diff --git a/test/integration/testdata/akamai-purger-queue-drain-config.json b/test/integration/testdata/akamai-purger-queue-drain-config.json index 0a09d857e1b..e1e02f5a7db 100644 --- a/test/integration/testdata/akamai-purger-queue-drain-config.json +++ b/test/integration/testdata/akamai-purger-queue-drain-config.json @@ -28,6 +28,7 @@ }, "grpc.health.v1.Health": { "clientNames": [ + "consul.boulder", "health-checker.boulder" ] } diff --git a/test/startservers.py b/test/startservers.py index 4f4b508bab5..8ece1f6bd19 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -219,9 +219,11 @@ def start(fakeclock): signal.signal(signal.SIGINT, lambda _, __: stop()) # Check that we can resolve the service names before we try to start any - # services. This prevents a confusing error (timed out health check). + # services. This prevents a confusing error (timed out health check). We use + # the boulder.service.consul name because it has no health check, and thus + # will be served by the DNS server even if it's non-responsive. try: - socket.getaddrinfo('publisher.service.consul', None) + socket.getaddrinfo('boulder.service.consul', None) except Exception as e: print("Error querying DNS. Is consul running? `docker compose ps bconsul`. %s" % (e)) return False