From c676e32ebc56a6b66540034462bd123d6fe9f0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCttler?= Date: Wed, 1 Apr 2026 15:56:50 +0200 Subject: [PATCH 1/4] Hot Reload of Credentials (Still Work in Progress) Fixes #1139 --- .gitignore | 1 + hcloud/cloud.go | 12 +-- hcloud/runtime_credentials.go | 87 ++++++++++++++++++ hcloud/runtime_credentials_test.go | 117 +++++++++++++++++++++++++ internal/config/runtime_credentials.go | 27 ++++++ 5 files changed, 234 insertions(+), 10 deletions(-) create mode 100644 hcloud/runtime_credentials.go create mode 100644 hcloud/runtime_credentials_test.go create mode 100644 internal/config/runtime_credentials.go diff --git a/.gitignore b/.gitignore index 0fd693953..cdd2977c3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ hcloud-cloud-controller-manager *.tgz hack/.* coverage/ +.vscode diff --git a/hcloud/cloud.go b/hcloud/cloud.go index 8840fd567..f3f5f60d3 100644 --- a/hcloud/cloud.go +++ b/hcloud/cloud.go @@ -19,7 +19,6 @@ package hcloud import ( "context" "fmt" - "net/http" "os" "strings" "time" @@ -71,13 +70,8 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) { } opts := []hcloud.ClientOption{ - hcloud.WithToken(cfg.HCloudClient.Token), hcloud.WithApplication("hcloud-cloud-controller", providerVersion), - hcloud.WithHTTPClient( - &http.Client{ - Timeout: apiClientTimeout, - }, - ), + hcloud.WithHTTPClient(newHCloudHTTPClient(apiClientTimeout)), } // start metrics server if enabled (enabled by default) @@ -100,9 +94,7 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) { c := hrobot.NewBasicAuthClientWithCustomHttpClient( cfg.Robot.User, cfg.Robot.Password, - &http.Client{ - Timeout: apiClientTimeout, - }, + newRobotHTTPClient(apiClientTimeout), ) robotClient = robot.NewRateLimitedClient( diff --git a/hcloud/runtime_credentials.go b/hcloud/runtime_credentials.go new file mode 100644 index 000000000..c80f7dc0d --- /dev/null +++ b/hcloud/runtime_credentials.go @@ -0,0 +1,87 @@ +package hcloud + +import ( + "fmt" + "net/http" + "time" + + "golang.org/x/net/http/httpguts" + + "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/config" +) + +const invalidAuthorizationTokenError = "authorization token contains invalid characters" + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func newHCloudHTTPClient(timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: newHCloudCredentialReloader(nil), + } +} + +func newRobotHTTPClient(timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: newRobotCredentialReloader(nil), + } +} + +func newHCloudCredentialReloader(next http.RoundTripper) http.RoundTripper { + next = transportOrDefault(next) + + return roundTripperFunc(func(req *http.Request) (*http.Response, error) { + token, err := config.LookupHCloudToken() + if err != nil { + return nil, err + } + if token != "" && !httpguts.ValidHeaderFieldValue(token) { + return nil, fmt.Errorf(invalidAuthorizationTokenError) + } + + cloned := cloneRequest(req) + if token == "" { + cloned.Header.Del("Authorization") + } else { + cloned.Header.Set("Authorization", "Bearer "+token) + } + return next.RoundTrip(cloned) + }) +} + +func newRobotCredentialReloader(next http.RoundTripper) http.RoundTripper { + next = transportOrDefault(next) + + return roundTripperFunc(func(req *http.Request) (*http.Response, error) { + user, password, err := config.LookupRobotCredentials() + if err != nil { + return nil, err + } + + cloned := cloneRequest(req) + if user == "" && password == "" { + cloned.Header.Del("Authorization") + } else { + cloned.SetBasicAuth(user, password) + } + return next.RoundTrip(cloned) + }) +} + +func cloneRequest(req *http.Request) *http.Request { + cloned := req.Clone(req.Context()) + cloned.Header = req.Header.Clone() + return cloned +} + +func transportOrDefault(next http.RoundTripper) http.RoundTripper { + if next != nil { + return next + } + return http.DefaultTransport +} diff --git a/hcloud/runtime_credentials_test.go b/hcloud/runtime_credentials_test.go new file mode 100644 index 000000000..d0773250c --- /dev/null +++ b/hcloud/runtime_credentials_test.go @@ -0,0 +1,117 @@ +package hcloud + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + hrobot "github.com/syself/hrobot-go" + + "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/testsupport" + "github.com/hetznercloud/hcloud-go/v2/hcloud" + "github.com/hetznercloud/hcloud-go/v2/hcloud/schema" +) + +func TestHCloudClientReloadsTokenFromFile(t *testing.T) { + defer unsetEnv(t, "HCLOUD_TOKEN")() + + var authorizations []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authorizations = append(authorizations, r.Header.Get("Authorization")) + assert.NoError(t, json.NewEncoder(w).Encode(schema.LocationListResponse{Locations: []schema.Location{}})) + })) + defer server.Close() + + tokenFile := filepath.Join(t.TempDir(), "hcloud-token") + assert.NoError(t, os.WriteFile(tokenFile, []byte("token-1"), 0o600)) + + resetEnv := testsupport.Setenv(t, "HCLOUD_TOKEN_FILE", tokenFile) + defer resetEnv() + + client := hcloud.NewClient( + hcloud.WithEndpoint(server.URL), + hcloud.WithHTTPClient(newHCloudHTTPClient(0)), + hcloud.WithPollOpts(hcloud.PollOpts{BackoffFunc: hcloud.ConstantBackoff(0)}), + hcloud.WithRetryOpts(hcloud.RetryOpts{BackoffFunc: hcloud.ConstantBackoff(0)}), + ) + + _, _, err := client.Location.List(t.Context(), hcloud.LocationListOpts{}) + assert.NoError(t, err) + + assert.NoError(t, os.WriteFile(tokenFile, []byte("token-2"), 0o600)) + + _, _, err = client.Location.List(t.Context(), hcloud.LocationListOpts{}) + assert.NoError(t, err) + + assert.Equal(t, []string{"Bearer token-1", "Bearer token-2"}, authorizations) +} + +func TestRobotClientReloadsCredentialsFromFile(t *testing.T) { + defer unsetEnv(t, "ROBOT_USER")() + defer unsetEnv(t, "ROBOT_PASSWORD")() + + var users []string + var passwords []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, password, ok := r.BasicAuth() + assert.True(t, ok) + users = append(users, user) + passwords = append(passwords, password) + assert.NoError(t, json.NewEncoder(w).Encode([]map[string]any{ + { + "server": map[string]any{ + "server_number": 1, + "server_name": "node-1", + "server_ip": "192.0.2.1", + }, + }, + })) + })) + defer server.Close() + + dir := t.TempDir() + userFile := filepath.Join(dir, "robot-user") + passwordFile := filepath.Join(dir, "robot-password") + assert.NoError(t, os.WriteFile(userFile, []byte("robot-user-1"), 0o600)) + assert.NoError(t, os.WriteFile(passwordFile, []byte("robot-password-1"), 0o600)) + + resetEnv := testsupport.Setenv(t, + "ROBOT_USER_FILE", userFile, + "ROBOT_PASSWORD_FILE", passwordFile, + ) + defer resetEnv() + + client := hrobot.NewBasicAuthClientWithCustomHttpClient("stale-user", "stale-password", newRobotHTTPClient(0)) + client.SetBaseURL(server.URL) + + _, err := client.ServerGetList() + assert.NoError(t, err) + + assert.NoError(t, os.WriteFile(userFile, []byte("robot-user-2"), 0o600)) + assert.NoError(t, os.WriteFile(passwordFile, []byte("robot-password-2"), 0o600)) + + _, err = client.ServerGetList() + assert.NoError(t, err) + + assert.Equal(t, []string{"robot-user-1", "robot-user-2"}, users) + assert.Equal(t, []string{"robot-password-1", "robot-password-2"}, passwords) +} + +func unsetEnv(t *testing.T, key string) func() { + t.Helper() + + value, ok := os.LookupEnv(key) + assert.NoError(t, os.Unsetenv(key)) + + return func() { + if !ok { + assert.NoError(t, os.Unsetenv(key)) + return + } + assert.NoError(t, os.Setenv(key, value)) + } +} diff --git a/internal/config/runtime_credentials.go b/internal/config/runtime_credentials.go new file mode 100644 index 000000000..2bfd630fd --- /dev/null +++ b/internal/config/runtime_credentials.go @@ -0,0 +1,27 @@ +package config + +import ( + "errors" + "fmt" + + "github.com/hetznercloud/hcloud-go/v2/hcloud/exp/kit/envutil" +) + +// LookupHCloudToken reads the current HCLOUD_TOKEN / HCLOUD_TOKEN_FILE value. +func LookupHCloudToken() (string, error) { + return envutil.LookupEnvWithFile(hcloudToken) +} + +// LookupRobotCredentials reads the current ROBOT_USER / ROBOT_USER_FILE and +// ROBOT_PASSWORD / ROBOT_PASSWORD_FILE values. +func LookupRobotCredentials() (string, string, error) { + user, userErr := envutil.LookupEnvWithFile(robotUser) + password, passwordErr := envutil.LookupEnvWithFile(robotPassword) + if userErr != nil || passwordErr != nil { + return "", "", errors.Join(userErr, passwordErr) + } + if (user == "") != (password == "") { + return "", "", fmt.Errorf("both %q and %q must be provided, or neither", robotUser, robotPassword) + } + return user, password, nil +} From b093d69d6e4d1fbd019d5695cf1eb90b8cd7ea56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCttler?= Date: Wed, 1 Apr 2026 16:32:57 +0200 Subject: [PATCH 2/4] hot reload via inotify. --- hcloud/cloud.go | 16 +- hcloud/runtime_credentials.go | 236 ++++++++++++++++++++++--- hcloud/runtime_credentials_test.go | 96 +++++++--- internal/config/runtime_credentials.go | 61 +++++++ 4 files changed, 364 insertions(+), 45 deletions(-) diff --git a/hcloud/cloud.go b/hcloud/cloud.go index f3f5f60d3..3dcf84718 100644 --- a/hcloud/cloud.go +++ b/hcloud/cloud.go @@ -51,6 +51,7 @@ type cloud struct { client *hcloud.Client robotClient robot.Client cfg config.HCCMConfiguration + credentials *runtimeCredentials recorder record.EventRecorder networkID int64 cidr string @@ -69,9 +70,14 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) { return nil, err } + credentials, err := newRuntimeCredentials() + if err != nil { + return nil, err + } + opts := []hcloud.ClientOption{ hcloud.WithApplication("hcloud-cloud-controller", providerVersion), - hcloud.WithHTTPClient(newHCloudHTTPClient(apiClientTimeout)), + hcloud.WithHTTPClient(newHCloudHTTPClient(apiClientTimeout, credentials)), } // start metrics server if enabled (enabled by default) @@ -94,7 +100,7 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) { c := hrobot.NewBasicAuthClientWithCustomHttpClient( cfg.Robot.User, cfg.Robot.Password, - newRobotHTTPClient(apiClientTimeout), + newRobotHTTPClient(apiClientTimeout, credentials), ) robotClient = robot.NewRateLimitedClient( @@ -137,6 +143,7 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) { client: client, robotClient: robotClient, cfg: cfg, + credentials: credentials, networkID: networkID, cidr: cidr, }, nil @@ -150,6 +157,11 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, go func() { <-stop + if c.credentials != nil { + if err := c.credentials.close(); err != nil { + klog.ErrorS(err, "close runtime credential watcher") + } + } eventBroadcaster.Shutdown() }() diff --git a/hcloud/runtime_credentials.go b/hcloud/runtime_credentials.go index c80f7dc0d..c70d44bc4 100644 --- a/hcloud/runtime_credentials.go +++ b/hcloud/runtime_credentials.go @@ -3,14 +3,18 @@ package hcloud import ( "fmt" "net/http" + "sync" "time" + "github.com/fsnotify/fsnotify" "golang.org/x/net/http/httpguts" + "k8s.io/klog/v2" "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/config" ) const invalidAuthorizationTokenError = "authorization token contains invalid characters" +const credentialsReloadDebounce = 100 * time.Millisecond type roundTripperFunc func(*http.Request) (*http.Response, error) @@ -18,52 +22,242 @@ func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } -func newHCloudHTTPClient(timeout time.Duration) *http.Client { +type runtimeCredentials struct { + mu sync.RWMutex + + hcloudToken string + robotUser string + robotPass string + + hcloudTokenPath string + robotUserPath string + robotPassPath string + + watcher *fsnotify.Watcher + closeOnce sync.Once +} + +func newRuntimeCredentials() (*runtimeCredentials, error) { + credentials := &runtimeCredentials{} + + if err := credentials.loadInitial(); err != nil { + return nil, err + } + + files := config.LookupRuntimeCredentialFiles() + credentials.hcloudTokenPath = files.HCloudToken + credentials.robotUserPath = files.RobotUser + credentials.robotPassPath = files.RobotPassword + + if !files.HasAny() { + return credentials, nil + } + + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + for _, dir := range files.Directories() { + if err := watcher.Add(dir); err != nil { + watcher.Close() + return nil, fmt.Errorf("watch credentials directory %q: %w", dir, err) + } + } + + credentials.watcher = watcher + go credentials.watch() + return credentials, nil +} + +func (c *runtimeCredentials) loadInitial() error { + token, err := config.LookupHCloudToken() + if err != nil { + return err + } + if token != "" && !httpguts.ValidHeaderFieldValue(token) { + return fmt.Errorf(invalidAuthorizationTokenError) + } + + user, password, err := config.LookupRobotCredentials() + if err != nil { + return err + } + + c.mu.Lock() + c.hcloudToken = token + c.robotUser = user + c.robotPass = password + c.mu.Unlock() + + return nil +} + +func (c *runtimeCredentials) watch() { + var debounceTimer *time.Timer + var debounceC <-chan time.Time + + stopDebounce := func() { + if debounceTimer == nil { + return + } + if !debounceTimer.Stop() { + select { + case <-debounceTimer.C: + default: + } + } + } + + defer stopDebounce() + + for { + select { + case event, ok := <-c.watcher.Events: + if !ok { + return + } + if !shouldReload(event) { + continue + } + if debounceTimer == nil { + debounceTimer = time.NewTimer(credentialsReloadDebounce) + debounceC = debounceTimer.C + continue + } + stopDebounce() + debounceTimer.Reset(credentialsReloadDebounce) + case err, ok := <-c.watcher.Errors: + if !ok { + return + } + klog.ErrorS(err, "watching mounted credential files") + case <-debounceC: + debounceC = nil + c.reload() + } + } +} + +func shouldReload(event fsnotify.Event) bool { + return event.Op&(fsnotify.Create|fsnotify.Write|fsnotify.Rename|fsnotify.Remove|fsnotify.Chmod) != 0 +} + +func (c *runtimeCredentials) reload() { + if c.hcloudTokenPath != "" { + token, err := config.ReadCredentialFile(c.hcloudTokenPath) + switch { + case err != nil: + klog.ErrorS(err, "reloading HCLOUD_TOKEN from mounted secret") + case token != "" && !httpguts.ValidHeaderFieldValue(token): + klog.ErrorS(fmt.Errorf(invalidAuthorizationTokenError), "reloading HCLOUD_TOKEN from mounted secret") + default: + c.mu.Lock() + c.hcloudToken = token + c.mu.Unlock() + } + } + + if c.robotUserPath != "" || c.robotPassPath != "" { + user, password, err := c.loadRobotCredentials() + if err != nil { + klog.ErrorS(err, "reloading Robot credentials from mounted secret") + return + } + c.mu.Lock() + c.robotUser = user + c.robotPass = password + c.mu.Unlock() + } +} + +func (c *runtimeCredentials) loadRobotCredentials() (string, string, error) { + c.mu.RLock() + user := c.robotUser + password := c.robotPass + c.mu.RUnlock() + + var err error + if c.robotUserPath != "" { + user, err = config.ReadCredentialFile(c.robotUserPath) + if err != nil { + return "", "", err + } + } + if c.robotPassPath != "" { + password, err = config.ReadCredentialFile(c.robotPassPath) + if err != nil { + return "", "", err + } + } + if (user == "") != (password == "") { + return "", "", fmt.Errorf("both %q and %q must be provided, or neither", "ROBOT_USER", "ROBOT_PASSWORD") + } + return user, password, nil +} + +func (c *runtimeCredentials) close() error { + var err error + c.closeOnce.Do(func() { + if c.watcher != nil { + err = c.watcher.Close() + } + }) + return err +} + +func (c *runtimeCredentials) hcloudAuthorization() string { + c.mu.RLock() + defer c.mu.RUnlock() + + if c.hcloudToken == "" { + return "" + } + return "Bearer " + c.hcloudToken +} + +func (c *runtimeCredentials) robotCredentials() (string, string) { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.robotUser, c.robotPass +} + +func newHCloudHTTPClient(timeout time.Duration, credentials *runtimeCredentials) *http.Client { return &http.Client{ Timeout: timeout, - Transport: newHCloudCredentialReloader(nil), + Transport: newHCloudCredentialReloader(credentials, nil), } } -func newRobotHTTPClient(timeout time.Duration) *http.Client { +func newRobotHTTPClient(timeout time.Duration, credentials *runtimeCredentials) *http.Client { return &http.Client{ Timeout: timeout, - Transport: newRobotCredentialReloader(nil), + Transport: newRobotCredentialReloader(credentials, nil), } } -func newHCloudCredentialReloader(next http.RoundTripper) http.RoundTripper { +func newHCloudCredentialReloader(credentials *runtimeCredentials, next http.RoundTripper) http.RoundTripper { next = transportOrDefault(next) return roundTripperFunc(func(req *http.Request) (*http.Response, error) { - token, err := config.LookupHCloudToken() - if err != nil { - return nil, err - } - if token != "" && !httpguts.ValidHeaderFieldValue(token) { - return nil, fmt.Errorf(invalidAuthorizationTokenError) - } - cloned := cloneRequest(req) - if token == "" { + auth := credentials.hcloudAuthorization() + if auth == "" { cloned.Header.Del("Authorization") } else { - cloned.Header.Set("Authorization", "Bearer "+token) + cloned.Header.Set("Authorization", auth) } return next.RoundTrip(cloned) }) } -func newRobotCredentialReloader(next http.RoundTripper) http.RoundTripper { +func newRobotCredentialReloader(credentials *runtimeCredentials, next http.RoundTripper) http.RoundTripper { next = transportOrDefault(next) return roundTripperFunc(func(req *http.Request) (*http.Response, error) { - user, password, err := config.LookupRobotCredentials() - if err != nil { - return nil, err - } - cloned := cloneRequest(req) + user, password := credentials.robotCredentials() if user == "" && password == "" { cloned.Header.Del("Authorization") } else { diff --git a/hcloud/runtime_credentials_test.go b/hcloud/runtime_credentials_test.go index d0773250c..c42845b91 100644 --- a/hcloud/runtime_credentials_test.go +++ b/hcloud/runtime_credentials_test.go @@ -6,7 +6,9 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync" "testing" + "time" "github.com/stretchr/testify/assert" hrobot "github.com/syself/hrobot-go" @@ -16,12 +18,16 @@ import ( "github.com/hetznercloud/hcloud-go/v2/hcloud/schema" ) -func TestHCloudClientReloadsTokenFromFile(t *testing.T) { +func TestHCloudClientReloadsTokenFromMountedSecret(t *testing.T) { defer unsetEnv(t, "HCLOUD_TOKEN")() - var authorizations []string + var mu sync.Mutex + lastAuthorization := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - authorizations = append(authorizations, r.Header.Get("Authorization")) + mu.Lock() + lastAuthorization = r.Header.Get("Authorization") + mu.Unlock() assert.NoError(t, json.NewEncoder(w).Encode(schema.LocationListResponse{Locations: []schema.Location{}})) })) defer server.Close() @@ -32,35 +38,57 @@ func TestHCloudClientReloadsTokenFromFile(t *testing.T) { resetEnv := testsupport.Setenv(t, "HCLOUD_TOKEN_FILE", tokenFile) defer resetEnv() + credentials, err := newRuntimeCredentials() + assert.NoError(t, err) + defer func() { + assert.NoError(t, credentials.close()) + }() + client := hcloud.NewClient( hcloud.WithEndpoint(server.URL), - hcloud.WithHTTPClient(newHCloudHTTPClient(0)), + hcloud.WithHTTPClient(newHCloudHTTPClient(0, credentials)), hcloud.WithPollOpts(hcloud.PollOpts{BackoffFunc: hcloud.ConstantBackoff(0)}), hcloud.WithRetryOpts(hcloud.RetryOpts{BackoffFunc: hcloud.ConstantBackoff(0)}), ) - _, _, err := client.Location.List(t.Context(), hcloud.LocationListOpts{}) + _, _, err = client.Location.List(t.Context(), hcloud.LocationListOpts{}) assert.NoError(t, err) - assert.NoError(t, os.WriteFile(tokenFile, []byte("token-2"), 0o600)) + mu.Lock() + assert.Equal(t, "Bearer token-1", lastAuthorization) + mu.Unlock() - _, _, err = client.Location.List(t.Context(), hcloud.LocationListOpts{}) - assert.NoError(t, err) + replaceFile(t, tokenFile, "token-2") - assert.Equal(t, []string{"Bearer token-1", "Bearer token-2"}, authorizations) + assert.Eventually(t, func() bool { + _, _, err = client.Location.List(t.Context(), hcloud.LocationListOpts{}) + if err != nil { + return false + } + + mu.Lock() + defer mu.Unlock() + return lastAuthorization == "Bearer token-2" + }, 3*time.Second, 50*time.Millisecond) } -func TestRobotClientReloadsCredentialsFromFile(t *testing.T) { +func TestRobotClientReloadsCredentialsFromMountedSecret(t *testing.T) { defer unsetEnv(t, "ROBOT_USER")() defer unsetEnv(t, "ROBOT_PASSWORD")() - var users []string - var passwords []string + var mu sync.Mutex + lastUser := "" + lastPassword := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, password, ok := r.BasicAuth() assert.True(t, ok) - users = append(users, user) - passwords = append(passwords, password) + + mu.Lock() + lastUser = user + lastPassword = password + mu.Unlock() + assert.NoError(t, json.NewEncoder(w).Encode([]map[string]any{ { "server": map[string]any{ @@ -85,20 +113,44 @@ func TestRobotClientReloadsCredentialsFromFile(t *testing.T) { ) defer resetEnv() - client := hrobot.NewBasicAuthClientWithCustomHttpClient("stale-user", "stale-password", newRobotHTTPClient(0)) - client.SetBaseURL(server.URL) - - _, err := client.ServerGetList() + credentials, err := newRuntimeCredentials() assert.NoError(t, err) + defer func() { + assert.NoError(t, credentials.close()) + }() - assert.NoError(t, os.WriteFile(userFile, []byte("robot-user-2"), 0o600)) - assert.NoError(t, os.WriteFile(passwordFile, []byte("robot-password-2"), 0o600)) + client := hrobot.NewBasicAuthClientWithCustomHttpClient("stale-user", "stale-password", newRobotHTTPClient(0, credentials)) + client.SetBaseURL(server.URL) _, err = client.ServerGetList() assert.NoError(t, err) - assert.Equal(t, []string{"robot-user-1", "robot-user-2"}, users) - assert.Equal(t, []string{"robot-password-1", "robot-password-2"}, passwords) + mu.Lock() + assert.Equal(t, "robot-user-1", lastUser) + assert.Equal(t, "robot-password-1", lastPassword) + mu.Unlock() + + replaceFile(t, userFile, "robot-user-2") + replaceFile(t, passwordFile, "robot-password-2") + + assert.Eventually(t, func() bool { + _, err = client.ServerGetList() + if err != nil { + return false + } + + mu.Lock() + defer mu.Unlock() + return lastUser == "robot-user-2" && lastPassword == "robot-password-2" + }, 3*time.Second, 50*time.Millisecond) +} + +func replaceFile(t *testing.T, path, content string) { + t.Helper() + + tmpPath := path + ".tmp" + assert.NoError(t, os.WriteFile(tmpPath, []byte(content), 0o600)) + assert.NoError(t, os.Rename(tmpPath, path)) } func unsetEnv(t *testing.T, key string) func() { diff --git a/internal/config/runtime_credentials.go b/internal/config/runtime_credentials.go index 2bfd630fd..6528c2405 100644 --- a/internal/config/runtime_credentials.go +++ b/internal/config/runtime_credentials.go @@ -3,10 +3,27 @@ package config import ( "errors" "fmt" + "os" + "path/filepath" + "strings" "github.com/hetznercloud/hcloud-go/v2/hcloud/exp/kit/envutil" ) +type RuntimeCredentialFiles struct { + HCloudToken string + RobotUser string + RobotPassword string +} + +func (f RuntimeCredentialFiles) HasAny() bool { + return f.HCloudToken != "" || f.RobotUser != "" || f.RobotPassword != "" +} + +func (f RuntimeCredentialFiles) Directories() []string { + return uniqueDirectories(f.HCloudToken, f.RobotUser, f.RobotPassword) +} + // LookupHCloudToken reads the current HCLOUD_TOKEN / HCLOUD_TOKEN_FILE value. func LookupHCloudToken() (string, error) { return envutil.LookupEnvWithFile(hcloudToken) @@ -25,3 +42,47 @@ func LookupRobotCredentials() (string, string, error) { } return user, password, nil } + +// LookupRuntimeCredentialFiles returns the file-backed credential sources that +// can be watched for hot reloads. Plain environment variables take precedence +// over file-backed sources and are therefore not watched. +func LookupRuntimeCredentialFiles() RuntimeCredentialFiles { + return RuntimeCredentialFiles{ + HCloudToken: lookupCredentialFile(hcloudToken), + RobotUser: lookupCredentialFile(robotUser), + RobotPassword: lookupCredentialFile(robotPassword), + } +} + +// ReadCredentialFile reads a mounted credential file and trims surrounding +// whitespace to match envutil.LookupEnvWithFile semantics. +func ReadCredentialFile(path string) (string, error) { + valueBytes, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(valueBytes)), nil +} + +func lookupCredentialFile(key string) string { + if _, ok := os.LookupEnv(key); ok { + return "" + } + return os.Getenv(key + "_FILE") +} + +func uniqueDirectories(paths ...string) []string { + dirs := map[string]struct{}{} + for _, path := range paths { + if path == "" { + continue + } + dirs[filepath.Dir(path)] = struct{}{} + } + + result := make([]string, 0, len(dirs)) + for dir := range dirs { + result = append(result, dir) + } + return result +} From d23501578fc5e207bbe77ca52059d25c9fbc7b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCttler?= Date: Wed, 1 Apr 2026 17:09:16 +0200 Subject: [PATCH 3/4] docs: clarify hot reload requires file-backed secrets --- chart/values.yaml | 4 ++++ docs/reference/helm/extra-envs.md | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/chart/values.yaml b/chart/values.yaml index 0893d29dd..b75c68c71 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -45,6 +45,10 @@ env: # This is currently possible for HCLOUD_TOKEN, ROBOT_USER, and ROBOT_PASSWORD. # Use the env var appended with _FILE (e.g. HCLOUD_TOKEN_FILE) and set the value to the file path that should be read # The file must be provided externally (e.g. via secret injection). + # Hot reloading only works with file-backed secrets. It does not work when + # credentials are provided via regular environment variables or + # valueFrom.secretKeyRef, because Kubernetes does not update the process + # environment of a running container. # Example: # HCLOUD_TOKEN_FILE: # value: "/etc/hetzner/token" diff --git a/docs/reference/helm/extra-envs.md b/docs/reference/helm/extra-envs.md index fe11c9300..ece0c0c69 100644 --- a/docs/reference/helm/extra-envs.md +++ b/docs/reference/helm/extra-envs.md @@ -2,6 +2,8 @@ You can define extra environment variables for the HCCM. Both Kubernetes formats are supported: `value` and `valueFrom`. The `valueFrom` field can reference multiple sources such as ConfigMaps and Secrets, but also supports other options. For more details, see the Kubernetes documentation on [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/#using-configmaps-as-environment-variables) and [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables). +If you want credential hot reloading, do not provide `HCLOUD_TOKEN`, `ROBOT_USER`, or `ROBOT_PASSWORD` via regular environment variables or `valueFrom.secretKeyRef`. Hot reloading only works when these credentials are read from files via `HCLOUD_TOKEN_FILE`, `ROBOT_USER_FILE`, and `ROBOT_PASSWORD_FILE`, backed by a mounted Secret volume. Kubernetes updates mounted Secret files, but it does not update the environment of a running container. + ```yaml env: ROBOT_USER: @@ -17,3 +19,18 @@ env: key: robot-user optional: true ``` + +Example for file-backed credentials with hot reloading: + +```yaml +env: + HCLOUD_TOKEN: null + ROBOT_USER: null + ROBOT_PASSWORD: null + HCLOUD_TOKEN_FILE: + value: /etc/hetzner/token + ROBOT_USER_FILE: + value: /etc/hetzner/robot-user + ROBOT_PASSWORD_FILE: + value: /etc/hetzner/robot-password +``` From d0cca3bfbbdeed73d2d6b273611f926cbfa775df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCttler?= Date: Wed, 1 Apr 2026 17:19:06 +0200 Subject: [PATCH 4/4] test: cover hot reload credential paths --- hcloud/runtime_credentials_test.go | 159 ++++++++++++++++++++ internal/config/runtime_credentials_test.go | 138 +++++++++++++++++ 2 files changed, 297 insertions(+) create mode 100644 internal/config/runtime_credentials_test.go diff --git a/hcloud/runtime_credentials_test.go b/hcloud/runtime_credentials_test.go index c42845b91..91c6b2252 100644 --- a/hcloud/runtime_credentials_test.go +++ b/hcloud/runtime_credentials_test.go @@ -2,10 +2,12 @@ package hcloud import ( "encoding/json" + "io" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -145,6 +147,163 @@ func TestRobotClientReloadsCredentialsFromMountedSecret(t *testing.T) { }, 3*time.Second, 50*time.Millisecond) } +func TestNewRuntimeCredentialsWithoutFileSources(t *testing.T) { + defer unsetEnv(t, "HCLOUD_TOKEN_FILE")() + defer unsetEnv(t, "ROBOT_USER_FILE")() + defer unsetEnv(t, "ROBOT_PASSWORD_FILE")() + + resetEnv := testsupport.Setenv(t, + "HCLOUD_TOKEN", "token-1", + "ROBOT_USER", "robot-user-1", + "ROBOT_PASSWORD", "robot-password-1", + ) + defer resetEnv() + + credentials, err := newRuntimeCredentials() + assert.NoError(t, err) + assert.Nil(t, credentials.watcher) + assert.Equal(t, "Bearer token-1", credentials.hcloudAuthorization()) + assert.Equal(t, "robot-user-1", credentials.robotUser) + assert.Equal(t, "robot-password-1", credentials.robotPass) + assert.NoError(t, credentials.close()) +} + +func TestNewRuntimeCredentialsRejectsInvalidAuthorizationToken(t *testing.T) { + defer unsetEnv(t, "HCLOUD_TOKEN_FILE")() + defer unsetEnv(t, "ROBOT_USER")() + defer unsetEnv(t, "ROBOT_PASSWORD")() + defer unsetEnv(t, "ROBOT_USER_FILE")() + defer unsetEnv(t, "ROBOT_PASSWORD_FILE")() + + resetEnv := testsupport.Setenv(t, "HCLOUD_TOKEN", "token\ninvalid") + defer resetEnv() + + _, err := newRuntimeCredentials() + assert.EqualError(t, err, invalidAuthorizationTokenError) +} + +func TestNewRuntimeCredentialsRejectsMissingMountedSecret(t *testing.T) { + defer unsetEnv(t, "HCLOUD_TOKEN")() + defer unsetEnv(t, "ROBOT_USER")() + defer unsetEnv(t, "ROBOT_PASSWORD")() + defer unsetEnv(t, "ROBOT_USER_FILE")() + defer unsetEnv(t, "ROBOT_PASSWORD_FILE")() + + resetEnv := testsupport.Setenv(t, "HCLOUD_TOKEN_FILE", filepath.Join(t.TempDir(), "missing")) + defer resetEnv() + + _, err := newRuntimeCredentials() + assert.ErrorContains(t, err, "no such file or directory") +} + +func TestRuntimeCredentialsLoadRobotCredentialsErrors(t *testing.T) { + t.Run("missing file", func(t *testing.T) { + dir := t.TempDir() + userFile := filepath.Join(dir, "robot-user") + assert.NoError(t, os.WriteFile(userFile, []byte("robot-user-1"), 0o600)) + + credentials := &runtimeCredentials{ + robotUserPath: userFile, + robotPassPath: filepath.Join(dir, "missing"), + } + + _, _, err := credentials.loadRobotCredentials() + assert.ErrorContains(t, err, "no such file or directory") + }) + + t.Run("partial credentials", func(t *testing.T) { + dir := t.TempDir() + userFile := filepath.Join(dir, "robot-user") + passwordFile := filepath.Join(dir, "robot-password") + assert.NoError(t, os.WriteFile(userFile, []byte("robot-user-1"), 0o600)) + assert.NoError(t, os.WriteFile(passwordFile, []byte(""), 0o600)) + + credentials := &runtimeCredentials{ + robotUserPath: userFile, + robotPassPath: passwordFile, + } + + _, _, err := credentials.loadRobotCredentials() + assert.EqualError(t, err, `both "ROBOT_USER" and "ROBOT_PASSWORD" must be provided, or neither`) + }) +} + +func TestRuntimeCredentialsReloadKeepsPreviousValuesOnErrors(t *testing.T) { + dir := t.TempDir() + tokenFile := filepath.Join(dir, "hcloud-token") + userFile := filepath.Join(dir, "robot-user") + passwordFile := filepath.Join(dir, "robot-password") + + assert.NoError(t, os.WriteFile(tokenFile, []byte("token\ninvalid"), 0o600)) + assert.NoError(t, os.WriteFile(userFile, []byte("robot-user-2"), 0o600)) + assert.NoError(t, os.WriteFile(passwordFile, []byte(""), 0o600)) + + credentials := &runtimeCredentials{ + hcloudToken: "token-1", + robotUser: "robot-user-1", + robotPass: "robot-password-1", + hcloudTokenPath: tokenFile, + robotUserPath: userFile, + robotPassPath: passwordFile, + } + + credentials.reload() + + assert.Equal(t, "Bearer token-1", credentials.hcloudAuthorization()) + user, password := credentials.robotCredentials() + assert.Equal(t, "robot-user-1", user) + assert.Equal(t, "robot-password-1", password) +} + +func TestCredentialReloadersClearAuthorizationHeadersWhenCredentialsAreEmpty(t *testing.T) { + captured := make(chan string, 2) + next := roundTripperFunc(func(req *http.Request) (*http.Response, error) { + captured <- req.Header.Get("Authorization") + return &http.Response{ + StatusCode: http.StatusNoContent, + Body: io.NopCloser(strings.NewReader("")), + Header: make(http.Header), + Request: req, + }, nil + }) + + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + assert.NoError(t, err) + req.Header.Set("Authorization", "stale") + + _, err = newHCloudCredentialReloader(&runtimeCredentials{}, next).RoundTrip(req) + assert.NoError(t, err) + assert.Equal(t, "", <-captured) + + req, err = http.NewRequest(http.MethodGet, "https://example.com", nil) + assert.NoError(t, err) + req.Header.Set("Authorization", "stale") + + _, err = newRobotCredentialReloader(&runtimeCredentials{}, next).RoundTrip(req) + assert.NoError(t, err) + assert.Equal(t, "", <-captured) +} + +func TestTransportOrDefault(t *testing.T) { + custom := roundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusCreated, + Body: io.NopCloser(strings.NewReader("created")), + Header: make(http.Header), + Request: req, + }, nil + }) + + assert.Same(t, http.DefaultTransport, transportOrDefault(nil)) + + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + assert.NoError(t, err) + + resp, err := transportOrDefault(custom).RoundTrip(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) +} + func replaceFile(t *testing.T, path, content string) { t.Helper() diff --git a/internal/config/runtime_credentials_test.go b/internal/config/runtime_credentials_test.go new file mode 100644 index 000000000..d5b25885f --- /dev/null +++ b/internal/config/runtime_credentials_test.go @@ -0,0 +1,138 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/hetznercloud/hcloud-cloud-controller-manager/internal/testsupport" +) + +func TestRuntimeCredentialFilesHelpers(t *testing.T) { + files := RuntimeCredentialFiles{} + assert.False(t, files.HasAny()) + assert.Empty(t, files.Directories()) + + files = RuntimeCredentialFiles{ + HCloudToken: "/tmp/one/token", + RobotUser: "/tmp/two/user", + RobotPassword: "/tmp/two/password", + } + + assert.True(t, files.HasAny()) + assert.ElementsMatch(t, []string{"/tmp/one", "/tmp/two"}, files.Directories()) +} + +func TestLookupRuntimeCredentialFiles(t *testing.T) { + defer unsetEnv(t, hcloudToken)() + defer unsetEnv(t, robotUser)() + defer unsetEnv(t, robotPassword)() + + resetEnv := testsupport.Setenv(t, + hcloudToken+"_FILE", "/tmp/hcloud-token", + robotUser+"_FILE", "/tmp/robot-user", + robotPassword+"_FILE", "/tmp/robot-password", + ) + defer resetEnv() + + files := LookupRuntimeCredentialFiles() + assert.Equal(t, RuntimeCredentialFiles{ + HCloudToken: "/tmp/hcloud-token", + RobotUser: "/tmp/robot-user", + RobotPassword: "/tmp/robot-password", + }, files) +} + +func TestLookupRuntimeCredentialFilesIgnoresPlainEnvironmentVariables(t *testing.T) { + resetEnv := testsupport.Setenv(t, + hcloudToken, "token", + hcloudToken+"_FILE", "/tmp/hcloud-token", + robotUser, "robot-user", + robotUser+"_FILE", "/tmp/robot-user", + robotPassword, "robot-password", + robotPassword+"_FILE", "/tmp/robot-password", + ) + defer resetEnv() + + files := LookupRuntimeCredentialFiles() + assert.Equal(t, RuntimeCredentialFiles{}, files) + assert.Empty(t, lookupCredentialFile(hcloudToken)) +} + +func TestLookupRobotCredentials(t *testing.T) { + resetEnv := testsupport.Setenv(t, + robotUser, "robot-user-1", + robotPassword, "robot-password-1", + ) + defer resetEnv() + + user, password, err := LookupRobotCredentials() + assert.NoError(t, err) + assert.Equal(t, "robot-user-1", user) + assert.Equal(t, "robot-password-1", password) +} + +func TestLookupHCloudToken(t *testing.T) { + resetEnv := testsupport.Setenv(t, hcloudToken, "token-1") + defer resetEnv() + + token, err := LookupHCloudToken() + assert.NoError(t, err) + assert.Equal(t, "token-1", token) +} + +func TestLookupRobotCredentialsRejectsPartialCredentials(t *testing.T) { + defer unsetEnv(t, robotPassword)() + + resetEnv := testsupport.Setenv(t, robotUser, "robot-user-1") + defer resetEnv() + + _, _, err := LookupRobotCredentials() + assert.EqualError(t, err, `both "ROBOT_USER" and "ROBOT_PASSWORD" must be provided, or neither`) +} + +func TestLookupRobotCredentialsJoinsFileErrors(t *testing.T) { + defer unsetEnv(t, robotUser)() + defer unsetEnv(t, robotPassword)() + + resetEnv := testsupport.Setenv(t, + robotUser+"_FILE", filepath.Join(t.TempDir(), "missing-user"), + robotPassword+"_FILE", filepath.Join(t.TempDir(), "missing-password"), + ) + defer resetEnv() + + _, _, err := LookupRobotCredentials() + assert.ErrorContains(t, err, "missing-user") + assert.ErrorContains(t, err, "missing-password") +} + +func TestReadCredentialFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "credential") + assert.NoError(t, os.WriteFile(path, []byte(" token-1\n"), 0o600)) + + value, err := ReadCredentialFile(path) + assert.NoError(t, err) + assert.Equal(t, "token-1", value) +} + +func TestReadCredentialFileReturnsReadError(t *testing.T) { + _, err := ReadCredentialFile(filepath.Join(t.TempDir(), "missing")) + assert.ErrorContains(t, err, "no such file or directory") +} + +func unsetEnv(t *testing.T, key string) func() { + t.Helper() + + value, ok := os.LookupEnv(key) + assert.NoError(t, os.Unsetenv(key)) + + return func() { + if !ok { + assert.NoError(t, os.Unsetenv(key)) + return + } + assert.NoError(t, os.Setenv(key, value)) + } +}