Hot Reload of Credentials (Still Work in Progress)

.gitignore

@@ -8,3 +8,4 @@ hcloud-cloud-controller-manager
 *.tgz
 hack/.*
 coverage/
+.vscode

chart/values.yaml

@@ -45,6 +45,10 @@ env:
   # This is currently possible for HCLOUD_TOKEN, ROBOT_USER, and ROBOT_PASSWORD.
   # Use the env var appended with _FILE (e.g. HCLOUD_TOKEN_FILE) and set the value to the file path that should be read
   # The file must be provided externally (e.g. via secret injection).
+  # Hot reloading only works with file-backed secrets. It does not work when
+  # credentials are provided via regular environment variables or
+  # valueFrom.secretKeyRef, because Kubernetes does not update the process
+  # environment of a running container.
   # Example:
   # HCLOUD_TOKEN_FILE:
   #   value: "/etc/hetzner/token"

docs/reference/helm/extra-envs.md

@@ -2,6 +2,8 @@
 
 You can define extra environment variables for the HCCM. Both Kubernetes formats are supported: `value` and `valueFrom`. The `valueFrom` field can reference multiple sources such as ConfigMaps and Secrets, but also supports other options. For more details, see the Kubernetes documentation on [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/#using-configmaps-as-environment-variables) and [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables).
 
+If you want credential hot reloading, do not provide `HCLOUD_TOKEN`, `ROBOT_USER`, or `ROBOT_PASSWORD` via regular environment variables or `valueFrom.secretKeyRef`. Hot reloading only works when these credentials are read from files via `HCLOUD_TOKEN_FILE`, `ROBOT_USER_FILE`, and `ROBOT_PASSWORD_FILE`, backed by a mounted Secret volume. Kubernetes updates mounted Secret files, but it does not update the environment of a running container.
+
 ```yaml
 env:
   ROBOT_USER:
@@ -17,3 +19,18 @@ env:
         key: robot-user
         optional: true
 ```
+
+Example for file-backed credentials with hot reloading:
+
+```yaml
+env:
+  HCLOUD_TOKEN: null
+  ROBOT_USER: null
+  ROBOT_PASSWORD: null
+  HCLOUD_TOKEN_FILE:
+    value: /etc/hetzner/token
+  ROBOT_USER_FILE:
+    value: /etc/hetzner/robot-user
+  ROBOT_PASSWORD_FILE:
+    value: /etc/hetzner/robot-password
+```

hcloud/cloud.go

@@ -19,7 +19,6 @@ package hcloud
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -52,6 +51,7 @@ type cloud struct {
 	client      *hcloud.Client
 	robotClient robot.Client
 	cfg         config.HCCMConfiguration
+	credentials *runtimeCredentials
 	recorder    record.EventRecorder
 	networkID   int64
 	cidr        string
@@ -70,14 +70,14 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) {
 		return nil, err
 	}
 
+	credentials, err := newRuntimeCredentials()
+	if err != nil {
+		return nil, err
+	}
+
 	opts := []hcloud.ClientOption{
-		hcloud.WithToken(cfg.HCloudClient.Token),
 		hcloud.WithApplication("hcloud-cloud-controller", providerVersion),
-		hcloud.WithHTTPClient(
-			&http.Client{
-				Timeout: apiClientTimeout,
-			},
-		),
+		hcloud.WithHTTPClient(newHCloudHTTPClient(apiClientTimeout, credentials)),
 	}
 
 	// start metrics server if enabled (enabled by default)
@@ -100,9 +100,7 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) {
 		c := hrobot.NewBasicAuthClientWithCustomHttpClient(
 			cfg.Robot.User,
 			cfg.Robot.Password,
-			&http.Client{
-				Timeout: apiClientTimeout,
-			},
+			newRobotHTTPClient(apiClientTimeout, credentials),
 		)
 
 		robotClient = robot.NewRateLimitedClient(
@@ -145,6 +143,7 @@ func NewCloud(cidr string) (cloudprovider.Interface, error) {
 		client:      client,
 		robotClient: robotClient,
 		cfg:         cfg,
+		credentials: credentials,
 		networkID:   networkID,
 		cidr:        cidr,
 	}, nil
@@ -158,6 +157,11 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder,
 
 	go func() {
 		<-stop
+		if c.credentials != nil {
+			if err := c.credentials.close(); err != nil {
+				klog.ErrorS(err, "close runtime credential watcher")
+			}
+		}
 		eventBroadcaster.Shutdown()
 	}()
 

hcloud/runtime_credentials.go

@@ -0,0 +1,281 @@
+package hcloud
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"golang.org/x/net/http/httpguts"
+	"k8s.io/klog/v2"
+
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/config"
+)
+
+const invalidAuthorizationTokenError = "authorization token contains invalid characters"
+const credentialsReloadDebounce = 100 * time.Millisecond
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+type runtimeCredentials struct {
+	mu sync.RWMutex
+
+	hcloudToken string
+	robotUser   string
+	robotPass   string
+
+	hcloudTokenPath string
+	robotUserPath   string
+	robotPassPath   string
+
+	watcher   *fsnotify.Watcher
+	closeOnce sync.Once
+}
+
+func newRuntimeCredentials() (*runtimeCredentials, error) {
+	credentials := &runtimeCredentials{}
+
+	if err := credentials.loadInitial(); err != nil {
+		return nil, err
+	}
+
+	files := config.LookupRuntimeCredentialFiles()
+	credentials.hcloudTokenPath = files.HCloudToken
+	credentials.robotUserPath = files.RobotUser
+	credentials.robotPassPath = files.RobotPassword
+
+	if !files.HasAny() {
+		return credentials, nil
+	}
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, dir := range files.Directories() {
+		if err := watcher.Add(dir); err != nil {
+			watcher.Close()
+			return nil, fmt.Errorf("watch credentials directory %q: %w", dir, err)
+		}
+	}
+
+	credentials.watcher = watcher
+	go credentials.watch()
+	return credentials, nil
+}
+
+func (c *runtimeCredentials) loadInitial() error {
+	token, err := config.LookupHCloudToken()
+	if err != nil {
+		return err
+	}
+	if token != "" && !httpguts.ValidHeaderFieldValue(token) {
+		return fmt.Errorf(invalidAuthorizationTokenError)
+	}
+
+	user, password, err := config.LookupRobotCredentials()
+	if err != nil {
+		return err
+	}
+
+	c.mu.Lock()
+	c.hcloudToken = token
+	c.robotUser = user
+	c.robotPass = password
+	c.mu.Unlock()
+
+	return nil
+}
+
+func (c *runtimeCredentials) watch() {
+	var debounceTimer *time.Timer
+	var debounceC <-chan time.Time
+
+	stopDebounce := func() {
+		if debounceTimer == nil {
+			return
+		}
+		if !debounceTimer.Stop() {
+			select {
+			case <-debounceTimer.C:
+			default:
+			}
+		}
+	}
+
+	defer stopDebounce()
+
+	for {
+		select {
+		case event, ok := <-c.watcher.Events:
+			if !ok {
+				return
+			}
+			if !shouldReload(event) {
+				continue
+			}
+			if debounceTimer == nil {
+				debounceTimer = time.NewTimer(credentialsReloadDebounce)
+				debounceC = debounceTimer.C
+				continue
+			}
+			stopDebounce()
+			debounceTimer.Reset(credentialsReloadDebounce)
+		case err, ok := <-c.watcher.Errors:
+			if !ok {
+				return
+			}
+			klog.ErrorS(err, "watching mounted credential files")
+		case <-debounceC:
+			debounceC = nil
+			c.reload()
+		}
+	}
+}
+
+func shouldReload(event fsnotify.Event) bool {
+	return event.Op&(fsnotify.Create|fsnotify.Write|fsnotify.Rename|fsnotify.Remove|fsnotify.Chmod) != 0
+}
+
+func (c *runtimeCredentials) reload() {
+	if c.hcloudTokenPath != "" {
+		token, err := config.ReadCredentialFile(c.hcloudTokenPath)
+		switch {
+		case err != nil:
+			klog.ErrorS(err, "reloading HCLOUD_TOKEN from mounted secret")
+		case token != "" && !httpguts.ValidHeaderFieldValue(token):
+			klog.ErrorS(fmt.Errorf(invalidAuthorizationTokenError), "reloading HCLOUD_TOKEN from mounted secret")
+		default:
+			c.mu.Lock()
+			c.hcloudToken = token
+			c.mu.Unlock()
+		}
+	}
+
+	if c.robotUserPath != "" || c.robotPassPath != "" {
+		user, password, err := c.loadRobotCredentials()
+		if err != nil {
+			klog.ErrorS(err, "reloading Robot credentials from mounted secret")
+			return
+		}
+		c.mu.Lock()
+		c.robotUser = user
+		c.robotPass = password
+		c.mu.Unlock()
+	}
+}
+
+func (c *runtimeCredentials) loadRobotCredentials() (string, string, error) {
+	c.mu.RLock()
+	user := c.robotUser
+	password := c.robotPass
+	c.mu.RUnlock()
+
+	var err error
+	if c.robotUserPath != "" {
+		user, err = config.ReadCredentialFile(c.robotUserPath)
+		if err != nil {
+			return "", "", err
+		}
+	}
+	if c.robotPassPath != "" {
+		password, err = config.ReadCredentialFile(c.robotPassPath)
+		if err != nil {
+			return "", "", err
+		}
+	}
+	if (user == "") != (password == "") {
+		return "", "", fmt.Errorf("both %q and %q must be provided, or neither", "ROBOT_USER", "ROBOT_PASSWORD")
+	}
+	return user, password, nil
+}
+
+func (c *runtimeCredentials) close() error {
+	var err error
+	c.closeOnce.Do(func() {
+		if c.watcher != nil {
+			err = c.watcher.Close()
+		}
+	})
+	return err
+}
+
+func (c *runtimeCredentials) hcloudAuthorization() string {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.hcloudToken == "" {
+		return ""
+	}
+	return "Bearer " + c.hcloudToken
+}
+
+func (c *runtimeCredentials) robotCredentials() (string, string) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.robotUser, c.robotPass
+}
+
+func newHCloudHTTPClient(timeout time.Duration, credentials *runtimeCredentials) *http.Client {
+	return &http.Client{
+		Timeout:   timeout,
+		Transport: newHCloudCredentialReloader(credentials, nil),
+	}
+}
+
+func newRobotHTTPClient(timeout time.Duration, credentials *runtimeCredentials) *http.Client {
+	return &http.Client{
+		Timeout:   timeout,
+		Transport: newRobotCredentialReloader(credentials, nil),
+	}
+}
+
+func newHCloudCredentialReloader(credentials *runtimeCredentials, next http.RoundTripper) http.RoundTripper {
+	next = transportOrDefault(next)
+
+	return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		cloned := cloneRequest(req)
+		auth := credentials.hcloudAuthorization()
+		if auth == "" {
+			cloned.Header.Del("Authorization")
+		} else {
+			cloned.Header.Set("Authorization", auth)
+		}
+		return next.RoundTrip(cloned)
+	})
+}
+
+func newRobotCredentialReloader(credentials *runtimeCredentials, next http.RoundTripper) http.RoundTripper {
+	next = transportOrDefault(next)
+
+	return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		cloned := cloneRequest(req)
+		user, password := credentials.robotCredentials()
+		if user == "" && password == "" {
+			cloned.Header.Del("Authorization")
+		} else {
+			cloned.SetBasicAuth(user, password)
+		}
+		return next.RoundTrip(cloned)
+	})
+}
+
+func cloneRequest(req *http.Request) *http.Request {
+	cloned := req.Clone(req.Context())
+	cloned.Header = req.Header.Clone()
+	return cloned
+}
+
+func transportOrDefault(next http.RoundTripper) http.RoundTripper {
+	if next != nil {
+		return next
+	}
+	return http.DefaultTransport
+}