diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index 47f828ed..61e415d6 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -2,6 +2,7 @@ package api import ( "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" @@ -27,6 +28,7 @@ type ApiService struct { BuildManager builds.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller VMMetricsManager *vm_metrics.Manager } @@ -44,6 +46,7 @@ func New( buildManager builds.Manager, resourceManager *resources.Manager, guestMemoryController guestmemory.Controller, + autoStandbyController *autostandby.Controller, vmMetricsManager *vm_metrics.Manager, ) *ApiService { return &ApiService{ @@ -57,6 +60,7 @@ func New( BuildManager: buildManager, ResourceManager: resourceManager, GuestMemoryController: guestMemoryController, + AutoStandbyController: autoStandbyController, VMMetricsManager: vmMetricsManager, } } diff --git a/cmd/api/api/auto_standby.go b/cmd/api/api/auto_standby.go new file mode 100644 index 00000000..c483727b --- /dev/null +++ b/cmd/api/api/auto_standby.go @@ -0,0 +1,62 @@ +package api + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/oapi" + "github.com/samber/lo" +) + +func toDomainAutoStandbyPolicy(policy *oapi.AutoStandbyPolicy) (*autostandby.Policy, error) { + if policy == nil { + return nil, nil + } + + out := &autostandby.Policy{} + if policy.Enabled != nil { + out.Enabled = *policy.Enabled + } + if policy.IdleTimeout != nil { + out.IdleTimeout = *policy.IdleTimeout + } + if policy.IgnoreSourceCidrs != nil { + out.IgnoreSourceCIDRs = append([]string(nil), (*policy.IgnoreSourceCidrs)...) + } + if policy.IgnoreDestinationPorts != nil { + out.IgnoreDestinationPorts = make([]uint16, 0, len(*policy.IgnoreDestinationPorts)) + for _, port := range *policy.IgnoreDestinationPorts { + if port < 1 || port > 65535 { + return nil, fmt.Errorf("auto_standby.ignore_destination_ports must be between 1 and 65535") + } + out.IgnoreDestinationPorts = append(out.IgnoreDestinationPorts, uint16(port)) + } + } + + return out, nil +} + +func toOAPIAutoStandbyPolicy(policy *autostandby.Policy) *oapi.AutoStandbyPolicy { + if policy == nil { + return nil + } + + out := &oapi.AutoStandbyPolicy{ + Enabled: lo.ToPtr(policy.Enabled), + } + if policy.IdleTimeout != "" { + out.IdleTimeout = lo.ToPtr(policy.IdleTimeout) + } + if len(policy.IgnoreSourceCIDRs) > 0 { + out.IgnoreSourceCidrs = lo.ToPtr(append([]string(nil), policy.IgnoreSourceCIDRs...)) + } + if len(policy.IgnoreDestinationPorts) > 0 { + ports := make([]int, 0, len(policy.IgnoreDestinationPorts)) + for _, port := range policy.IgnoreDestinationPorts { + ports = append(ports, int(port)) + } + out.IgnoreDestinationPorts = &ports + } + + return out +} diff --git a/cmd/api/api/auto_standby_status.go b/cmd/api/api/auto_standby_status.go new file mode 100644 index 00000000..5393a1e8 --- /dev/null +++ b/cmd/api/api/auto_standby_status.go @@ -0,0 +1,79 @@ +package api + +import ( + "context" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/oapi" + "github.com/samber/lo" +) + +func (s *ApiService) GetAutoStandbyStatus(ctx context.Context, request oapi.GetAutoStandbyStatusRequestObject) (oapi.GetAutoStandbyStatusResponseObject, error) { + log := logger.FromContext(ctx) + + inst, err := s.InstanceManager.GetInstance(ctx, request.Id) + if err != nil { + if err == instances.ErrNotFound || err == instances.ErrAmbiguousName { + return oapi.GetAutoStandbyStatus404JSONResponse{ + Code: "not_found", + Message: "instance not found", + }, nil + } + log.ErrorContext(ctx, "failed to resolve instance for auto-standby status", "instance_id", request.Id, "error", err) + return oapi.GetAutoStandbyStatus500JSONResponse{ + Code: "internal_error", + Message: "failed to load instance", + }, nil + } + + snapshot := autostandby.StatusSnapshot{ + Supported: false, + Configured: inst.AutoStandby != nil, + Enabled: inst.AutoStandby != nil && inst.AutoStandby.Enabled, + TrackingMode: "conntrack_events_v4_tcp", + Status: autostandby.StatusUnsupported, + Reason: autostandby.ReasonUnsupportedPlatform, + } + if s.AutoStandbyController != nil { + snapshot = s.AutoStandbyController.Describe(instanceToAutoStandby(*inst)) + } + + return oapi.GetAutoStandbyStatus200JSONResponse(toOAPIAutoStandbyStatus(snapshot)), nil +} + +func instanceToAutoStandby(inst instances.Instance) autostandby.Instance { + return autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + } +} + +func toOAPIAutoStandbyStatus(status autostandby.StatusSnapshot) oapi.AutoStandbyStatus { + out := oapi.AutoStandbyStatus{ + ActiveInboundConnections: status.ActiveInboundCount, + Configured: status.Configured, + Eligible: status.Eligible, + Enabled: status.Enabled, + Reason: oapi.AutoStandbyStatusReason(status.Reason), + Status: oapi.AutoStandbyStatusStatus(status.Status), + Supported: status.Supported, + TrackingMode: status.TrackingMode, + } + if status.IdleTimeout != "" { + out.IdleTimeout = lo.ToPtr(status.IdleTimeout) + } + out.IdleSince = status.IdleSince + out.LastInboundActivityAt = status.LastInboundActivityAt + out.NextStandbyAt = status.NextStandbyAt + if status.CountdownRemaining != nil { + out.CountdownRemaining = lo.ToPtr(status.CountdownRemaining.String()) + } + return out +} diff --git a/cmd/api/api/auto_standby_status_test.go b/cmd/api/api/auto_standby_status_test.go new file mode 100644 index 00000000..b4aecfd1 --- /dev/null +++ b/cmd/api/api/auto_standby_status_test.go @@ -0,0 +1,170 @@ +package api + +import ( + "context" + "net/netip" + "testing" + "time" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type captureStatusManager struct { + instances.Manager + instance *instances.Instance + err error +} + +func (m *captureStatusManager) GetInstance(context.Context, string) (*instances.Instance, error) { + if m.err != nil { + return nil, m.err + } + return m.instance, nil +} + +type statusStore struct { + instances []autostandby.Instance + runtime map[string]*autostandby.Runtime + events chan autostandby.InstanceEvent +} + +func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, error) { + return append([]autostandby.Instance(nil), s.instances...), nil +} + +func (s *statusStore) StandbyInstance(context.Context, string) error { return nil } + +func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { + if s.runtime == nil { + s.runtime = make(map[string]*autostandby.Runtime) + } + s.runtime[id] = runtime + return nil +} + +func (s *statusStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { + if s.events == nil { + s.events = make(chan autostandby.InstanceEvent) + } + return s.events, func() {}, nil +} + +type statusConnectionSource struct { + connections []autostandby.Connection +} + +func (s *statusConnectionSource) ListConnections(context.Context) ([]autostandby.Connection, error) { + return append([]autostandby.Connection(nil), s.connections...), nil +} + +func (s *statusConnectionSource) OpenStream(context.Context) (autostandby.ConnectionStream, error) { + return &statusConnectionStream{ + events: make(chan autostandby.ConnectionEvent), + errs: make(chan error), + }, nil +} + +type statusConnectionStream struct { + events chan autostandby.ConnectionEvent + errs chan error +} + +func (s *statusConnectionStream) Events() <-chan autostandby.ConnectionEvent { return s.events } + +func (s *statusConnectionStream) Errors() <-chan error { return s.errs } + +func (s *statusConnectionStream) Close() error { return nil } + +func TestGetAutoStandbyStatusUnsupportedWithoutController(t *testing.T) { + t.Parallel() + + base := newTestService(t) + base.InstanceManager = &captureStatusManager{ + Manager: base.InstanceManager, + instance: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-1", + Name: "inst-1", + NetworkEnabled: true, + IP: "192.168.100.10", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + }, + State: instances.StateRunning, + }, + } + + resp, err := base.GetAutoStandbyStatus(ctx(), oapi.GetAutoStandbyStatusRequestObject{Id: "inst-1"}) + require.NoError(t, err) + + statusResp, ok := resp.(oapi.GetAutoStandbyStatus200JSONResponse) + require.True(t, ok) + assert.False(t, statusResp.Supported) + assert.Equal(t, oapi.AutoStandbyStatusStatusUnsupported, statusResp.Status) + assert.Equal(t, oapi.AutoStandbyStatusReasonUnsupportedPlatform, statusResp.Reason) +} + +func TestGetAutoStandbyStatusActive(t *testing.T) { + t.Parallel() + + inst := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-2", + Name: "inst-2", + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + }, + State: instances.StateRunning, + } + + now := time.Date(2026, 4, 6, 12, 0, 0, 0, time.UTC) + store := &statusStore{ + instances: []autostandby.Instance{{ + ID: "inst-2", + Name: "inst-2", + State: autostandby.StateRunning, + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + }}, + } + source := &statusConnectionSource{connections: []autostandby.Connection{{ + OriginalSourceIP: mustStatusAddr("1.2.3.4"), + OriginalSourcePort: 51234, + OriginalDestinationIP: mustStatusAddr("192.168.100.20"), + OriginalDestinationPort: 8080, + TCPState: autostandby.TCPStateEstablished, + }}} + controller := autostandby.NewController(store, source, autostandby.ControllerOptions{ + Now: func() time.Time { return now }, + }) + require.NoError(t, controller.Run(withCanceledContext(t))) + + base := newTestService(t) + base.InstanceManager = &captureStatusManager{Manager: base.InstanceManager, instance: inst} + base.AutoStandbyController = controller + + resp, err := base.GetAutoStandbyStatus(ctx(), oapi.GetAutoStandbyStatusRequestObject{Id: "inst-2"}) + require.NoError(t, err) + + statusResp, ok := resp.(oapi.GetAutoStandbyStatus200JSONResponse) + require.True(t, ok) + assert.True(t, statusResp.Supported) + assert.Equal(t, oapi.AutoStandbyStatusStatusActive, statusResp.Status) + assert.Equal(t, 1, statusResp.ActiveInboundConnections) +} + +func withCanceledContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +func mustStatusAddr(raw string) netip.Addr { + return netip.MustParseAddr(raw) +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index b7248207..90f3457a 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -278,6 +278,13 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if request.Body.Cmd != nil { cmd = *request.Body.Cmd } + autoStandby, err := toDomainAutoStandbyPolicy(request.Body.AutoStandby) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_auto_standby", + Message: err.Error(), + }, nil + } domainReq := instances.CreateInstanceRequest{ Name: request.Body.Name, @@ -302,6 +309,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Cmd: cmd, SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, + AutoStandby: autoStandby, } if request.Body.SnapshotPolicy != nil { snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) @@ -924,9 +932,17 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst if request.Body.Env != nil { env = *request.Body.Env } + autoStandby, err := toDomainAutoStandbyPolicy(request.Body.AutoStandby) + if err != nil { + return oapi.UpdateInstance400JSONResponse{ + Code: "invalid_auto_standby", + Message: err.Error(), + }, nil + } result, err := s.InstanceManager.UpdateInstance(ctx, inst.Id, instances.UpdateInstanceRequest{ - Env: env, + Env: env, + AutoStandby: autoStandby, }) if err != nil { switch { @@ -1057,6 +1073,7 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { oapiPolicy := toOAPISnapshotPolicy(*inst.SnapshotPolicy) oapiInst.SnapshotPolicy = &oapiPolicy } + oapiInst.AutoStandby = toOAPIAutoStandbyPolicy(inst.AutoStandby) // Convert volume attachments if len(inst.Volumes) > 0 { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b3f74ea2..3244e9c2 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" @@ -276,6 +277,7 @@ func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, re Name: "updated-instance", Image: "docker.io/library/alpine:latest", Env: req.Env, + AutoStandby: req.AutoStandby, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -297,6 +299,7 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances HotplugSize: req.HotplugSize, OverlaySize: req.OverlaySize, Vcpus: req.Vcpus, + AutoStandby: req.AutoStandby, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -477,6 +480,49 @@ func TestCreateInstance_MapsNetworkEgressEnforcementMode(t *testing.T) { assert.Equal(t, instances.EgressEnforcementModeHTTPHTTPSOnly, mockMgr.lastReq.NetworkEgress.EnforcementMode) } +func TestCreateInstance_MapsAutoStandbyPolicy(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + enabled := true + idleTimeout := "5m" + ignoreSourceCidrs := []string{"10.0.0.0/8", "192.168.0.0/16"} + ignoreDestinationPorts := []int{22, 9000} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-auto-standby", + Image: "docker.io/library/alpine:latest", + AutoStandby: &oapi.AutoStandbyPolicy{ + Enabled: &enabled, + IdleTimeout: &idleTimeout, + IgnoreSourceCidrs: &ignoreSourceCidrs, + IgnoreDestinationPorts: &ignoreDestinationPorts, + }, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.AutoStandby) + assert.True(t, mockMgr.lastReq.AutoStandby.Enabled) + assert.Equal(t, "5m", mockMgr.lastReq.AutoStandby.IdleTimeout) + assert.Equal(t, []string{"10.0.0.0/8", "192.168.0.0/16"}, mockMgr.lastReq.AutoStandby.IgnoreSourceCIDRs) + assert.Equal(t, []uint16{22, 9000}, mockMgr.lastReq.AutoStandby.IgnoreDestinationPorts) + + instance := oapi.Instance(created) + require.NotNil(t, instance.AutoStandby) + require.NotNil(t, instance.AutoStandby.Enabled) + assert.True(t, *instance.AutoStandby.Enabled) + assert.Equal(t, idleTimeout, *instance.AutoStandby.IdleTimeout) +} + func TestUpdateInstance_MapsEnvPatch(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -524,6 +570,114 @@ func TestUpdateInstance_MapsEnvPatch(t *testing.T) { assert.Equal(t, "rotated-key-456", mockMgr.lastReq.Env["OUTBOUND_OPENAI_KEY"]) } +func TestUpdateInstance_MapsAutoStandbyPatch(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{ + Manager: origMgr, + result: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-auto-standby", + Name: "inst-update-auto-standby", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "10m0s", + }, + }, + State: instances.StateStopped, + }, + } + svc.InstanceManager = mockMgr + + enabled := true + idleTimeout := "10m" + ignoreDestinationPorts := []int{22} + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-auto-standby", + Name: "inst-update-auto-standby", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + AutoStandby: &oapi.AutoStandbyPolicy{ + Enabled: &enabled, + IdleTimeout: &idleTimeout, + IgnoreDestinationPorts: &ignoreDestinationPorts, + }, + }, + }) + require.NoError(t, err) + updated, ok := resp.(oapi.UpdateInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.AutoStandby) + assert.Equal(t, resolved.Id, mockMgr.lastID) + assert.True(t, mockMgr.lastReq.AutoStandby.Enabled) + assert.Equal(t, "10m", mockMgr.lastReq.AutoStandby.IdleTimeout) + assert.Equal(t, []uint16{22}, mockMgr.lastReq.AutoStandby.IgnoreDestinationPorts) + + instance := oapi.Instance(updated) + require.NotNil(t, instance.AutoStandby) + require.NotNil(t, instance.AutoStandby.Enabled) + assert.True(t, *instance.AutoStandby.Enabled) +} + +func TestUpdateInstance_RejectsZeroAutoStandbyIgnoreDestinationPort(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-auto-standby", + Name: "inst-update-auto-standby", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + enabled := true + idleTimeout := "10m" + ignoreDestinationPorts := []int{0} + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + AutoStandby: &oapi.AutoStandbyPolicy{ + Enabled: &enabled, + IdleTimeout: &idleTimeout, + IgnoreDestinationPorts: &ignoreDestinationPorts, + }, + }, + }) + require.NoError(t, err) + + badReq, ok := resp.(oapi.UpdateInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_auto_standby", badReq.Code) + assert.Contains(t, badReq.Message, "between 1 and 65535") + assert.Nil(t, mockMgr.lastReq) +} + func TestUpdateInstance_RequiresBody(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/cmd/api/main.go b/cmd/api/main.go index 3430b9fe..42de6b88 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -495,6 +495,12 @@ func run() error { logger.Info("starting guest memory controller") return app.GuestMemoryController.Start(gctx) }) + if app.AutoStandbyController != nil { + grp.Go(func() error { + logger.Info("starting auto-standby controller") + return app.AutoStandbyController.Run(gctx) + }) + } // Run the server grp.Go(func() error { diff --git a/cmd/api/wire.go b/cmd/api/wire.go index b50c27e2..0888ec4a 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -9,6 +9,7 @@ import ( "github.com/google/wire" "github.com/kernel/hypeman/cmd/api/api" "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" @@ -39,6 +40,7 @@ type application struct { BuildManager builds.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller VMMetricsManager *vm_metrics.Manager Registry *registry.Registry ApiService *api.ApiService @@ -61,6 +63,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideBuildManager, providers.ProvideResourceManager, providers.ProvideGuestMemoryController, + providers.ProvideAutoStandbyController, providers.ProvideVMMetricsManager, providers.ProvideRegistry, api.New, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index a5007a7e..57f55c6d 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -10,6 +10,7 @@ import ( "context" "github.com/kernel/hypeman/cmd/api/api" "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" @@ -69,6 +70,7 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } + autostandbyController := providers.ProvideAutoStandbyController(instancesManager, logger) vm_metricsManager, err := providers.ProvideVMMetricsManager(instancesManager, config, logger) if err != nil { return nil, nil, err @@ -77,7 +79,7 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } - apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, controller, vm_metricsManager) + apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, controller, autostandbyController, vm_metricsManager) mainApplication := &application{ Ctx: context, Logger: logger, @@ -92,6 +94,7 @@ func initializeApp() (*application, func(), error) { BuildManager: buildsManager, ResourceManager: resourcesManager, GuestMemoryController: controller, + AutoStandbyController: autostandbyController, VMMetricsManager: vm_metricsManager, Registry: registry, ApiService: apiService, @@ -117,6 +120,7 @@ type application struct { BuildManager builds.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller VMMetricsManager *vm_metrics.Manager Registry *registry.Registry ApiService *api.ApiService diff --git a/lib/autostandby/README.md b/lib/autostandby/README.md new file mode 100644 index 00000000..d3ee41c4 --- /dev/null +++ b/lib/autostandby/README.md @@ -0,0 +1,67 @@ +# Auto Standby + +This feature automatically puts a Linux VM into `Standby` after it has stopped serving inbound TCP traffic for a configured amount of time. + +## What counts as activity + +The feature looks at host-side conntrack state, not ingress configuration and not TAP byte counters. + +A VM is considered active when there is at least one tracked TCP flow where: + +- the original destination is the VM's private IP +- the VM is the server/responding side of the connection +- the flow is currently tracked as live by conntrack + +That means: + +- inbound client connections keep the VM awake +- replies to outbound guest requests do not keep the VM awake +- same-host clients count by default + +## Idle behavior + +Hypeman seeds its controller from a conntrack snapshot on startup, then keeps state current with conntrack netlink events. + +- new inbound TCP flows are tracked from conntrack `NEW` events +- TCP teardown is treated as inactivity once conntrack reports a terminal state or the flow disappears +- connections that were already open when Hypeman started are reconciled against fresh conntrack snapshots until they drain, so restart-seeded traffic can still age out correctly +- Hypeman also performs a full snapshot sync every 5 minutes by default as a low-frequency consistency check; the controller interval is configurable + +When the active inbound TCP connection count reaches zero, Hypeman starts an idle timer for that instance. + +- if a new inbound TCP connection appears before the timer expires, the timer is cleared +- if the count stays at zero for the full `idle_timeout`, Hypeman places the VM into `Standby` + +The idle timestamps are also persisted in instance metadata. + +- if Hypeman restarts and a startup conntrack snapshot shows current inbound connections, the instance is treated as active immediately and any old idle countdown is cleared +- if Hypeman restarts and the snapshot shows zero current inbound connections, Hypeman resumes the persisted idle countdown + +This keeps the restart behavior conservative about current traffic while still allowing long idle windows to carry across control-plane restarts. + +## Exclusions + +Instances can ignore some traffic when deciding whether they are active: + +- `ignore_source_cidrs` excludes matching client source ranges +- `ignore_destination_ports` excludes matching VM destination ports + +This is intended for probes, internal callers, or ports that should not keep a VM warm. + +## Limits + +- Linux only +- TCP only +- IPv4 conntrack only +- Wake-on-traffic is not part of this feature + +## Status endpoint + +Hypeman exposes a diagnostic status endpoint for each instance that reports: + +- whether auto-standby is supported, configured, enabled, and currently eligible +- how many qualifying inbound TCP connections are currently keeping the VM awake +- the current idle timer timestamps and next planned standby time +- the current controller reason, such as active inbound traffic, countdown still running, or observer failure + +Wake-on-traffic would require a separate host-owned listener or forwarding layer that can accept a connection while the VM is asleep, trigger restore, and then hand traffic through once the VM is running. diff --git a/lib/autostandby/classifier.go b/lib/autostandby/classifier.go new file mode 100644 index 00000000..1091bf91 --- /dev/null +++ b/lib/autostandby/classifier.go @@ -0,0 +1,57 @@ +package autostandby + +import ( + "fmt" + "net/netip" + "time" +) + +// ActiveInboundCount returns the number of active inbound TCP connections for an instance +// and the compiled idle timeout that should be applied to it. +func ActiveInboundCount(inst Instance, conns []Connection) (int, time.Duration, error) { + compiled, err := compilePolicy(inst.AutoStandby) + if err != nil { + return 0, 0, err + } + if compiled == nil { + return 0, 0, nil + } + + instanceIP, err := netip.ParseAddr(inst.IP) + if err != nil { + return 0, 0, fmt.Errorf("parse instance IP %q: %w", inst.IP, err) + } + + count := 0 + for _, conn := range conns { + if matchesInboundConnection(instanceIP, compiled, conn) { + count++ + } + } + + return count, compiled.idleTimeout, nil +} + +func matchesInboundConnection(instanceIP netip.Addr, policy *compiledPolicy, conn Connection) bool { + if policy == nil { + return false + } + if !conn.TCPState.Active() { + return false + } + if !conn.OriginalDestinationIP.IsValid() || conn.OriginalDestinationIP != instanceIP { + return false + } + if _, ignored := policy.ignorePorts[conn.OriginalDestinationPort]; ignored { + return false + } + if !conn.OriginalSourceIP.IsValid() { + return false + } + for _, prefix := range policy.ignoreSourceCIDRs { + if prefix.Contains(conn.OriginalSourceIP) { + return false + } + } + return true +} diff --git a/lib/autostandby/classifier_test.go b/lib/autostandby/classifier_test.go new file mode 100644 index 00000000..bc778069 --- /dev/null +++ b/lib/autostandby/classifier_test.go @@ -0,0 +1,77 @@ +package autostandby + +import ( + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestActiveInboundCountCountsOnlyQualifyingInboundTCP(t *testing.T) { + t.Parallel() + + inst := Instance{ + ID: "inst-1", + IP: "192.168.100.10", + State: StateRunning, + AutoStandby: &Policy{ + Enabled: true, + IdleTimeout: "5m", + IgnoreSourceCIDRs: []string{"10.0.0.0/8"}, + IgnoreDestinationPorts: []uint16{22}, + }, + } + + count, idleTimeout, err := ActiveInboundCount(inst, []Connection{ + { + OriginalSourceIP: netip.MustParseAddr("1.2.3.4"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationIP: netip.MustParseAddr("8.8.8.8"), + OriginalDestinationPort: 443, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("10.1.2.3"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("5.6.7.8"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 22, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("9.9.9.9"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateTimeWait, + }, + }) + require.NoError(t, err) + + assert.Equal(t, 1, count) + assert.Equal(t, 5*time.Minute, idleTimeout) +} + +func TestActiveInboundCountRejectsInvalidInstanceIP(t *testing.T) { + t.Parallel() + + _, _, err := ActiveInboundCount(Instance{ + IP: "not-an-ip", + AutoStandby: &Policy{ + Enabled: true, + IdleTimeout: "5m", + }, + }, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "parse instance IP") +} diff --git a/lib/autostandby/conntrack_events_linux.go b/lib/autostandby/conntrack_events_linux.go new file mode 100644 index 00000000..627d0bc1 --- /dev/null +++ b/lib/autostandby/conntrack_events_linux.go @@ -0,0 +1,338 @@ +//go:build linux + +package autostandby + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "sync" + "syscall" + "time" + + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +type conntrackStream struct { + fd int + closeOnce sync.Once + events chan ConnectionEvent + errs chan error +} + +var errUnsupportedConntrackProtocol = errors.New("unsupported conntrack protocol") + +// OpenStream subscribes to IPv4 conntrack NEW, UPDATE, and DESTROY events. +func (s *ConntrackSource) OpenStream(ctx context.Context) (ConnectionStream, error) { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_NETFILTER) + if err != nil { + return nil, fmt.Errorf("open netfilter netlink socket: %w", err) + } + + closeFD := func() { _ = unix.Close(fd) } + if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + closeFD() + return nil, fmt.Errorf("bind netfilter netlink socket: %w", err) + } + if err := unix.SetsockoptInt(fd, unix.SOL_NETLINK, unix.NETLINK_ADD_MEMBERSHIP, unix.NFNLGRP_CONNTRACK_NEW); err != nil { + closeFD() + return nil, fmt.Errorf("subscribe conntrack new events: %w", err) + } + if err := unix.SetsockoptInt(fd, unix.SOL_NETLINK, unix.NETLINK_ADD_MEMBERSHIP, unix.NFNLGRP_CONNTRACK_UPDATE); err != nil { + closeFD() + return nil, fmt.Errorf("subscribe conntrack update events: %w", err) + } + if err := unix.SetsockoptInt(fd, unix.SOL_NETLINK, unix.NETLINK_ADD_MEMBERSHIP, unix.NFNLGRP_CONNTRACK_DESTROY); err != nil { + closeFD() + return nil, fmt.Errorf("subscribe conntrack destroy events: %w", err) + } + + stream := &conntrackStream{ + fd: fd, + events: make(chan ConnectionEvent, 256), + errs: make(chan error, 16), + } + go stream.run(ctx) + return stream, nil +} + +func (s *conntrackStream) Events() <-chan ConnectionEvent { return s.events } + +func (s *conntrackStream) Errors() <-chan error { return s.errs } + +func (s *conntrackStream) Close() error { + var err error + s.closeOnce.Do(func() { + err = unix.Close(s.fd) + }) + return err +} + +func (s *conntrackStream) run(ctx context.Context) { + defer close(s.events) + defer close(s.errs) + + go func() { + <-ctx.Done() + _ = s.Close() + }() + + buf := make([]byte, 1<<20) + for { + n, _, err := unix.Recvfrom(s.fd, buf, 0) + if err != nil { + if ctx.Err() != nil || err == unix.EBADF || err == syscall.ENOTCONN { + return + } + select { + case s.errs <- fmt.Errorf("recv conntrack events: %w", err): + default: + } + return + } + + msgs, err := syscall.ParseNetlinkMessage(buf[:n]) + if err != nil { + select { + case s.errs <- fmt.Errorf("parse conntrack netlink messages: %w", err): + default: + } + continue + } + + for _, msg := range msgs { + event, ok, err := connectionEventFromNetlinkMessage(msg) + if err != nil { + if errors.Is(err, errUnsupportedConntrackProtocol) { + continue + } + select { + case s.errs <- err: + default: + } + continue + } + if !ok { + continue + } + event.ObservedAt = time.Now().UTC() + select { + case s.events <- event: + case <-ctx.Done(): + return + } + } + } +} + +func connectionEventFromNetlinkMessage(msg syscall.NetlinkMessage) (ConnectionEvent, bool, error) { + switch msg.Header.Type { + case unix.NLMSG_NOOP, unix.NLMSG_DONE: + return ConnectionEvent{}, false, nil + case unix.NLMSG_ERROR: + if len(msg.Data) >= 4 { + errno := -int32(nl.NativeEndian().Uint32(msg.Data[:4])) + if errno != 0 { + return ConnectionEvent{}, false, unix.Errno(errno) + } + } + return ConnectionEvent{}, false, nil + } + + if len(msg.Data) < nl.SizeofNfgenmsg { + return ConnectionEvent{}, false, nil + } + if msg.Data[0] != unix.AF_INET { + return ConnectionEvent{}, false, nil + } + + var eventType ConnectionEventType + switch int(msg.Header.Type & 0x00ff) { + case nl.IPCTNL_MSG_CT_NEW: + eventType = ConnectionEventNew + case nl.IPCTNL_MSG_CT_DELETE: + eventType = ConnectionEventDestroy + default: + return ConnectionEvent{}, false, nil + } + + conn, ok, err := connectionFromRawData(msg.Data) + if err != nil || !ok { + return ConnectionEvent{}, false, err + } + return ConnectionEvent{Type: eventType, Connection: conn}, true, nil +} + +func connectionFromRawData(data []byte) (Connection, bool, error) { + if len(data) < nl.SizeofNfgenmsg { + return Connection{}, false, nil + } + + reader := bytes.NewReader(data[nl.SizeofNfgenmsg:]) + conn := Connection{} + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return Connection{}, false, err + } + switch attr.typ { + case nl.CTA_TUPLE_ORIG: + if err := parseTuple(attr.payload, &conn); err != nil { + return Connection{}, false, err + } + case nl.CTA_PROTOINFO: + if err := parseProtoInfo(attr.payload, &conn); err != nil { + return Connection{}, false, err + } + } + } + + if !conn.OriginalSourceIP.IsValid() || !conn.OriginalDestinationIP.IsValid() { + return Connection{}, false, nil + } + return conn, true, nil +} + +type nfAttr struct { + typ uint16 + nested bool + payload []byte +} + +func readNfAttr(reader *bytes.Reader) (nfAttr, error) { + var rawLen uint16 + var rawType uint16 + if err := binary.Read(reader, nl.NativeEndian(), &rawLen); err != nil { + return nfAttr{}, err + } + if err := binary.Read(reader, nl.NativeEndian(), &rawType); err != nil { + return nfAttr{}, err + } + if rawLen < nl.SizeofNfattr { + return nfAttr{}, fmt.Errorf("invalid netfilter attribute length %d", rawLen) + } + + payloadLen := int(rawLen) - nl.SizeofNfattr + payload := make([]byte, payloadLen) + if _, err := reader.Read(payload); err != nil { + return nfAttr{}, err + } + + padding := nlaAlignedLen(rawLen) - rawLen + if padding > 0 { + if _, err := reader.Seek(int64(padding), 1); err != nil { + return nfAttr{}, err + } + } + + return nfAttr{ + typ: rawType & nl.NLA_TYPE_MASK, + nested: rawType&nl.NLA_F_NESTED == nl.NLA_F_NESTED, + payload: payload, + }, nil +} + +func parseTuple(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + switch attr.typ { + case nl.CTA_TUPLE_IP: + if err := parseTupleIP(attr.payload, conn); err != nil { + return err + } + case nl.CTA_TUPLE_PROTO: + if err := parseTupleProto(attr.payload, conn); err != nil { + return err + } + } + } + return nil +} + +func parseTupleIP(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + switch attr.typ { + case nl.CTA_IP_V4_SRC: + addr, ok := addrFromIP(attr.payload) + if ok { + conn.OriginalSourceIP = addr + } + case nl.CTA_IP_V4_DST: + addr, ok := addrFromIP(attr.payload) + if ok { + conn.OriginalDestinationIP = addr + } + } + } + return nil +} + +func parseTupleProto(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + var protocol uint8 + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + switch attr.typ { + case nl.CTA_PROTO_NUM: + if len(attr.payload) > 0 { + protocol = attr.payload[0] + } + case nl.CTA_PROTO_SRC_PORT: + if len(attr.payload) >= 2 { + conn.OriginalSourcePort = binary.BigEndian.Uint16(attr.payload[:2]) + } + case nl.CTA_PROTO_DST_PORT: + if len(attr.payload) >= 2 { + conn.OriginalDestinationPort = binary.BigEndian.Uint16(attr.payload[:2]) + } + } + } + if protocol != unix.IPPROTO_TCP { + return fmt.Errorf("%w %d", errUnsupportedConntrackProtocol, protocol) + } + return nil +} + +func parseProtoInfo(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + if attr.typ != nl.CTA_PROTOINFO_TCP { + continue + } + + tcpReader := bytes.NewReader(attr.payload) + for tcpReader.Len() > 0 { + tcpAttr, err := readNfAttr(tcpReader) + if err != nil { + return err + } + if tcpAttr.typ == nl.CTA_PROTOINFO_TCP_STATE && len(tcpAttr.payload) > 0 { + conn.TCPState = TCPState(tcpAttr.payload[0]) + } + } + } + return nil +} + +func nlaAlignedLen(length uint16) uint16 { + return (length + nl.NLA_ALIGNTO - 1) & ^(nl.NLA_ALIGNTO - 1) +} diff --git a/lib/autostandby/conntrack_events_linux_test.go b/lib/autostandby/conntrack_events_linux_test.go new file mode 100644 index 00000000..935ddaf9 --- /dev/null +++ b/lib/autostandby/conntrack_events_linux_test.go @@ -0,0 +1,69 @@ +//go:build linux + +package autostandby + +import ( + "encoding/binary" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +func TestConnectionEventFromNetlinkMessageParsesIPv4TCPEvent(t *testing.T) { + t.Parallel() + + msg := syscall.NetlinkMessage{ + Header: syscall.NlMsghdr{Type: 0x100 | uint16(0)}, + Data: []byte{ + 2, 0, 0, 0, + 52, 0, 1, 128, + 20, 0, 1, 128, + 8, 0, 1, 0, 192, 168, 0, 10, + 8, 0, 2, 0, 192, 168, 77, 73, + 28, 0, 2, 128, + 5, 0, 1, 0, 6, 0, 0, 0, + 6, 0, 2, 0, 166, 129, 0, 0, + 6, 0, 3, 0, 13, 5, 0, 0, + 48, 0, 4, 128, + 44, 0, 1, 128, + 5, 0, 1, 0, 8, 0, 0, 0, + 5, 0, 2, 0, 0, 0, 0, 0, + 5, 0, 3, 0, 0, 0, 0, 0, + 6, 0, 4, 0, 39, 0, 0, 0, + 6, 0, 5, 0, 32, 0, 0, 0, + }, + } + + event, ok, err := connectionEventFromNetlinkMessage(msg) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, ConnectionEventNew, event.Type) + assert.Equal(t, mustAddr("192.168.0.10"), event.Connection.OriginalSourceIP) + assert.Equal(t, uint16(42625), event.Connection.OriginalSourcePort) + assert.Equal(t, mustAddr("192.168.77.73"), event.Connection.OriginalDestinationIP) + assert.Equal(t, uint16(3333), event.Connection.OriginalDestinationPort) + assert.Equal(t, TCPStateClose, event.Connection.TCPState) +} + +func TestConnectionEventFromNetlinkMessageParsesNativeEndianNLMSGError(t *testing.T) { + t.Parallel() + + data := make([]byte, 4) + nl.NativeEndian().PutUint32(data, uint32(int32(-unix.EPERM))) + + _, ok, err := connectionEventFromNetlinkMessage(syscall.NetlinkMessage{ + Header: syscall.NlMsghdr{Type: unix.NLMSG_ERROR}, + Data: data, + }) + require.ErrorIs(t, err, unix.EPERM) + require.False(t, ok) + + // Sanity-check the fixture is using native byte order rather than an accidental little-endian match. + if nl.NativeEndian() != binary.LittleEndian { + require.NotEqual(t, binary.LittleEndian.Uint32(data), nl.NativeEndian().Uint32(data)) + } +} diff --git a/lib/autostandby/conntrack_linux.go b/lib/autostandby/conntrack_linux.go new file mode 100644 index 00000000..d308d7b3 --- /dev/null +++ b/lib/autostandby/conntrack_linux.go @@ -0,0 +1,80 @@ +//go:build linux + +package autostandby + +import ( + "context" + "net/netip" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// ConntrackSource reads current IPv4 TCP conntrack entries from the host. +type ConntrackSource struct { + listFlows func(table netlink.ConntrackTableType, family netlink.InetFamily) ([]*netlink.ConntrackFlow, error) +} + +// NewConntrackSource creates a conntrack-backed connection source. +func NewConntrackSource() *ConntrackSource { + return &ConntrackSource{ + listFlows: netlink.ConntrackTableList, + } +} + +// ListConnections returns normalized TCP flows from the host conntrack table. +func (s *ConntrackSource) ListConnections(context.Context) ([]Connection, error) { + flows, err := s.listFlows(netlink.ConntrackTable, netlink.InetFamily(unix.AF_INET)) + if err != nil { + return nil, err + } + return connectionsFromFlows(flows), nil +} + +func connectionsFromFlows(flows []*netlink.ConntrackFlow) []Connection { + result := make([]Connection, 0, len(flows)) + for _, flow := range flows { + conn, ok := connectionFromFlow(flow) + if !ok { + continue + } + result = append(result, conn) + } + return result +} + +func connectionFromFlow(flow *netlink.ConntrackFlow) (Connection, bool) { + if flow == nil || flow.Forward.Protocol != unix.IPPROTO_TCP { + return Connection{}, false + } + + tcpInfo, ok := flow.ProtoInfo.(*netlink.ProtoInfoTCP) + if !ok || tcpInfo == nil { + return Connection{}, false + } + + srcIP, ok := addrFromIP(flow.Forward.SrcIP) + if !ok { + return Connection{}, false + } + dstIP, ok := addrFromIP(flow.Forward.DstIP) + if !ok { + return Connection{}, false + } + + return Connection{ + OriginalSourceIP: srcIP, + OriginalSourcePort: flow.Forward.SrcPort, + OriginalDestinationIP: dstIP, + OriginalDestinationPort: flow.Forward.DstPort, + TCPState: TCPState(tcpInfo.State), + }, true +} + +func addrFromIP(ip []byte) (netip.Addr, bool) { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Addr{}, false + } + return addr.Unmap(), true +} diff --git a/lib/autostandby/conntrack_linux_test.go b/lib/autostandby/conntrack_linux_test.go new file mode 100644 index 00000000..aecc2936 --- /dev/null +++ b/lib/autostandby/conntrack_linux_test.go @@ -0,0 +1,35 @@ +//go:build linux + +package autostandby + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +func TestConnectionFromFlowNormalizesTCPConntrackEntry(t *testing.T) { + t.Parallel() + + conn, ok := connectionFromFlow(&netlink.ConntrackFlow{ + Forward: netlink.IPTuple{ + Protocol: unix.IPPROTO_TCP, + SrcIP: net.ParseIP("1.2.3.4").To4(), + SrcPort: 12345, + DstIP: net.ParseIP("192.168.100.10").To4(), + DstPort: 8080, + }, + ProtoInfo: &netlink.ProtoInfoTCP{State: uint8(TCPStateEstablished)}, + }) + require.True(t, ok) + + assert.Equal(t, mustAddr("1.2.3.4"), conn.OriginalSourceIP) + assert.Equal(t, uint16(12345), conn.OriginalSourcePort) + assert.Equal(t, mustAddr("192.168.100.10"), conn.OriginalDestinationIP) + assert.Equal(t, uint16(8080), conn.OriginalDestinationPort) + assert.Equal(t, TCPStateEstablished, conn.TCPState) +} diff --git a/lib/autostandby/conntrack_unsupported.go b/lib/autostandby/conntrack_unsupported.go new file mode 100644 index 00000000..2cdbd38d --- /dev/null +++ b/lib/autostandby/conntrack_unsupported.go @@ -0,0 +1,26 @@ +//go:build !linux + +package autostandby + +import ( + "context" + "fmt" +) + +// ConntrackSource is unavailable on non-Linux platforms. +type ConntrackSource struct{} + +// NewConntrackSource creates an unsupported conntrack source. +func NewConntrackSource() *ConntrackSource { + return &ConntrackSource{} +} + +// ListConnections reports that conntrack-backed auto-standby is unsupported. +func (*ConntrackSource) ListConnections(context.Context) ([]Connection, error) { + return nil, fmt.Errorf("conntrack-backed auto-standby is only supported on Linux") +} + +// OpenStream reports that conntrack-backed auto-standby is unsupported. +func (*ConntrackSource) OpenStream(context.Context) (ConnectionStream, error) { + return nil, fmt.Errorf("conntrack-backed auto-standby is only supported on Linux") +} diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go new file mode 100644 index 00000000..6b47dda0 --- /dev/null +++ b/lib/autostandby/controller.go @@ -0,0 +1,1005 @@ +package autostandby + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/netip" + "sync" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +const ( + trackingModeConntrackEventsV4TCP = "conntrack_events_v4_tcp" + defaultReconnectDelay = 2 * time.Second + defaultReconcileDelay = 2 * time.Second + defaultSnapshotSyncInterval = 5 * time.Minute +) + +// InstanceEventAction identifies an instance lifecycle change relevant to auto-standby. +type InstanceEventAction string + +const ( + InstanceEventCreate InstanceEventAction = "create" + InstanceEventUpdate InstanceEventAction = "update" + InstanceEventStart InstanceEventAction = "start" + InstanceEventStop InstanceEventAction = "stop" + InstanceEventStandby InstanceEventAction = "standby" + InstanceEventRestore InstanceEventAction = "restore" + InstanceEventDelete InstanceEventAction = "delete" + InstanceEventFork InstanceEventAction = "fork" +) + +// InstanceEvent carries an instance lifecycle update into the controller. +type InstanceEvent struct { + Action InstanceEventAction + InstanceID string + Instance *Instance +} + +// InstanceStore supplies the controller with instance state, lifecycle events, +// runtime persistence, and standby actions. +type InstanceStore interface { + ListInstances(ctx context.Context) ([]Instance, error) + StandbyInstance(ctx context.Context, id string) error + SetRuntime(ctx context.Context, id string, runtime *Runtime) error + SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) +} + +type ConnectionEventType string + +const ( + ConnectionEventNew ConnectionEventType = "new" + ConnectionEventDestroy ConnectionEventType = "destroy" +) + +// ConnectionEvent is a single conntrack event delivered from the host observer. +type ConnectionEvent struct { + Type ConnectionEventType + Connection Connection + ObservedAt time.Time +} + +// ConnectionStream is a live conntrack event stream. +type ConnectionStream interface { + Events() <-chan ConnectionEvent + Errors() <-chan error + Close() error +} + +// ConnectionSource provides startup snapshots and live conntrack events. +type ConnectionSource interface { + ListConnections(ctx context.Context) ([]Connection, error) + OpenStream(ctx context.Context) (ConnectionStream, error) +} + +// ControllerOptions configures logging, timing, and observability. +type ControllerOptions struct { + Log *slog.Logger + Meter metric.Meter + Tracer trace.Tracer + Now func() time.Time + ReconnectDelay time.Duration + ReconcileDelay time.Duration + SnapshotSyncInterval time.Duration +} + +type controllerState struct { + instance Instance + compiledPolicy *compiledPolicy + activeInbound map[ConnectionKey]struct{} + idleTimeout time.Duration + idleSince *time.Time + lastInboundAt *time.Time + nextStandbyAt *time.Time + timer *time.Timer + reconcileTimer *time.Timer + standbyRequested bool +} + +// Controller decides when eligible instances should transition to standby. +type Controller struct { + store InstanceStore + source ConnectionSource + log *slog.Logger + now func() time.Time + tracer trace.Tracer + metrics *Metrics + + reconnectDelay time.Duration + reconcileDelay time.Duration + snapshotSyncInterval time.Duration + timerFired chan string + reconcileFired chan string + streamReady chan ConnectionStream + + mu sync.RWMutex + states map[string]*controllerState + observerConnected bool + lastObserverErr error +} + +// NewController creates a new event-driven auto-standby controller. +func NewController(store InstanceStore, source ConnectionSource, opts ControllerOptions) *Controller { + log := opts.Log + if log == nil { + log = slog.Default() + } + now := opts.Now + if now == nil { + now = time.Now + } + reconnectDelay := opts.ReconnectDelay + if reconnectDelay <= 0 { + reconnectDelay = defaultReconnectDelay + } + reconcileDelay := opts.ReconcileDelay + if reconcileDelay <= 0 { + reconcileDelay = defaultReconcileDelay + } + snapshotSyncInterval := opts.SnapshotSyncInterval + if snapshotSyncInterval <= 0 { + snapshotSyncInterval = defaultSnapshotSyncInterval + } + + c := &Controller{ + store: store, + source: source, + log: log, + now: now, + tracer: opts.Tracer, + reconnectDelay: reconnectDelay, + reconcileDelay: reconcileDelay, + snapshotSyncInterval: snapshotSyncInterval, + timerFired: make(chan string, 128), + reconcileFired: make(chan string, 128), + streamReady: make(chan ConnectionStream, 4), + states: make(map[string]*controllerState), + } + c.metrics = newMetrics(opts.Meter, opts.Tracer, c) + return c +} + +// Run starts the controller and blocks until ctx is cancelled. +func (c *Controller) Run(ctx context.Context) error { + log := c.log.With("tracking_mode", trackingModeConntrackEventsV4TCP) + log.Info("auto-standby controller started", "snapshot_sync_interval", c.snapshotSyncInterval) + + var stream ConnectionStream + if c.source != nil { + initialStream, err := c.source.OpenStream(ctx) + if err != nil { + c.setObserverError(err) + c.recordObserverError("connect") + log.Warn("auto-standby conntrack subscription failed", "error", err) + go c.reconnectStream(ctx) + } else { + stream = initialStream + c.setObserverConnected(true) + } + } + + if err := c.startupResync(ctx); err != nil { + return err + } + + instanceEvents, unsubscribe, err := c.store.SubscribeInstanceEvents() + if err != nil { + return err + } + defer unsubscribe() + defer c.stopAllTimers() + if stream != nil { + defer stream.Close() + } + snapshotTicker := time.NewTicker(c.snapshotSyncInterval) + defer snapshotTicker.Stop() + + for { + var streamEvents <-chan ConnectionEvent + var streamErrors <-chan error + if stream != nil { + streamEvents = stream.Events() + streamErrors = stream.Errors() + } + + select { + case <-ctx.Done(): + return nil + case replacement := <-c.streamReady: + if stream != nil { + _ = stream.Close() + } + stream = replacement + c.setObserverConnected(true) + log.Info("auto-standby conntrack subscription restored") + case err := <-streamErrors: + if err == nil { + continue + } + c.setObserverError(err) + c.recordObserverError("stream") + log.Warn("auto-standby conntrack subscription failed", "error", err) + if stream != nil { + _ = stream.Close() + stream = nil + } + go c.reconnectStream(ctx) + case event, ok := <-streamEvents: + if !ok { + if stream != nil { + _ = stream.Close() + stream = nil + } + c.setObserverError(errors.New("conntrack stream closed")) + c.recordObserverError("stream_closed") + go c.reconnectStream(ctx) + continue + } + c.handleConnectionEvent(ctx, event) + case event, ok := <-instanceEvents: + if !ok { + return nil + } + if err := c.handleInstanceEvent(ctx, event); err != nil { + c.recordObserverError("instance_event") + log.Warn("auto-standby instance event handling failed", "action", event.Action, "instance_id", event.InstanceID, "error", err) + } + case id := <-c.timerFired: + c.handleStandbyTimer(ctx, id) + case id := <-c.reconcileFired: + c.handleActiveReconcile(ctx, id) + case <-snapshotTicker.C: + if err := c.periodicSnapshotSync(ctx); err != nil { + c.recordControllerError("snapshot_sync") + log.Warn("auto-standby periodic snapshot sync failed", "error", err) + } + } + } +} + +// Describe returns the current diagnostic view for an instance. +func (c *Controller) Describe(inst Instance) StatusSnapshot { + snapshot := StatusSnapshot{ + Supported: c != nil, + Configured: inst.AutoStandby != nil, + Enabled: inst.AutoStandby != nil && inst.AutoStandby.Enabled, + TrackingMode: trackingModeConntrackEventsV4TCP, + } + if c == nil { + snapshot.Status = StatusUnsupported + snapshot.Reason = ReasonUnsupportedPlatform + return snapshot + } + if inst.AutoStandby != nil { + snapshot.IdleTimeout = inst.AutoStandby.IdleTimeout + } + + if inst.AutoStandby == nil { + snapshot.Status = StatusDisabled + snapshot.Reason = ReasonPolicyMissing + return snapshot + } + if !inst.AutoStandby.Enabled { + snapshot.Status = StatusDisabled + snapshot.Reason = ReasonPolicyDisabled + return snapshot + } + if inst.State != StateRunning { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonInstanceNotRunning + return snapshot + } + if !inst.NetworkEnabled { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonNetworkDisabled + return snapshot + } + if inst.IP == "" { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonMissingIP + return snapshot + } + if inst.HasVGPU { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonHasVGPU + return snapshot + } + + snapshot.Eligible = true + + var ( + activeInboundCount int + idleSince *time.Time + lastInboundAt *time.Time + nextStandbyAt *time.Time + standbyRequested bool + hasState bool + ) + + c.mu.RLock() + state := c.states[inst.ID] + observerConnected := c.observerConnected + lastObserverErr := c.lastObserverErr + if state != nil { + hasState = true + activeInboundCount = len(state.activeInbound) + idleSince = cloneTimePtr(state.idleSince) + lastInboundAt = cloneTimePtr(state.lastInboundAt) + nextStandbyAt = cloneTimePtr(state.nextStandbyAt) + standbyRequested = state.standbyRequested + } + c.mu.RUnlock() + + if hasState { + snapshot.ActiveInboundCount = activeInboundCount + snapshot.IdleSince = idleSince + snapshot.LastInboundActivityAt = lastInboundAt + snapshot.NextStandbyAt = nextStandbyAt + if nextStandbyAt != nil { + remaining := nextStandbyAt.Sub(c.now().UTC()) + if remaining < 0 { + remaining = 0 + } + snapshot.CountdownRemaining = &remaining + } + if standbyRequested { + snapshot.Status = StatusStandbyRequested + snapshot.Reason = ReasonReadyForStandby + return snapshot + } + } + + if !observerConnected && lastObserverErr != nil { + snapshot.Status = StatusError + snapshot.Reason = ReasonObserverError + return snapshot + } + if hasState && activeInboundCount > 0 { + snapshot.Status = StatusActive + snapshot.Reason = ReasonActiveInbound + return snapshot + } + if hasState && nextStandbyAt != nil { + if nextStandbyAt.After(c.now().UTC()) { + snapshot.Status = StatusIdleCountdown + snapshot.Reason = ReasonIdleTimeoutNotElapsed + return snapshot + } + snapshot.Status = StatusReadyForStandby + snapshot.Reason = ReasonReadyForStandby + return snapshot + } + + snapshot.Status = StatusIdleCountdown + snapshot.Reason = ReasonIdleTimeoutNotElapsed + return snapshot +} + +func (c *Controller) startupResync(ctx context.Context) error { + start := c.now() + ctx, span := c.startSpan(ctx, "AutoStandbyStartupResync") + defer func() { + if span != nil { + span.End() + } + }() + + instances, err := c.store.ListInstances(ctx) + if err != nil { + c.recordStartupResync(start, "error") + recordSpanError(span, err) + return err + } + conns, err := c.source.ListConnections(ctx) + if err != nil { + c.recordStartupResync(start, "error") + recordSpanError(span, err) + return err + } + + now := c.now().UTC() + c.log.Info("auto-standby startup resync seeded state", "instance_count", len(instances), "current_connection_count", len(conns)) + for _, inst := range instances { + if err := c.seedInstanceState(ctx, inst, conns, now); err != nil { + c.log.Warn("auto-standby startup resync failed for instance", "instance_id", inst.ID, "instance_name", inst.Name, "error", err) + } + } + + c.recordStartupResync(start, "success") + return nil +} + +func (c *Controller) periodicSnapshotSync(ctx context.Context) error { + instances, err := c.store.ListInstances(ctx) + if err != nil { + return err + } + conns, err := c.source.ListConnections(ctx) + if err != nil { + return err + } + + now := c.now().UTC() + c.log.Debug("auto-standby periodic snapshot sync completed", "instance_count", len(instances), "current_connection_count", len(conns)) + for _, inst := range instances { + if err := c.seedInstanceState(ctx, inst, conns, now); err != nil { + c.log.Warn("auto-standby periodic snapshot sync failed for instance", "instance_id", inst.ID, "instance_name", inst.Name, "error", err) + } + } + return nil +} + +func (c *Controller) seedInstanceState(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.refreshInstanceLocked(ctx, inst, conns, now) +} + +func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEvent) error { + if event.Action == InstanceEventDelete { + c.mu.Lock() + defer c.mu.Unlock() + c.removeStateLocked(event.InstanceID) + return nil + } + if event.Instance == nil { + return nil + } + + conns, err := c.source.ListConnections(ctx) + if err != nil { + return err + } + + c.mu.Lock() + defer c.mu.Unlock() + return c.refreshInstanceLocked(ctx, *event.Instance, conns, c.now().UTC()) +} + +func (c *Controller) refreshInstanceLocked(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { + state := c.ensureStateLocked(inst.ID) + state.instance = cloneInstance(inst) + state.standbyRequested = false + + if !eligible(inst) { + c.clearStateLocked(state) + if inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil { + return c.persistRuntime(ctx, inst.ID, nil) + } + return nil + } + + compiled, err := compilePolicy(inst.AutoStandby) + if err != nil { + return err + } + state.compiledPolicy = compiled + state.idleTimeout = compiled.idleTimeout + + activeSet, err := matchingConnections(inst, compiled, conns) + if err != nil { + return err + } + state.activeInbound = activeSet + + runtime := cloneRuntime(inst.Runtime) + if len(activeSet) > 0 { + state.idleSince = nil + if runtime != nil && runtime.LastInboundActivityAt != nil { + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + } else { + state.lastInboundAt = &now + } + c.cancelTimerLocked(state) + c.armReconcileLocked(inst.ID, state) + return c.persistRuntime(ctx, inst.ID, &Runtime{ + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }) + } + + if runtime != nil && runtime.IdleSince != nil { + state.idleSince = cloneTimePtr(runtime.IdleSince) + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + } else { + state.idleSince = &now + if runtime != nil { + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + } else { + state.lastInboundAt = nil + } + runtime = &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + } + if err := c.persistRuntime(ctx, inst.ID, runtime); err != nil { + return err + } + } + c.armTimerLocked(inst.ID, state, now) + return nil +} + +func (c *Controller) handleConnectionEvent(ctx context.Context, event ConnectionEvent) { + ctx, span := c.startSpan(ctx, "AutoStandbyHandleConntrackEvent", + attribute.String("event", string(event.Type)), + ) + defer func() { + if span != nil { + span.End() + } + }() + + if event.ObservedAt.IsZero() { + event.ObservedAt = c.now().UTC() + } + c.recordConntrackEvent(string(event.Type), "received") + + c.mu.Lock() + defer c.mu.Unlock() + + for id, state := range c.states { + if state.compiledPolicy == nil { + continue + } + key := connectionKey(event.Connection) + matches := matchesInboundConnectionForEvent(state.instance, state.compiledPolicy, event.Connection) + switch event.Type { + case ConnectionEventNew: + if !matches { + continue + } + if !event.Connection.TCPState.Active() { + if _, ok := state.activeInbound[key]; !ok { + continue + } + delete(state.activeInbound, key) + if len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + continue + } + idleSince := event.ObservedAt.UTC() + state.idleSince = &idleSince + state.standbyRequested = false + c.cancelReconcileLocked(state) + c.armTimerLocked(id, state, idleSince) + if err := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime when idle countdown started", "instance_id", id, "error", err) + } + c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) + continue + } + if state.activeInbound == nil { + state.activeInbound = make(map[ConnectionKey]struct{}) + } + state.activeInbound[key] = struct{}{} + state.idleSince = nil + state.lastInboundAt = cloneTimePtr(&event.ObservedAt) + state.standbyRequested = false + c.cancelTimerLocked(state) + c.armReconcileLocked(id, state) + if err := c.persistRuntime(ctx, id, &Runtime{ + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after inbound activity", "instance_id", id, "error", err) + } + c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) + case ConnectionEventDestroy: + if _, ok := state.activeInbound[key]; !ok { + continue + } + delete(state.activeInbound, key) + if len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + continue + } + idleSince := event.ObservedAt.UTC() + state.idleSince = &idleSince + state.standbyRequested = false + c.cancelReconcileLocked(state) + c.armTimerLocked(id, state, idleSince) + if err := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime when idle countdown started", "instance_id", id, "error", err) + } + c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) + } + } +} + +func (c *Controller) handleStandbyTimer(ctx context.Context, id string) { + ctx, span := c.startSpan(ctx, "AutoStandbyStandbyAttempt", + attribute.String("instance_id", id), + ) + defer func() { + if span != nil { + span.End() + } + }() + + c.mu.Lock() + state := c.states[id] + if state == nil || state.compiledPolicy == nil || len(state.activeInbound) > 0 { + c.mu.Unlock() + return + } + state.timer = nil + state.nextStandbyAt = nil + state.standbyRequested = true + instanceName := state.instance.Name + idleTimeout := state.idleTimeout + c.mu.Unlock() + + c.log.Info("auto-standby standby timer fired", "instance_id", id, "instance_name", instanceName) + + if err := c.store.StandbyInstance(ctx, id); err != nil { + recordSpanError(span, err) + c.recordStandbyAttempt("error") + c.recordControllerError("standby") + c.log.Warn("auto-standby standby attempt failed", "instance_id", id, "instance_name", instanceName, "error", err) + + c.mu.Lock() + defer c.mu.Unlock() + if state := c.states[id]; state != nil { + state.standbyRequested = false + idleSince := c.now().UTC() + state.idleSince = &idleSince + c.armTimerLocked(id, state, idleSince) + if persistErr := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); persistErr != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after standby failure", "instance_id", id, "error", persistErr) + } + } + return + } + + c.recordStandbyAttempt("success") + c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) + + c.mu.Lock() + defer c.mu.Unlock() + if state := c.states[id]; state != nil { + c.clearStateLocked(state) + if err := c.persistRuntime(ctx, id, nil); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) + } + } +} + +func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { + conns, err := c.source.ListConnections(ctx) + if err != nil { + c.recordControllerError("reconcile") + c.log.Warn("auto-standby active connection reconcile failed", "instance_id", id, "error", err) + + c.mu.Lock() + defer c.mu.Unlock() + if state := c.states[id]; state != nil && len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + } + return + } + + now := c.now().UTC() + + c.mu.Lock() + defer c.mu.Unlock() + + state := c.states[id] + if state == nil || state.compiledPolicy == nil { + return + } + + activeSet, err := matchingConnections(state.instance, state.compiledPolicy, conns) + if err != nil { + c.recordControllerError("reconcile") + c.log.Warn("auto-standby active connection reconcile failed to classify connections", "instance_id", id, "error", err) + if len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + } + return + } + + state.activeInbound = activeSet + if len(activeSet) > 0 { + c.armReconcileLocked(id, state) + return + } + + state.idleSince = &now + state.standbyRequested = false + c.cancelReconcileLocked(state) + c.armTimerLocked(id, state, now) + if err := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after active connection reconcile drained", "instance_id", id, "error", err) + } + c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", state.idleTimeout) +} + +func (c *Controller) reconnectStream(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + default: + } + + stream, err := c.source.OpenStream(ctx) + if err == nil { + select { + case c.streamReady <- stream: + case <-ctx.Done(): + _ = stream.Close() + } + return + } + + c.setObserverError(err) + c.recordObserverError("reconnect") + c.log.Warn("auto-standby conntrack subscription reconnect failed", "error", err) + + timer := time.NewTimer(c.reconnectDelay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } +} + +func (c *Controller) ensureStateLocked(id string) *controllerState { + state, ok := c.states[id] + if !ok { + state = &controllerState{} + c.states[id] = state + } + return state +} + +func (c *Controller) removeStateLocked(id string) { + state := c.states[id] + if state != nil { + c.cancelTimerLocked(state) + c.cancelReconcileLocked(state) + } + delete(c.states, id) +} + +func (c *Controller) clearStateLocked(state *controllerState) { + state.compiledPolicy = nil + state.activeInbound = nil + state.idleTimeout = 0 + state.idleSince = nil + state.lastInboundAt = nil + state.nextStandbyAt = nil + state.standbyRequested = false + c.cancelTimerLocked(state) + c.cancelReconcileLocked(state) +} + +func (c *Controller) armTimerLocked(id string, state *controllerState, now time.Time) { + if state.idleSince == nil || state.idleTimeout <= 0 { + c.cancelTimerLocked(state) + return + } + + when := state.idleSince.Add(state.idleTimeout) + delay := when.Sub(now) + if delay < 0 { + delay = 0 + } + + c.cancelTimerLocked(state) + state.nextStandbyAt = &when + c.log.Debug("auto-standby standby timer armed", "instance_id", id, "next_standby_at", when, "idle_timeout", state.idleTimeout) + state.timer = time.AfterFunc(delay, func() { + select { + case c.timerFired <- id: + default: + } + }) +} + +func (c *Controller) cancelTimerLocked(state *controllerState) { + if state.timer != nil { + state.timer.Stop() + c.log.Debug("auto-standby standby timer cancelled", "instance_id", state.instance.ID) + state.timer = nil + } + state.nextStandbyAt = nil +} + +func (c *Controller) armReconcileLocked(id string, state *controllerState) { + if len(state.activeInbound) == 0 || c.reconcileDelay <= 0 { + c.cancelReconcileLocked(state) + return + } + + c.cancelReconcileLocked(state) + state.reconcileTimer = time.AfterFunc(c.reconcileDelay, func() { + select { + case c.reconcileFired <- id: + default: + } + }) +} + +func (c *Controller) cancelReconcileLocked(state *controllerState) { + if state.reconcileTimer != nil { + state.reconcileTimer.Stop() + state.reconcileTimer = nil + } +} + +func (c *Controller) stopAllTimers() { + c.mu.Lock() + defer c.mu.Unlock() + for _, state := range c.states { + c.cancelTimerLocked(state) + c.cancelReconcileLocked(state) + } +} + +func (c *Controller) persistRuntime(ctx context.Context, id string, runtime *Runtime) error { + return c.store.SetRuntime(ctx, id, runtime) +} + +func (c *Controller) setObserverConnected(connected bool) { + c.mu.Lock() + defer c.mu.Unlock() + c.observerConnected = connected + if connected { + c.lastObserverErr = nil + } +} + +func (c *Controller) setObserverError(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.observerConnected = false + c.lastObserverErr = err +} + +func matchingConnections(inst Instance, compiled *compiledPolicy, conns []Connection) (map[ConnectionKey]struct{}, error) { + instanceIP, err := netip.ParseAddr(inst.IP) + if err != nil { + return nil, fmt.Errorf("parse instance IP %q: %w", inst.IP, err) + } + + out := make(map[ConnectionKey]struct{}) + for _, conn := range conns { + if matchesInboundConnection(instanceIP, compiled, conn) { + out[connectionKey(conn)] = struct{}{} + } + } + return out, nil +} + +func matchesInboundConnectionForEvent(inst Instance, policy *compiledPolicy, conn Connection) bool { + instanceIP, err := netip.ParseAddr(inst.IP) + if err != nil { + return false + } + if !conn.OriginalDestinationIP.IsValid() || conn.OriginalDestinationIP != instanceIP { + return false + } + if _, ignored := policy.ignorePorts[conn.OriginalDestinationPort]; ignored { + return false + } + if !conn.OriginalSourceIP.IsValid() { + return false + } + for _, prefix := range policy.ignoreSourceCIDRs { + if prefix.Contains(conn.OriginalSourceIP) { + return false + } + } + return true +} + +type ConnectionKey struct { + OriginalSourceIP netip.Addr + OriginalSourcePort uint16 + OriginalDestinationIP netip.Addr + OriginalDestinationPort uint16 +} + +func connectionKey(conn Connection) ConnectionKey { + return ConnectionKey{ + OriginalSourceIP: conn.OriginalSourceIP, + OriginalSourcePort: conn.OriginalSourcePort, + OriginalDestinationIP: conn.OriginalDestinationIP, + OriginalDestinationPort: conn.OriginalDestinationPort, + } +} + +func cloneRuntime(runtime *Runtime) *Runtime { + if runtime == nil { + return nil + } + return &Runtime{ + IdleSince: cloneTimePtr(runtime.IdleSince), + LastInboundActivityAt: cloneTimePtr(runtime.LastInboundActivityAt), + } +} + +func cloneInstance(inst Instance) Instance { + cloned := inst + cloned.AutoStandby = clonePolicy(inst.AutoStandby) + cloned.Runtime = cloneRuntime(inst.Runtime) + return cloned +} + +func clonePolicy(policy *Policy) *Policy { + if policy == nil { + return nil + } + out := &Policy{ + Enabled: policy.Enabled, + IdleTimeout: policy.IdleTimeout, + } + if len(policy.IgnoreSourceCIDRs) > 0 { + out.IgnoreSourceCIDRs = append([]string(nil), policy.IgnoreSourceCIDRs...) + } + if len(policy.IgnoreDestinationPorts) > 0 { + out.IgnoreDestinationPorts = append([]uint16(nil), policy.IgnoreDestinationPorts...) + } + return out +} + +func cloneTimePtr(t *time.Time) *time.Time { + if t == nil { + return nil + } + copied := t.UTC() + return &copied +} + +func eligible(inst Instance) bool { + if inst.State != StateRunning { + return false + } + if !inst.NetworkEnabled || inst.IP == "" || inst.HasVGPU { + return false + } + return inst.AutoStandby != nil && inst.AutoStandby.Enabled +} + +func (c *Controller) startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + if c.tracer == nil { + return ctx, nil + } + return c.tracer.Start(ctx, name, trace.WithAttributes(attrs...)) +} + +func recordSpanError(span trace.Span, err error) { + if span == nil || err == nil { + return + } + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go new file mode 100644 index 00000000..7bb50823 --- /dev/null +++ b/lib/autostandby/controller_test.go @@ -0,0 +1,369 @@ +package autostandby + +import ( + "context" + "errors" + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeInstanceStore struct { + instances []Instance + standbyIDs []string + persistedRuntime map[string]*Runtime + events chan InstanceEvent + standbyErr error +} + +func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { + return &fakeInstanceStore{ + instances: append([]Instance(nil), instances...), + persistedRuntime: make(map[string]*Runtime), + events: make(chan InstanceEvent, 16), + } +} + +func (f *fakeInstanceStore) ListInstances(context.Context) ([]Instance, error) { + out := make([]Instance, 0, len(f.instances)) + for _, inst := range f.instances { + out = append(out, cloneInstance(inst)) + } + return out, nil +} + +func (f *fakeInstanceStore) StandbyInstance(_ context.Context, id string) error { + f.standbyIDs = append(f.standbyIDs, id) + return f.standbyErr +} + +func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { + f.persistedRuntime[id] = cloneRuntime(runtime) + for i := range f.instances { + if f.instances[i].ID == id { + f.instances[i].Runtime = cloneRuntime(runtime) + } + } + return nil +} + +func (f *fakeInstanceStore) SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) { + return f.events, func() {}, nil +} + +type fakeConnectionSource struct { + connections []Connection +} + +func (f *fakeConnectionSource) ListConnections(context.Context) ([]Connection, error) { + return append([]Connection(nil), f.connections...), nil +} + +func (f *fakeConnectionSource) OpenStream(context.Context) (ConnectionStream, error) { + return &fakeConnectionStream{ + events: make(chan ConnectionEvent), + errs: make(chan error), + }, nil +} + +type fakeConnectionStream struct { + events chan ConnectionEvent + errs chan error +} + +func (f *fakeConnectionStream) Events() <-chan ConnectionEvent { return f.events } + +func (f *fakeConnectionStream) Errors() <-chan error { return f.errs } + +func (f *fakeConnectionStream) Close() error { + select { + case <-f.events: + default: + } + return nil +} + +func TestStartupResyncClearsPersistedIdleWhenCurrentConnectionsExist(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 0, 0, 0, time.UTC) + lastInbound := idleSince.Add(-time.Minute) + store := newFakeInstanceStore([]Instance{{ + ID: "inst-active", + Name: "inst-active", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.10", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "5m"}, + Runtime: &Runtime{ + IdleSince: &idleSince, + LastInboundActivityAt: &lastInbound, + }, + }}) + source := &fakeConnectionSource{connections: []Connection{{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 51234, + OriginalDestinationIP: mustAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }}} + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, source, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + require.Nil(t, status.IdleSince) + require.NotNil(t, store.persistedRuntime["inst-active"]) + require.Nil(t, store.persistedRuntime["inst-active"].IdleSince) +} + +func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{{ + ID: "inst-idle", + Name: "inst-idle", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, + Runtime: &Runtime{ + IdleSince: &idleSince, + }, + }}) + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.NextStandbyAt) + assert.Equal(t, idleSince.Add(10*time.Minute), *status.NextStandbyAt) +} + +func TestPeriodicSnapshotSyncRefreshesTrackedState(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-periodic", + Name: "inst-periodic", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.21", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, + }}) + source := &fakeConnectionSource{} + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, source, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + + source.connections = []Connection{{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 51235, + OriginalDestinationIP: mustAddr("192.168.100.21"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }} + now = now.Add(time.Minute) + + require.NoError(t, controller.periodicSnapshotSync(context.Background())) + + status = controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + require.Equal(t, 1, status.ActiveInboundCount) +} + +func TestConnectionEventsClearIdleAndStartCountdown(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-1", + Name: "inst-1", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.30", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + require.NoError(t, controller.startupResync(context.Background())) + + newEvent := ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50000, + OriginalDestinationIP: mustAddr("192.168.100.30"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: now.Add(5 * time.Second), + } + controller.handleConnectionEvent(context.Background(), newEvent) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + require.Nil(t, status.IdleSince) + + destroyEvent := newEvent + destroyEvent.Type = ConnectionEventDestroy + destroyEvent.ObservedAt = now.Add(10 * time.Second) + controller.handleConnectionEvent(context.Background(), destroyEvent) + + status = controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.IdleSince) +} + +func TestConnectionUpdateWithInactiveTCPStateStartsCountdown(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-update", + Name: "inst-update", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.31", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + require.NoError(t, controller.startupResync(context.Background())) + + event := ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50001, + OriginalDestinationIP: mustAddr("192.168.100.31"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: now.Add(5 * time.Second), + } + controller.handleConnectionEvent(context.Background(), event) + + event.ObservedAt = now.Add(10 * time.Second) + event.Connection.TCPState = TCPStateTimeWait + controller.handleConnectionEvent(context.Background(), event) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.IdleSince) +} + +func TestActiveReconcileStartsCountdownForStartupSeededConnections(t *testing.T) { + t.Parallel() + + idleTimeout := 30 * time.Second + store := newFakeInstanceStore([]Instance{{ + ID: "inst-reconcile", + Name: "inst-reconcile", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.32", + AutoStandby: &Policy{Enabled: true, IdleTimeout: idleTimeout.String()}, + }}) + source := &fakeConnectionSource{connections: []Connection{{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50002, + OriginalDestinationIP: mustAddr("192.168.100.32"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }}} + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, source, ControllerOptions{ + Now: func() time.Time { return now }, + ReconcileDelay: time.Second, + }) + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + + source.connections = nil + now = now.Add(5 * time.Second) + controller.handleActiveReconcile(context.Background(), "inst-reconcile") + + status = controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.IdleSince) + require.NotNil(t, status.NextStandbyAt) +} + +func TestDuplicateDestroyDoesNotGoNegative(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-dup", + Name: "inst-dup", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.40", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: time.Now, + }) + require.NoError(t, controller.startupResync(context.Background())) + + event := ConnectionEvent{ + Type: ConnectionEventDestroy, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50000, + OriginalDestinationIP: mustAddr("192.168.100.40"), + OriginalDestinationPort: 8080, + }, + ObservedAt: time.Now().UTC(), + } + controller.handleConnectionEvent(context.Background(), event) + controller.handleConnectionEvent(context.Background(), event) + + status := controller.Describe(store.instances[0]) + require.Equal(t, 0, status.ActiveInboundCount) +} + +func TestStatusReportsObserverError(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-err", + Name: "inst-err", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.50", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) + controller.setObserverError(errors.New("boom")) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusError, status.Status) + require.Equal(t, ReasonObserverError, status.Reason) +} + +func mustAddr(raw string) netip.Addr { + return netip.MustParseAddr(raw) +} diff --git a/lib/autostandby/metrics.go b/lib/autostandby/metrics.go new file mode 100644 index 00000000..9d29a76f --- /dev/null +++ b/lib/autostandby/metrics.go @@ -0,0 +1,163 @@ +package autostandby + +import ( + "context" + "time" + + hypotel "github.com/kernel/hypeman/lib/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +type Metrics struct { + conntrackEventsTotal metric.Int64Counter + startupResyncDuration metric.Float64Histogram + standbyAttemptsTotal metric.Int64Counter + controllerErrorsTotal metric.Int64Counter + trackedInstancesGauge metric.Int64ObservableGauge + activeConnectionsGauge metric.Int64ObservableGauge + tracer trace.Tracer +} + +func newMetrics(meter metric.Meter, tracer trace.Tracer, controller *Controller) *Metrics { + if meter == nil { + return &Metrics{tracer: tracer} + } + + conntrackEventsTotal, err := meter.Int64Counter( + "hypeman_auto_standby_conntrack_events_total", + metric.WithDescription("Total conntrack events processed by the auto-standby controller"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + startupResyncDuration, err := meter.Float64Histogram( + "hypeman_auto_standby_startup_resync_duration_seconds", + metric.WithDescription("Time spent rebuilding auto-standby state during controller startup"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(hypotel.CommonDurationHistogramBuckets()...), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + standbyAttemptsTotal, err := meter.Int64Counter( + "hypeman_auto_standby_standby_attempts_total", + metric.WithDescription("Total standby attempts issued by the auto-standby controller"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + controllerErrorsTotal, err := meter.Int64Counter( + "hypeman_auto_standby_controller_errors_total", + metric.WithDescription("Total controller and observer errors encountered by auto-standby"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + trackedInstancesGauge, err := meter.Int64ObservableGauge( + "hypeman_auto_standby_tracked_instances_total", + metric.WithDescription("Tracked instances by auto-standby controller phase"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + activeConnectionsGauge, err := meter.Int64ObservableGauge( + "hypeman_auto_standby_active_connections_total", + metric.WithDescription("Current number of active inbound connections tracked by auto-standby"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + + m := &Metrics{ + conntrackEventsTotal: conntrackEventsTotal, + startupResyncDuration: startupResyncDuration, + standbyAttemptsTotal: standbyAttemptsTotal, + controllerErrorsTotal: controllerErrorsTotal, + trackedInstancesGauge: trackedInstancesGauge, + activeConnectionsGauge: activeConnectionsGauge, + tracer: tracer, + } + + _, _ = meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error { + if controller == nil { + return nil + } + active, countdown, ready, ineligible, totalConnections := controller.metricSnapshot() + observer.ObserveInt64(m.trackedInstancesGauge, int64(active), metric.WithAttributes(attribute.String("phase", "active"))) + observer.ObserveInt64(m.trackedInstancesGauge, int64(countdown), metric.WithAttributes(attribute.String("phase", "idle_countdown"))) + observer.ObserveInt64(m.trackedInstancesGauge, int64(ready), metric.WithAttributes(attribute.String("phase", "ready_for_standby"))) + observer.ObserveInt64(m.trackedInstancesGauge, int64(ineligible), metric.WithAttributes(attribute.String("phase", "ineligible"))) + observer.ObserveInt64(m.activeConnectionsGauge, int64(totalConnections)) + return nil + }, trackedInstancesGauge, activeConnectionsGauge) + + return m +} + +func (c *Controller) recordConntrackEvent(event, result string) { + if c.metrics == nil || c.metrics.conntrackEventsTotal == nil { + return + } + c.metrics.conntrackEventsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("event", event), + attribute.String("result", result), + )) +} + +func (c *Controller) recordStartupResync(start time.Time, status string) { + if c.metrics == nil || c.metrics.startupResyncDuration == nil { + return + } + c.metrics.startupResyncDuration.Record(context.Background(), time.Since(start).Seconds(), metric.WithAttributes( + attribute.String("status", status), + )) +} + +func (c *Controller) recordStandbyAttempt(status string) { + if c.metrics == nil || c.metrics.standbyAttemptsTotal == nil { + return + } + c.metrics.standbyAttemptsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("status", status), + )) +} + +func (c *Controller) recordControllerError(operation string) { + if c.metrics == nil || c.metrics.controllerErrorsTotal == nil { + return + } + c.metrics.controllerErrorsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("operation", operation), + )) +} + +func (c *Controller) recordObserverError(operation string) { + c.recordControllerError(operation) +} + +func (c *Controller) metricSnapshot() (active, countdown, ready, ineligible, totalConnections int) { + c.mu.RLock() + defer c.mu.RUnlock() + + now := c.now().UTC() + for _, state := range c.states { + if state.compiledPolicy == nil { + ineligible++ + continue + } + totalConnections += len(state.activeInbound) + switch { + case state.standbyRequested: + ready++ + case len(state.activeInbound) > 0: + active++ + case state.nextStandbyAt != nil && state.nextStandbyAt.After(now): + countdown++ + default: + ready++ + } + } + return +} diff --git a/lib/autostandby/policy.go b/lib/autostandby/policy.go new file mode 100644 index 00000000..f763b290 --- /dev/null +++ b/lib/autostandby/policy.go @@ -0,0 +1,109 @@ +package autostandby + +import ( + "fmt" + "net/netip" + "slices" + "strings" + "time" +) + +// NormalizePolicy validates and canonicalizes a policy for storage. +func NormalizePolicy(policy *Policy) (*Policy, error) { + if policy == nil { + return nil, nil + } + + if !policy.Enabled { + return &Policy{Enabled: false}, nil + } + + idleTimeout := strings.TrimSpace(policy.IdleTimeout) + if idleTimeout == "" { + return nil, fmt.Errorf("auto_standby.idle_timeout is required when enabled") + } + + parsedTimeout, err := time.ParseDuration(idleTimeout) + if err != nil { + return nil, fmt.Errorf("auto_standby.idle_timeout must be a valid duration: %w", err) + } + if parsedTimeout <= 0 { + return nil, fmt.Errorf("auto_standby.idle_timeout must be positive") + } + + normalized := &Policy{ + Enabled: true, + IdleTimeout: parsedTimeout.String(), + } + + if len(policy.IgnoreSourceCIDRs) > 0 { + seen := make(map[string]struct{}, len(policy.IgnoreSourceCIDRs)) + for _, raw := range policy.IgnoreSourceCIDRs { + cidr := strings.TrimSpace(raw) + if cidr == "" { + continue + } + prefix, err := netip.ParsePrefix(cidr) + if err != nil { + return nil, fmt.Errorf("auto_standby.ignore_source_cidrs contains invalid CIDR %q: %w", cidr, err) + } + canonical := prefix.Masked().String() + if _, ok := seen[canonical]; ok { + continue + } + seen[canonical] = struct{}{} + normalized.IgnoreSourceCIDRs = append(normalized.IgnoreSourceCIDRs, canonical) + } + slices.Sort(normalized.IgnoreSourceCIDRs) + } + + if len(policy.IgnoreDestinationPorts) > 0 { + seen := make(map[uint16]struct{}, len(policy.IgnoreDestinationPorts)) + for _, port := range policy.IgnoreDestinationPorts { + if port == 0 { + return nil, fmt.Errorf("auto_standby.ignore_destination_ports must not contain 0") + } + if _, ok := seen[port]; ok { + continue + } + seen[port] = struct{}{} + normalized.IgnoreDestinationPorts = append(normalized.IgnoreDestinationPorts, port) + } + slices.Sort(normalized.IgnoreDestinationPorts) + } + + return normalized, nil +} + +func compilePolicy(policy *Policy) (*compiledPolicy, error) { + normalized, err := NormalizePolicy(policy) + if err != nil { + return nil, err + } + if normalized == nil || !normalized.Enabled { + return nil, nil + } + + idleTimeout, err := time.ParseDuration(normalized.IdleTimeout) + if err != nil { + return nil, fmt.Errorf("parse normalized auto-standby idle timeout: %w", err) + } + + compiled := &compiledPolicy{ + idleTimeout: idleTimeout, + ignorePorts: make(map[uint16]struct{}, len(normalized.IgnoreDestinationPorts)), + } + + for _, raw := range normalized.IgnoreSourceCIDRs { + prefix, err := netip.ParsePrefix(raw) + if err != nil { + return nil, fmt.Errorf("parse normalized auto-standby source CIDR %q: %w", raw, err) + } + compiled.ignoreSourceCIDRs = append(compiled.ignoreSourceCIDRs, prefix) + } + for _, port := range normalized.IgnoreDestinationPorts { + compiled.ignorePorts[port] = struct{}{} + } + + return compiled, nil +} diff --git a/lib/autostandby/policy_test.go b/lib/autostandby/policy_test.go new file mode 100644 index 00000000..451c5eea --- /dev/null +++ b/lib/autostandby/policy_test.go @@ -0,0 +1,45 @@ +package autostandby + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePolicyCanonicalizesValues(t *testing.T) { + t.Parallel() + + normalized, err := NormalizePolicy(&Policy{ + Enabled: true, + IdleTimeout: "300s", + IgnoreSourceCIDRs: []string{"10.0.0.0/8", " 10.0.0.0/8 ", "192.168.1.1/24"}, + IgnoreDestinationPorts: []uint16{9000, 22, 9000}, + }) + require.NoError(t, err) + require.NotNil(t, normalized) + + assert.True(t, normalized.Enabled) + assert.Equal(t, "5m0s", normalized.IdleTimeout) + assert.Equal(t, []string{"10.0.0.0/8", "192.168.1.0/24"}, normalized.IgnoreSourceCIDRs) + assert.Equal(t, []uint16{22, 9000}, normalized.IgnoreDestinationPorts) +} + +func TestNormalizePolicyRejectsInvalidValues(t *testing.T) { + t.Parallel() + + _, err := NormalizePolicy(&Policy{ + Enabled: true, + IdleTimeout: "0s", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be positive") + + _, err = NormalizePolicy(&Policy{ + Enabled: true, + IdleTimeout: "5m", + IgnoreDestinationPorts: []uint16{0}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain 0") +} diff --git a/lib/autostandby/status.go b/lib/autostandby/status.go new file mode 100644 index 00000000..bb48be57 --- /dev/null +++ b/lib/autostandby/status.go @@ -0,0 +1,49 @@ +package autostandby + +import "time" + +type Status string + +const ( + StatusUnsupported Status = "unsupported" + StatusDisabled Status = "disabled" + StatusIneligible Status = "ineligible" + StatusActive Status = "active" + StatusIdleCountdown Status = "idle_countdown" + StatusReadyForStandby Status = "ready_for_standby" + StatusStandbyRequested Status = "standby_requested" + StatusError Status = "error" +) + +type Reason string + +const ( + ReasonUnsupportedPlatform Reason = "unsupported_platform" + ReasonPolicyMissing Reason = "policy_missing" + ReasonPolicyDisabled Reason = "policy_disabled" + ReasonInstanceNotRunning Reason = "instance_not_running" + ReasonNetworkDisabled Reason = "network_disabled" + ReasonMissingIP Reason = "missing_ip" + ReasonHasVGPU Reason = "has_vgpu" + ReasonActiveInbound Reason = "active_inbound_connections" + ReasonIdleTimeoutNotElapsed Reason = "idle_timeout_not_elapsed" + ReasonObserverError Reason = "observer_error" + ReasonReadyForStandby Reason = "ready_for_standby" +) + +// StatusSnapshot is a diagnostic view of the controller's current state for one VM. +type StatusSnapshot struct { + Supported bool + Configured bool + Enabled bool + Eligible bool + Status Status + Reason Reason + ActiveInboundCount int + IdleTimeout string + IdleSince *time.Time + LastInboundActivityAt *time.Time + NextStandbyAt *time.Time + CountdownRemaining *time.Duration + TrackingMode string +} diff --git a/lib/autostandby/types.go b/lib/autostandby/types.go new file mode 100644 index 00000000..38165056 --- /dev/null +++ b/lib/autostandby/types.go @@ -0,0 +1,79 @@ +package autostandby + +import ( + "net/netip" + "time" +) + +const ( + StateRunning = "Running" +) + +// Policy configures per-instance automatic standby behavior. +type Policy struct { + Enabled bool `json:"enabled"` + IdleTimeout string `json:"idle_timeout,omitempty"` + IgnoreSourceCIDRs []string `json:"ignore_source_cidrs,omitempty"` + IgnoreDestinationPorts []uint16 `json:"ignore_destination_ports,omitempty"` +} + +// Instance is the minimal instance view needed by the auto-standby controller. +type Instance struct { + ID string + Name string + State string + NetworkEnabled bool + IP string + HasVGPU bool + AutoStandby *Policy + Runtime *Runtime +} + +// Connection is the normalized network view used by activity classification. +type Connection struct { + OriginalSourceIP netip.Addr + OriginalSourcePort uint16 + OriginalDestinationIP netip.Addr + OriginalDestinationPort uint16 + TCPState TCPState +} + +// Runtime stores persisted and in-memory idle-tracking timestamps. +type Runtime struct { + IdleSince *time.Time `json:"idle_since,omitempty"` + LastInboundActivityAt *time.Time `json:"last_inbound_activity_at,omitempty"` +} + +// TCPState is the conntrack TCP state for a flow. +type TCPState uint8 + +const ( + TCPStateNone TCPState = 0 + TCPStateSynSent TCPState = 1 + TCPStateSynRecv TCPState = 2 + TCPStateEstablished TCPState = 3 + TCPStateFinWait TCPState = 4 + TCPStateCloseWait TCPState = 5 + TCPStateLastAck TCPState = 6 + TCPStateTimeWait TCPState = 7 + TCPStateClose TCPState = 8 + TCPStateListen TCPState = 9 + TCPStateIgnore TCPState = 10 + TCPStateRetrans TCPState = 11 +) + +// Active reports whether the TCP state should keep a VM awake. +func (s TCPState) Active() bool { + switch s { + case TCPStateSynRecv, TCPStateEstablished, TCPStateFinWait, TCPStateCloseWait, TCPStateLastAck: + return true + default: + return false + } +} + +type compiledPolicy struct { + idleTimeout time.Duration + ignoreSourceCIDRs []netip.Prefix + ignorePorts map[uint16]struct{} +} diff --git a/lib/autostandby/types_test.go b/lib/autostandby/types_test.go new file mode 100644 index 00000000..a77bdb0d --- /dev/null +++ b/lib/autostandby/types_test.go @@ -0,0 +1,14 @@ +package autostandby + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTCPStateConstantsMatchExpectedKernelValues(t *testing.T) { + t.Parallel() + + assert.Equal(t, TCPState(10), TCPStateIgnore) + assert.Equal(t, TCPState(11), TCPStateRetrans) +} diff --git a/lib/instances/README.md b/lib/instances/README.md index b56e276d..13ada056 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -52,6 +52,8 @@ Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEM memory-ranges # Memory state ``` +`metadata.json` also carries controller-owned auto-standby runtime timestamps when that feature is enabled, so idle countdown state can survive Hypeman restarts. + **Benefits:** - Content-addressable IDs (ULID = time-ordered) - Self-contained: all instance data in one directory diff --git a/lib/instances/auto_standby.go b/lib/instances/auto_standby.go new file mode 100644 index 00000000..de061dbf --- /dev/null +++ b/lib/instances/auto_standby.go @@ -0,0 +1,50 @@ +package instances + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/autostandby" +) + +func cloneAutoStandbyPolicy(policy *autostandby.Policy) *autostandby.Policy { + if policy == nil { + return nil + } + + cloned := &autostandby.Policy{ + Enabled: policy.Enabled, + IdleTimeout: policy.IdleTimeout, + } + if len(policy.IgnoreSourceCIDRs) > 0 { + cloned.IgnoreSourceCIDRs = append([]string(nil), policy.IgnoreSourceCIDRs...) + } + if len(policy.IgnoreDestinationPorts) > 0 { + cloned.IgnoreDestinationPorts = append([]uint16(nil), policy.IgnoreDestinationPorts...) + } + return cloned +} + +func cloneAutoStandbyRuntime(runtime *autostandby.Runtime) *autostandby.Runtime { + if runtime == nil { + return nil + } + + cloned := &autostandby.Runtime{} + if runtime.IdleSince != nil { + idleSince := runtime.IdleSince.UTC() + cloned.IdleSince = &idleSince + } + if runtime.LastInboundActivityAt != nil { + lastInboundActivityAt := runtime.LastInboundActivityAt.UTC() + cloned.LastInboundActivityAt = &lastInboundActivityAt + } + return cloned +} + +func normalizeAutoStandbyPolicy(policy *autostandby.Policy) (*autostandby.Policy, error) { + normalized, err := autostandby.NormalizePolicy(policy) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return normalized, nil +} diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go new file mode 100644 index 00000000..7114da3d --- /dev/null +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -0,0 +1,230 @@ +//go:build linux + +package instances + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +const autoStandbyE2EManualEnv = "HYPEMAN_RUN_AUTO_STANDBY_E2E" + +func requireAutoStandbyE2EManualRun(t *testing.T) { + t.Helper() + if os.Getenv(autoStandbyE2EManualEnv) != "1" { + t.Skipf("set %s=1 to run auto-standby end-to-end integration tests", autoStandbyE2EManualEnv) + } +} + +type integrationAutoStandbyStore struct { + manager *manager +} + +func (s integrationAutoStandbyStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { + insts, err := s.manager.ListInstances(ctx, nil) + if err != nil { + return nil, err + } + + out := make([]autostandby.Instance, 0, len(insts)) + for _, inst := range insts { + out = append(out, autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + }) + } + return out, nil +} + +func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id string) error { + _, err := s.manager.StandbyInstance(ctx, id, StandbyInstanceRequest{}) + return err +} + +func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { + return s.manager.SetAutoStandbyRuntime(ctx, id, runtime) +} + +func (s integrationAutoStandbyStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { + src, unsub := s.manager.SubscribeLifecycleEvents(LifecycleEventConsumerAutoStandby) + dst := make(chan autostandby.InstanceEvent, 16) + go func() { + defer close(dst) + for event := range src { + var inst *autostandby.Instance + if event.Instance != nil { + inst = &autostandby.Instance{ + ID: event.Instance.Id, + Name: event.Instance.Name, + State: string(event.Instance.State), + NetworkEnabled: event.Instance.NetworkEnabled, + IP: event.Instance.IP, + HasVGPU: event.Instance.GPUProfile != "" || event.Instance.GPUMdevUUID != "", + AutoStandby: event.Instance.AutoStandby, + } + } + dst <- autostandby.InstanceEvent{ + Action: autostandby.InstanceEventAction(event.Action), + InstanceID: event.InstanceID, + Instance: inst, + } + } + }() + return dst, unsub, nil +} + +func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { + requireAutoStandbyE2EManualRun(t) + requireKVMAccess(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + mgr, _ := setupCompressionTestManagerForHypervisor(t, hypervisor.TypeCloudHypervisor) + require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) + require.NoError(t, mgr.systemManager.EnsureSystemFiles(ctx)) + createNginxImageAndWait(t, ctx, mgr.imageManager) + + connSource := autostandby.NewConntrackSource() + if _, err := connSource.ListConnections(ctx); err != nil { + if errors.Is(err, unix.EPERM) || errors.Is(err, unix.EACCES) { + t.Skipf("conntrack access unavailable for auto-standby e2e test; rerun as root or with CAP_NET_ADMIN: %v", err) + } + require.NoError(t, err) + } + + inst, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "auto-standby-e2e", + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeCloudHypervisor, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "3s", + }, + }) + require.NoError(t, err) + instanceID := inst.Id + + t.Cleanup(func() { + logInstanceArtifactsOnFailure(t, mgr, instanceID) + _ = mgr.DeleteInstance(context.Background(), instanceID) + }) + + inst, err = waitForInstanceState(ctx, mgr, instanceID, StateRunning, 30*time.Second) + require.NoError(t, err) + require.NoError(t, waitForVMReady(ctx, inst.SocketPath, 10*time.Second)) + require.NoError(t, waitForExecAgent(ctx, mgr, inst.Id, 30*time.Second)) + require.NoError(t, waitForLogMessage(ctx, mgr, inst.Id, "start worker processes", 45*time.Second)) + + conn, err := dialGuestPortWithRetry(inst.IP, 80, 15*time.Second) + require.NoError(t, err) + defer func() { + if conn != nil { + _ = conn.Close() + } + }() + + require.Eventually(t, func() bool { + conns, err := connSource.ListConnections(ctx) + if err != nil { + t.Logf("conntrack read while waiting for inbound activity failed: %v", err) + return false + } + + count, _, err := autostandby.ActiveInboundCount(autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(StateRunning), + NetworkEnabled: true, + IP: inst.IP, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "3s", + }, + }, conns) + if err != nil { + t.Logf("active inbound count failed: %v", err) + return false + } + return count > 0 + }, 10*time.Second, 200*time.Millisecond, "host->guest TCP connection never appeared in conntrack") + + controllerCtx, controllerCancel := context.WithCancel(ctx) + controllerDone := make(chan error, 1) + controller := autostandby.NewController( + integrationAutoStandbyStore{manager: mgr}, + connSource, + autostandby.ControllerOptions{ + Log: slog.Default(), + ReconnectDelay: 250 * time.Millisecond, + }, + ) + go func() { + controllerDone <- controller.Run(controllerCtx) + }() + t.Cleanup(func() { + controllerCancel() + select { + case err := <-controllerDone: + if err != nil { + t.Logf("auto-standby controller exited with error during cleanup: %v", err) + } + case <-time.After(2 * time.Second): + t.Log("timed out waiting for auto-standby controller shutdown") + } + }) + + time.Sleep(5 * time.Second) + + current, err := mgr.GetInstance(ctx, instanceID) + require.NoError(t, err) + require.Equal(t, StateRunning, current.State, "instance should remain running while inbound TCP connection is open") + + require.NoError(t, conn.Close()) + conn = nil + + inst, err = waitForInstanceState(ctx, mgr, instanceID, StateStandby, 45*time.Second) + require.NoError(t, err) + require.Equal(t, StateStandby, inst.State) +} + +func dialGuestPortWithRetry(ip string, port int, timeout time.Duration) (net.Conn, error) { + deadline := time.Now().Add(timeout) + address := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) + var lastErr error + + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", address, 1*time.Second) + if err == nil { + return conn, nil + } + lastErr = err + time.Sleep(250 * time.Millisecond) + } + + if lastErr == nil { + lastErr = fmt.Errorf("timed out dialing %s", address) + } + return nil, lastErr +} diff --git a/lib/instances/auto_standby_runtime.go b/lib/instances/auto_standby_runtime.go new file mode 100644 index 00000000..a625bc63 --- /dev/null +++ b/lib/instances/auto_standby_runtime.go @@ -0,0 +1,34 @@ +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/autostandby" +) + +// GetAutoStandbyRuntime returns the persisted auto-standby runtime metadata for an instance. +func (m *manager) GetAutoStandbyRuntime(_ context.Context, id string) (*autostandby.Runtime, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + return cloneAutoStandbyRuntime(meta.AutoStandbyRuntime), nil +} + +// SetAutoStandbyRuntime persists auto-standby runtime metadata for an instance. +func (m *manager) SetAutoStandbyRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.AutoStandbyRuntime = cloneAutoStandbyRuntime(runtime) + return m.saveMetadata(meta) +} diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index f58b5b1f..75ceef96 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -35,6 +35,7 @@ const compressionGuestExecTimeout = 20 * time.Second func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { t.Parallel() + requireStandbyRestoreCompressionManualRun(t) runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "cloud-hypervisor", diff --git a/lib/instances/compression_integration_test_helpers_test.go b/lib/instances/compression_integration_test_helpers_test.go new file mode 100644 index 00000000..deff89f8 --- /dev/null +++ b/lib/instances/compression_integration_test_helpers_test.go @@ -0,0 +1,17 @@ +//go:build linux + +package instances + +import ( + "os" + "testing" +) + +const standbyRestoreCompressionManualEnv = "HYPEMAN_RUN_STANDBY_RESTORE_COMPRESSION_TESTS" + +func requireStandbyRestoreCompressionManualRun(t *testing.T) { + t.Helper() + if os.Getenv(standbyRestoreCompressionManualEnv) != "1" { + t.Skipf("set %s=1 to run standby/restore compression integration tests", standbyRestoreCompressionManualEnv) + } +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 1abd4c21..9d3b616f 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -355,6 +355,7 @@ func (m *manager) createInstance( SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), + AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), } // 12. Ensure directories @@ -580,6 +581,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { return err } } + normalizedAutoStandby, err := normalizeAutoStandbyPolicy(req.AutoStandby) + if err != nil { + return err + } + req.AutoStandby = normalizedAutoStandby // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 2c75c94e..31d2efb0 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -269,7 +269,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin } now := time.Now() - forkMeta := cloneStoredMetadataForFork(meta.StoredMetadata) + forkMeta := cloneStoredMetadata(meta.StoredMetadata) forkMeta.Id = forkID forkMeta.Name = req.Name forkMeta.CreatedAt = now @@ -470,7 +470,7 @@ func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) return err } -func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { +func cloneStoredMetadata(src StoredMetadata) StoredMetadata { dst := src if src.Env != nil { @@ -482,6 +482,9 @@ func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { if src.NetworkEgress != nil { dst.NetworkEgress = cloneNetworkEgressPolicy(src.NetworkEgress) } + if src.AutoStandby != nil { + dst.AutoStandby = cloneAutoStandbyPolicy(src.AutoStandby) + } if src.Credentials != nil { dst.Credentials = cloneCredentialPolicies(src.Credentials) } diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index cb1a2299..1e73927c 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -295,9 +296,15 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { StoppedAt: &stoppedAt, HypervisorPID: &pid, ExitCode: &exitCode, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "5m", + IgnoreSourceCIDRs: []string{"10.0.0.0/8"}, + IgnoreDestinationPorts: []uint16{22}, + }, } - cloned := cloneStoredMetadataForFork(src) + cloned := cloneStoredMetadata(src) require.Equal(t, src, cloned) cloned.Env["A"] = "2" @@ -308,6 +315,8 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { cloned.Cmd[0] = "printf" *cloned.HypervisorPID = 4321 *cloned.ExitCode = 42 + cloned.AutoStandby.IgnoreSourceCIDRs[0] = "192.168.0.0/16" + cloned.AutoStandby.IgnoreDestinationPorts[0] = 443 now := time.Now() *cloned.StartedAt = now *cloned.StoppedAt = now @@ -320,6 +329,8 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { require.Equal(t, "echo", src.Cmd[0]) require.Equal(t, 1234, *src.HypervisorPID) require.Equal(t, 17, *src.ExitCode) + require.Equal(t, "10.0.0.0/8", src.AutoStandby.IgnoreSourceCIDRs[0]) + require.Equal(t, uint16(22), src.AutoStandby.IgnoreDestinationPorts[0]) require.Equal(t, startedAt, *src.StartedAt) require.Equal(t, stoppedAt, *src.StoppedAt) } diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go new file mode 100644 index 00000000..d6efa4b5 --- /dev/null +++ b/lib/instances/metadata_clone.go @@ -0,0 +1,14 @@ +package instances + +// deepCopyMetadata returns a metadata copy that can be mutated without +// affecting the originally loaded instance metadata. +func deepCopyMetadata(src *metadata) *metadata { + if src == nil { + return nil + } + + return &metadata{ + StoredMetadata: cloneStoredMetadata(src.StoredMetadata), + AutoStandbyRuntime: cloneAutoStandbyRuntime(src.AutoStandbyRuntime), + } +} diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index be982f74..9ea0296e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -147,7 +147,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps SourceHypervisor: stored.HypervisorType, CreatedAt: time.Now(), }, - StoredMetadata: cloneStoredMetadataForFork(meta.StoredMetadata), + StoredMetadata: cloneStoredMetadata(meta.StoredMetadata), } sizeBytes, err := snapshotstore.DirectoryFileSize(snapshotGuestDir) if err != nil { @@ -199,7 +199,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps SourceHypervisor: stored.HypervisorType, CreatedAt: time.Now(), }, - StoredMetadata: cloneStoredMetadataForFork(meta.StoredMetadata), + StoredMetadata: cloneStoredMetadata(meta.StoredMetadata), } sizeBytes, err := snapshotstore.DirectoryFileSize(snapshotGuestDir) if err != nil { @@ -289,7 +289,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } - restored := cloneStoredMetadataForFork(rec.StoredMetadata) + restored := cloneStoredMetadata(rec.StoredMetadata) restored.Id = sourceMeta.Id restored.Name = sourceMeta.Name restored.DataDir = m.paths.InstanceDir(id) @@ -427,7 +427,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } now := time.Now() - forkMeta := cloneStoredMetadataForFork(rec.StoredMetadata) + forkMeta := cloneStoredMetadata(rec.StoredMetadata) forkMeta.Id = forkID forkMeta.Name = req.Name forkMeta.CreatedAt = now diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 4bf7f45b..3ba47b0e 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/images" ) @@ -27,6 +28,7 @@ import ( // metadata wraps StoredMetadata for JSON serialization type metadata struct { StoredMetadata + AutoStandbyRuntime *autostandby.Runtime `json:"auto_standby_runtime,omitempty"` } // ensureDirectories creates the instance directory structure diff --git a/lib/instances/types.go b/lib/instances/types.go index fb8f355d..141a6b9d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -3,6 +3,7 @@ package instances import ( "time" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/tags" @@ -139,6 +140,9 @@ type StoredMetadata struct { // Snapshot policy defaults for this instance. SnapshotPolicy *SnapshotPolicy + // Automatic standby policy driven by host-observed inbound TCP activity. + AutoStandby *autostandby.Policy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -220,6 +224,7 @@ type CreateInstanceRequest struct { SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance + AutoStandby *autostandby.Policy // Optional automatic standby policy } // StartInstanceRequest is the domain request for starting a stopped instance @@ -228,11 +233,10 @@ type StartInstanceRequest struct { Cmd []string // Override cmd (nil = keep previous/image default) } -// UpdateInstanceRequest is the domain request for updating a running instance. -// Currently supports updating env vars referenced by credential policies -// to enable secret/key rotation without instance restart. +// UpdateInstanceRequest is the domain request for updating mutable instance properties. type UpdateInstanceRequest struct { - Env map[string]string // Updated environment variables (merged with existing) + Env map[string]string // Updated environment variables (merged with existing) + AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil } // ForkInstanceRequest is the domain request for forking an instance. diff --git a/lib/instances/update.go b/lib/instances/update.go index 74ae3384..f8556219 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -14,10 +14,9 @@ type updateInstanceRulesService interface { UpdateInstanceRules(ctx context.Context, instanceID string, rules []egressproxy.HeaderInjectRuleConfig) error } -// updateInstance updates mutable properties of a running instance. -// Currently supports updating env vars referenced by credential policies, -// which causes the egress proxy header inject rules to be recomputed -// with the new secret values — enabling key rotation without restart. +// updateInstance updates mutable instance properties. +// Env updates recompute egress proxy header inject rules with the new secret +// values. Auto-standby updates only change persisted metadata. func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInstanceRequest) (*Instance, error) { log := logger.FromContext(ctx) @@ -32,17 +31,38 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta if err != nil { return nil, fmt.Errorf("get instance: %w", err) } - - if inst.State != StateRunning && inst.State != StateInitializing { - return nil, fmt.Errorf("%w: instance must be running or initializing to update (current state: %s)", ErrInvalidState, inst.State) + normalizedAutoStandby, err := normalizeAutoStandbyPolicy(req.AutoStandby) + if err != nil { + return nil, err } + req.AutoStandby = normalizedAutoStandby if err := validateUpdateInstanceRequest(meta, req); err != nil { return nil, err } + if len(req.Env) > 0 && inst.State != StateRunning && inst.State != StateInitializing { + return nil, fmt.Errorf("%w: instance must be running or initializing to update env (current state: %s)", ErrInvalidState, inst.State) + } + nextMeta := deepCopyMetadata(meta) + if req.AutoStandby != nil { + nextMeta.AutoStandby = cloneAutoStandbyPolicy(req.AutoStandby) + } + if len(req.Env) == 0 { + if err := m.saveMetadata(nextMeta); err != nil { + return nil, fmt.Errorf("save metadata: %w", err) + } - prevEnv := cloneEnvMap(meta.Env) - nextEnv := cloneEnvMap(meta.Env) + log.InfoContext(ctx, "instance updated", "instance_id", id) + + updated, err := m.getInstance(ctx, id) + if err != nil { + return nil, fmt.Errorf("get updated instance: %w", err) + } + return updated, nil + } + + prevEnv := cloneEnvMap(nextMeta.Env) + nextEnv := cloneEnvMap(nextMeta.Env) if nextEnv == nil { nextEnv = make(map[string]string) } @@ -50,7 +70,7 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta nextEnv[k] = v } - if err := validateCredentialEnvBindings(meta.Credentials, nextEnv); err != nil { + if err := validateCredentialEnvBindings(nextMeta.Credentials, nextEnv); err != nil { return nil, err } @@ -60,7 +80,7 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta return nil, fmt.Errorf("egress proxy service unavailable") } - if err := applyUpdatedInstanceEnv(ctx, log, id, meta, prevEnv, nextEnv, m.saveMetadata, svc); err != nil { + if err := applyUpdatedInstanceEnv(ctx, log, id, nextMeta, prevEnv, nextEnv, m.saveMetadata, svc); err != nil { return nil, err } @@ -74,8 +94,11 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta } func validateUpdateInstanceRequest(meta *metadata, req UpdateInstanceRequest) error { + if len(req.Env) == 0 && req.AutoStandby == nil { + return fmt.Errorf("%w: request must include env and/or auto_standby", ErrInvalidRequest) + } if len(req.Env) == 0 { - return fmt.Errorf("%w: env must include at least one credential source env var", ErrInvalidRequest) + return nil } if meta == nil || len(meta.Credentials) == 0 || meta.NetworkEgress == nil || !meta.NetworkEgress.Enabled { return fmt.Errorf("%w: instance has no credential-backed env vars to update", ErrInvalidRequest) diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 338483be..b55b7d67 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/egressproxy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,11 +23,11 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { }, } - t.Run("requires at least one env key", func(t *testing.T) { + t.Run("requires at least one update field", func(t *testing.T) { err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{}) require.Error(t, err) assert.ErrorIs(t, err, ErrInvalidRequest) - assert.Contains(t, err.Error(), "at least one credential source env var") + assert.Contains(t, err.Error(), "env and/or auto_standby") }) t.Run("rejects instances without credential backed envs", func(t *testing.T) { @@ -54,6 +55,16 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { }) require.NoError(t, err) }) + + t.Run("allows auto standby without env changes", func(t *testing.T) { + err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{ + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "5m", + }, + }) + require.NoError(t, err) + }) } type fakeUpdateInstanceRulesService struct { @@ -170,3 +181,61 @@ func TestApplyUpdatedInstanceEnvReturnsRollbackFailure(t *testing.T) { assert.Equal(t, prevEnv, meta.Env) require.Len(t, svc.calls, 2) } + +func TestApplyUpdatedInstanceEnvSavesAutoStandbyAlongsideEnvWithoutMutatingOriginal(t *testing.T) { + t.Parallel() + + original := &metadata{ + StoredMetadata: StoredMetadata{ + Id: "inst-autostandby-copy", + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + Credentials: map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{{ + As: CredentialInjectAs{ + Header: "Authorization", + Format: "Bearer ${value}", + }, + }}, + }, + }, + Env: map[string]string{"OUTBOUND_OPENAI_KEY": "old"}, + AutoStandby: &autostandby.Policy{ + Enabled: false, + IdleTimeout: "5m0s", + }, + }, + } + updated := deepCopyMetadata(original) + updated.AutoStandby = &autostandby.Policy{ + Enabled: true, + IdleTimeout: "10m0s", + IgnoreSourceCIDRs: []string{"10.0.0.0/8"}, + IgnoreDestinationPorts: []uint16{22}, + } + + prevEnv := cloneEnvMap(updated.Env) + nextEnv := map[string]string{"OUTBOUND_OPENAI_KEY": "new"} + svc := &fakeUpdateInstanceRulesService{} + + var saved *metadata + err := applyUpdatedInstanceEnv(context.Background(), nil, updated.Id, updated, prevEnv, nextEnv, func(meta *metadata) error { + saved = deepCopyMetadata(meta) + return nil + }, svc) + require.NoError(t, err) + + require.NotNil(t, saved) + require.NotNil(t, saved.AutoStandby) + assert.True(t, saved.AutoStandby.Enabled) + assert.Equal(t, "10m0s", saved.AutoStandby.IdleTimeout) + assert.Equal(t, []string{"10.0.0.0/8"}, saved.AutoStandby.IgnoreSourceCIDRs) + assert.Equal(t, []uint16{22}, saved.AutoStandby.IgnoreDestinationPorts) + assert.Equal(t, nextEnv, saved.Env) + + require.NotNil(t, original.AutoStandby) + assert.False(t, original.AutoStandby.Enabled) + assert.Equal(t, "5m0s", original.AutoStandby.IdleTimeout) + assert.Equal(t, map[string]string{"OUTBOUND_OPENAI_KEY": "old"}, original.Env) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index d4d336f0..8ceb515c 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -29,6 +29,33 @@ const ( BearerAuthScopes = "bearerAuth.Scopes" ) +// Defines values for AutoStandbyStatusReason. +const ( + AutoStandbyStatusReasonActiveInboundConnections AutoStandbyStatusReason = "active_inbound_connections" + AutoStandbyStatusReasonHasVgpu AutoStandbyStatusReason = "has_vgpu" + AutoStandbyStatusReasonIdleTimeoutNotElapsed AutoStandbyStatusReason = "idle_timeout_not_elapsed" + AutoStandbyStatusReasonInstanceNotRunning AutoStandbyStatusReason = "instance_not_running" + AutoStandbyStatusReasonMissingIp AutoStandbyStatusReason = "missing_ip" + AutoStandbyStatusReasonNetworkDisabled AutoStandbyStatusReason = "network_disabled" + AutoStandbyStatusReasonObserverError AutoStandbyStatusReason = "observer_error" + AutoStandbyStatusReasonPolicyDisabled AutoStandbyStatusReason = "policy_disabled" + AutoStandbyStatusReasonPolicyMissing AutoStandbyStatusReason = "policy_missing" + AutoStandbyStatusReasonReadyForStandby AutoStandbyStatusReason = "ready_for_standby" + AutoStandbyStatusReasonUnsupportedPlatform AutoStandbyStatusReason = "unsupported_platform" +) + +// Defines values for AutoStandbyStatusStatus. +const ( + AutoStandbyStatusStatusActive AutoStandbyStatusStatus = "active" + AutoStandbyStatusStatusDisabled AutoStandbyStatusStatus = "disabled" + AutoStandbyStatusStatusError AutoStandbyStatusStatus = "error" + AutoStandbyStatusStatusIdleCountdown AutoStandbyStatusStatus = "idle_countdown" + AutoStandbyStatusStatusIneligible AutoStandbyStatusStatus = "ineligible" + AutoStandbyStatusStatusReadyForStandby AutoStandbyStatusStatus = "ready_for_standby" + AutoStandbyStatusStatusStandbyRequested AutoStandbyStatusStatus = "standby_requested" + AutoStandbyStatusStatusUnsupported AutoStandbyStatusStatus = "unsupported" +) + // Defines values for BuildEventType. const ( Heartbeat BuildEventType = "heartbeat" @@ -194,6 +221,67 @@ type AttachVolumeRequest struct { Readonly *bool `json:"readonly,omitempty"` } +// AutoStandbyPolicy Linux-only automatic standby policy based on active inbound TCP connections +// observed from the host conntrack table. +type AutoStandbyPolicy struct { + // Enabled Whether automatic standby is enabled for this instance. + Enabled *bool `json:"enabled,omitempty"` + + // IdleTimeout How long the instance must have zero qualifying inbound TCP connections + // before Hypeman places it into standby. + IdleTimeout *string `json:"idle_timeout,omitempty"` + + // IgnoreDestinationPorts Optional destination TCP ports that should not keep the instance awake. + IgnoreDestinationPorts *[]int `json:"ignore_destination_ports,omitempty"` + + // IgnoreSourceCidrs Optional client CIDRs that should not keep the instance awake. + IgnoreSourceCidrs *[]string `json:"ignore_source_cidrs,omitempty"` +} + +// AutoStandbyStatus defines model for AutoStandbyStatus. +type AutoStandbyStatus struct { + // ActiveInboundConnections Number of currently tracked qualifying inbound TCP connections. + ActiveInboundConnections int `json:"active_inbound_connections"` + + // Configured Whether the instance has any auto-standby policy configured. + Configured bool `json:"configured"` + + // CountdownRemaining Remaining time before the controller attempts standby, when applicable. + CountdownRemaining *string `json:"countdown_remaining"` + + // Eligible Whether the instance is currently eligible to enter standby. + Eligible bool `json:"eligible"` + + // Enabled Whether the configured auto-standby policy is enabled. + Enabled bool `json:"enabled"` + + // IdleSince When the controller most recently observed the instance become idle. + IdleSince *time.Time `json:"idle_since"` + + // IdleTimeout Configured idle timeout from the auto-standby policy. + IdleTimeout *string `json:"idle_timeout"` + + // LastInboundActivityAt Timestamp of the most recent qualifying inbound TCP activity the controller observed. + LastInboundActivityAt *time.Time `json:"last_inbound_activity_at"` + + // NextStandbyAt When the controller expects to attempt standby next, if a countdown is active. + NextStandbyAt *time.Time `json:"next_standby_at"` + Reason AutoStandbyStatusReason `json:"reason"` + Status AutoStandbyStatusStatus `json:"status"` + + // Supported Whether the current host platform supports auto-standby diagnostics. + Supported bool `json:"supported"` + + // TrackingMode Diagnostic identifier for the runtime tracking mode in use. + TrackingMode string `json:"tracking_mode"` +} + +// AutoStandbyStatusReason defines model for AutoStandbyStatus.Reason. +type AutoStandbyStatusReason string + +// AutoStandbyStatusStatus defines model for AutoStandbyStatus.Status. +type AutoStandbyStatusStatus string + // AvailableDevice defines model for AvailableDevice. type AvailableDevice struct { // CurrentDriver Currently bound driver (null if none) @@ -332,6 +420,10 @@ type CreateIngressRequest struct { // CreateInstanceRequest defines model for CreateInstanceRequest. type CreateInstanceRequest struct { + // AutoStandby Linux-only automatic standby policy based on active inbound TCP connections + // observed from the host conntrack table. + AutoStandby *AutoStandbyPolicy `json:"auto_standby,omitempty"` + // Cmd Override image CMD (like docker run ). Omit to use image default. Cmd *[]string `json:"cmd,omitempty"` @@ -764,6 +856,10 @@ type IngressTarget struct { // Instance defines model for Instance. type Instance struct { + // AutoStandby Linux-only automatic standby policy based on active inbound TCP connections + // observed from the host conntrack table. + AutoStandby *AutoStandbyPolicy `json:"auto_standby,omitempty"` + // CreatedAt Creation timestamp (RFC3339) CreatedAt time.Time `json:"created_at"` @@ -1227,6 +1323,10 @@ type Tags map[string]string // UpdateInstanceRequest defines model for UpdateInstanceRequest. type UpdateInstanceRequest struct { + // AutoStandby Linux-only automatic standby policy based on active inbound TCP connections + // observed from the host conntrack table. + AutoStandby *AutoStandbyPolicy `json:"auto_standby,omitempty"` + // Env Environment variables to update (merged with existing). // Only keys referenced by the instance's existing credential `source.env` bindings // are accepted. Use this to rotate real credential values without restarting the VM. @@ -1676,6 +1776,9 @@ type ClientInterface interface { UpdateInstance(ctx context.Context, id string, body UpdateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetAutoStandbyStatus request + GetAutoStandbyStatus(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // ForkInstanceWithBody request with any body ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2126,6 +2229,18 @@ func (c *Client) UpdateInstance(ctx context.Context, id string, body UpdateInsta return c.Client.Do(req) } +func (c *Client) GetAutoStandbyStatus(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetAutoStandbyStatusRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewForkInstanceRequestWithBody(c.Server, id, contentType, body) if err != nil { @@ -3517,6 +3632,40 @@ func NewUpdateInstanceRequestWithBody(server string, id string, contentType stri return req, nil } +// NewGetAutoStandbyStatusRequest generates requests for GetAutoStandbyStatus +func NewGetAutoStandbyStatusRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/auto-standby/status", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewForkInstanceRequest calls the generic ForkInstance builder with application/json body func NewForkInstanceRequest(server string, id string, body ForkInstanceJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -4947,6 +5096,9 @@ type ClientWithResponsesInterface interface { UpdateInstanceWithResponse(ctx context.Context, id string, body UpdateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateInstanceResponse, error) + // GetAutoStandbyStatusWithResponse request + GetAutoStandbyStatusWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetAutoStandbyStatusResponse, error) + // ForkInstanceWithBodyWithResponse request with any body ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) @@ -5633,6 +5785,30 @@ func (r UpdateInstanceResponse) StatusCode() int { return 0 } +type GetAutoStandbyStatusResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *AutoStandbyStatus + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetAutoStandbyStatusResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetAutoStandbyStatusResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type ForkInstanceResponse struct { Body []byte HTTPResponse *http.Response @@ -6558,6 +6734,15 @@ func (c *ClientWithResponses) UpdateInstanceWithResponse(ctx context.Context, id return ParseUpdateInstanceResponse(rsp) } +// GetAutoStandbyStatusWithResponse request returning *GetAutoStandbyStatusResponse +func (c *ClientWithResponses) GetAutoStandbyStatusWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetAutoStandbyStatusResponse, error) { + rsp, err := c.GetAutoStandbyStatus(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetAutoStandbyStatusResponse(rsp) +} + // ForkInstanceWithBodyWithResponse request with arbitrary body returning *ForkInstanceResponse func (c *ClientWithResponses) ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) { rsp, err := c.ForkInstanceWithBody(ctx, id, contentType, body, reqEditors...) @@ -7897,6 +8082,46 @@ func ParseUpdateInstanceResponse(rsp *http.Response) (*UpdateInstanceResponse, e return response, nil } +// ParseGetAutoStandbyStatusResponse parses an HTTP response from a GetAutoStandbyStatusWithResponse call +func ParseGetAutoStandbyStatusResponse(rsp *http.Response) (*GetAutoStandbyStatusResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetAutoStandbyStatusResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest AutoStandbyStatus + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseForkInstanceResponse parses an HTTP response from a ForkInstanceWithResponse call func ParseForkInstanceResponse(rsp *http.Response) (*ForkInstanceResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -9198,6 +9423,9 @@ type ServerInterface interface { // Update instance properties // (PATCH /instances/{id}) UpdateInstance(w http.ResponseWriter, r *http.Request, id string) + // Get auto-standby diagnostic status + // (GET /instances/{id}/auto-standby/status) + GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request, id string) // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) ForkInstance(w http.ResponseWriter, r *http.Request, id string) @@ -9429,6 +9657,12 @@ func (_ Unimplemented) UpdateInstance(w http.ResponseWriter, r *http.Request, id w.WriteHeader(http.StatusNotImplemented) } +// Get auto-standby diagnostic status +// (GET /instances/{id}/auto-standby/status) +func (_ Unimplemented) GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) func (_ Unimplemented) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { @@ -10290,6 +10524,37 @@ func (siw *ServerInterfaceWrapper) UpdateInstance(w http.ResponseWriter, r *http handler.ServeHTTP(w, r) } +// GetAutoStandbyStatus operation middleware +func (siw *ServerInterfaceWrapper) GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetAutoStandbyStatus(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // ForkInstance operation middleware func (siw *ServerInterfaceWrapper) ForkInstance(w http.ResponseWriter, r *http.Request) { @@ -11453,6 +11718,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Patch(options.BaseURL+"/instances/{id}", wrapper.UpdateInstance) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/instances/{id}/auto-standby/status", wrapper.GetAutoStandbyStatus) + }) r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/fork", wrapper.ForkInstance) }) @@ -12509,6 +12777,41 @@ func (response UpdateInstance500JSONResponse) VisitUpdateInstanceResponse(w http return json.NewEncoder(w).Encode(response) } +type GetAutoStandbyStatusRequestObject struct { + Id string `json:"id"` +} + +type GetAutoStandbyStatusResponseObject interface { + VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error +} + +type GetAutoStandbyStatus200JSONResponse AutoStandbyStatus + +func (response GetAutoStandbyStatus200JSONResponse) VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetAutoStandbyStatus404JSONResponse Error + +func (response GetAutoStandbyStatus404JSONResponse) VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetAutoStandbyStatus500JSONResponse Error + +func (response GetAutoStandbyStatus500JSONResponse) VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type ForkInstanceRequestObject struct { Id string `json:"id"` Body *ForkInstanceJSONRequestBody @@ -13774,6 +14077,9 @@ type StrictServerInterface interface { // Update instance properties // (PATCH /instances/{id}) UpdateInstance(ctx context.Context, request UpdateInstanceRequestObject) (UpdateInstanceResponseObject, error) + // Get auto-standby diagnostic status + // (GET /instances/{id}/auto-standby/status) + GetAutoStandbyStatus(ctx context.Context, request GetAutoStandbyStatusRequestObject) (GetAutoStandbyStatusResponseObject, error) // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) ForkInstance(ctx context.Context, request ForkInstanceRequestObject) (ForkInstanceResponseObject, error) @@ -14539,6 +14845,32 @@ func (sh *strictHandler) UpdateInstance(w http.ResponseWriter, r *http.Request, } } +// GetAutoStandbyStatus operation middleware +func (sh *strictHandler) GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request, id string) { + var request GetAutoStandbyStatusRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetAutoStandbyStatus(ctx, request.(GetAutoStandbyStatusRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetAutoStandbyStatus") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetAutoStandbyStatusResponseObject); ok { + if err := validResponse.VisitGetAutoStandbyStatusResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // ForkInstance operation middleware func (sh *strictHandler) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { var request ForkInstanceRequestObject @@ -15316,258 +15648,272 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZIw+ioIntloaoakqItlWRMde2TJdmvbsnUs2312mv4osAok0aoCqgEUJdrh", - "v/MA84jzJF8gAdSNKLIoW5I19sbGtMzCNZGZyEzk5VMr4HHCGWFKtg4+tWQwJTGGPw+VwsH0PY/SmLwh", - "f6ZEKv1zInhChKIEGsU8ZWqYYDXV/wqJDARNFOWsddA6w2qKrqZEEDSDUZCc8jQK0Ygg6EfCVqdFrnGc", - "RKR10NqMmdoMscKtTkvNE/2TVIKySetzpyUIDjmL5maaMU4j1ToY40iSTmXaUz00whLpLl3ok4034jwi", - "mLU+w4h/plSQsHXwe3EbH7LGfPQHCZSe/HCGaYRHETkmMxqQRTAEqRCEqWEo6IyIRVAcme/RHI14ykJk", - "2qE2S6MI0TFinJGNEjDYjIZUQ0I30VO3DpRIiQcyIaxpSEPPCRydIPMZnRyj9pRclyfZfjzab9UPyXBM", - "Fgf9JY0x62rg6mW58aFtceyXu76RKY/jdDgRPE0WRz55fXr6DsFHxNJ4RERxxP3tbDzKFJkQoQdMAjrE", - "YSiIlP79u4/FtfX7/f4B3j7o93t93ypnhIVc1ILUfPaDdKsfkiVDNgKpHX8BpK/enxyfHKIjLhIuMPRd", - "mKmC2EXwFPdVRJvyqfjw/2lKo3AR60f6ZyKGlEmFWQ0OntiPGlx8jNSUINsPvT9F7TEXKCSjdDKhbLLR", - "BN81w4qIIuEQq8XpYKnItqGcIUVjIhWOk1anNeYi1p1aIVakq780mlAQvGI63aLRZIuklpqTHMaybnTX", - "BFGGYhpFVJKAs1AW56BM7e3Wb6ZAMEQI7uFQz/TPKCZS4glBbc02Ne9mSCqsUomoRGNMIxI2OiMfIpjN", - "/MFHiIaEKTqmZfo26NTFo2Bre8fLO2I8IcOQTuxNVB7+GH7XKKbHUQha+zeiCW3ebB8wpSDjxfmeA+uG", - "SQQZE0E0jn/hdIngM8I0tej5/gLztv6fzfyK3rT38yYA8yxv/rnT+jMlKRkmXFKzwgXOZb9oNAJQI+jh", - "XzN8WnbWBYySCovl9AEtvgIlmvU1gs25afq501J4srLLW92myjuBNdopS1yglkU+mxHmEZICzpT9UIbO", - "Sz5BEWUE2Rb2LDRP1BP8HHFgiV8JDhn4F4lfr/sGzMv8UDOa/tZpEZbGGpgRnxShOSVYqBEpAbPmCrMD", - "5aurBf9ZiXwqdxWWZLicg5xRxkiIdEtL2KYlSiVIqgvbByq6pGo4I0J6aQ6W9StVyLaoHSriweWYRmQ4", - "xXJqVozDEOgVR2elnXiktZL4ixPNBN2AIEVIpDg6/+Vw+9EeshN4YCh5KgKzgsWdFHrr4U1bpLAY4Sjy", - "4kY9uq1/Ry9iiB8DzjPCqLt7Mgx0iGk4Xcueph6+00pSOTV/Ae/Wq4K7T7MBjV6R/vuDZ9NHwCSMllCr", - "M/llwNeJOWw0ibiG6RyljP6ZlgTsHjrRuoJC+qKgIQk7CMMHzbJxqnh3QhgRmk+hseAxSFsFIRi1SW/S", - "66CBlgu7Wgru4u1uv9/tD1plMTba7U6SVIMCK0WEXuD/+R13Px52/9HvPvmQ/znsdT/87S8+BGgqmTup", - "0O6z7Wi/g9xii+J6daGrRPkbc//i8n0cxxz1ieYT65700cmi4GD2GvLgkoge5ZsRHQks5ptsQtn1QYQV", - "kaq88+VtvyosYB9LgMAmGkxrgqGi9AAatyN+RUSgOXBENOLJjmbCVMkOwlpvBuaF9C35dxRgpmnBCBdc", - "IMJCdEXVFGFoV4ZWPO/ihHapWWqr04rx9UvCJmraOtjbWcBzjeRt+0f3w1/dTxv/7UV1kUbEg+RveKoo", - "myD4bG71KZUoXwNVJF55Ig66aQRiXkzZiem2la0EC4HnX37CbiPLTtooc7VHHcQeyf/1jAhBQ3erHp0e", - "o3ZEL4lFdyRShgZpv78TQAP4k9hfAh7HmIXmt40eeh1TpW+zNL+kjTWoVzzu31skmHKQM6KI6w1loK4R", - "YnIYBoKAfoKjpdfwMhB7gXWUjbt4af/CperGmOEJAW3SNkQjwS+JXihKeEQDSiS6JHMtpMzRRA/anVFJ", - "NfkQNkMzbIwGvQF7O+WSmCbuk1ZEAkJnBMU8uERJhAMy5aCIz3CUEtlBV1MtMWhmLAiO7M9IkBhTNmBT", - "vUgZ8ISEWocwzWBr6IKw2QWKcQJUigUBEkUxVkRQHNGPJETcdIlJSPUFNWAE8BolWJNsEHChb199tgQH", - "0wIUfpLowsgbFzD8BWUaKy8MXfUGrHjyn1qv3719+vrdq+Ph67Nnrw5Phr8++1/9s+nUOvj9U8vYNzNB", - "4ynBggj0l0+w389GOg2JaB20DlM15YJ+NMaWz52WhoHU+IUT2uMJYZj2Ah63Oq2/Fv/54fMHJ0/pqQib", - "aTLwLOyzV5YxV6GHoxw7Y55E1kAEoh0GUy1wmBdn7zb15ZpgKdVU8HQyLROGvdnXIomQyssh5cNR4lsT", - "lZfoZPM10nIHiqgm0EzO2Or3T59uykFL/+OR+8dGDx0bqoXlaxbChRV/5FSjjxbCAWWOzt4hHEU8sCaQ", - "sdaVxnSSChL2KpY3GN3HnwlTYp5w6tPBKswpb7rIo7rd/OsarGhzRNmm1MfQDdaDO+DNjTWBZ2xGBWex", - "1sZmWFB9zcoyrbx6ffxs+OzV+9aB5uNhGlij4tnrN29bB62dfr/f8iGoxqAVPPDF2bsjOClDNiqJ0slQ", - "0o8eSeAw2x+KScyF0YBtH9SelgUFQ7cIDmfQ2nnx1CDX1gvAK3coIZXQ2o1iBi5jzPaLpz5smc4TImZU", - "+sxkv2Tf3MkXrnXD7su4LYmYEZEhLWBxr6B+BBFPw25hyk5rTAUJBNZo1+q0/iSxlsNnHzXq5Gv39PNb", - "rxrJnysESxwllJElkuU3IuFdcXEZcRx2t76ygMeI0mMvbvGV+VA+X4sTJEOJVmfBGsHCKxqq6TDkV0wv", - "2cNX7ReUNc6Y67XeCY7+/c9/vT/N1aStF6PEctqt7UdfyGkrvFUP7TWBZBtJE/823iX+Tbw//fc//+V2", - "cr+bMILIjYQ6e/7PzAjAsjWuh6VnSmPNLIPltylRUyIKt7dDFv2T0YehO3K4V9hKyTxafNNcYNR8RkSE", - "5wXGa9fU2uoD96usSlAFtGr7aTZ6iXTnFWxYj+Yu+RdVHX2772e0nkV51vRU8wp7LzRZSbaQre1T++f2", - "4pJqVnRJkyFIzUM8yUy2y16bzy9pYkVx6GGOMYoMIwhTEN5HnKvegP02JQzB2cEBk2sSAM+TCit0eHYi", - "0RWNIjDwAFNZvFq0YJ+zFdNcKv2/ImUdNEqVlta5IsjqTTBJCmuBxiOCUobdc3ZFdrYbrOKVBcslEYxE", - "QyMby4aQMZ2Q7VQLHNjqGEtFhOH2aVKG1/Gvp+eofTxnOKYB+tWMesrDNCLoPE00P9goQ68zYIkgM61C", - "sAkYG6mdl48RT1WXj7tKEOKWGMNgmYnMvrXOXpy9s6/1cqM3YG+IBixhIQlhze7GkUhNsUIhZz9piiVh", - "edji/BWg+2m505IMJ3LK1RA0v/kq7nRum5+Z1mvZAjqtWZCk5SPdrh7nK3iQ18CbUaFSHGleWxInve/z", - "xvPDozYYx5Ki+mL5XobdWJUfVpsaTMzI4AayKFT77R5GUmps9yio8gsWEKdnfmq22BXjnzC3kKV2n1zV", - "/IK5zs0gVRDZsTtuZzeA0kkGkzKs8NcBz6EsqOa1dvWQSEWZQSfdFlmJUKL2hdbmLR5r/f2igy7+WvpB", - "075TLbR8cYUMNICfMP1TcfyqUWKluaC5Ulg5HCxvfh6HstZRCc22kBKYSX23ahkrIT30CzBxpEicaE7G", - "JohKJA3zJSFi/OrviBuhxnUdML00adw8LDgyo5GkE0bZZEOL+fpiwmFoLEvjVKVCt5tRmUOzjDrOelPd", - "wFuzOmL4cZxKfSMHURoSdOEsPBdluXDR/rOoElqD0IKGY0ACmg0oe2ozTpWeXm84xiqYajjxVBm/L7t1", - "WV5A2cq06j3UriV7KbvB+Z9n7KIMVGtvqDB+vTn7RgNmwYJ9ss4MaAUVv4nykszhyJ05Ei8YJIuWSL+9", - "UBDJoxmx127RljnCwaW5SozrhTVjGoOktUFq8q+QqNc6t+ooNLwag7+sKiyiEpiA7WZzjLHSv7H/zjMu", - "pDdn5utoxVgSAD6oHgcIxLGLjtGVCFggENPIEqGQChKoheEpmwwYuIBc2F96drQLTeRaRvERoU/Z8cqC", - "BW3H9CkdLSqcrBP7YBi9NR5TpUjYKcsGl4QkcvWmtHhtDdce67ogV4I6RmYNRmFD8YywMRcBia2S8GWK", - "47PCYF41br0hFj0yDHwLa7b4hHCSRJSExv3HnAeYWaU9J7CxVl1+w4rWZjwAylNe4Ci6QG3baAMJovci", - "3VkxznJkf3t05lAge7V+f9rRGKm5wMVUqWSo/0cONRVfVAezfR2F6+H0nSTRfh/0q93dHXuq1uhmFlwZ", - "tmxf83o11B+NE7/rH8Z4rGnRuYk0EeWP8i65JfWSsrDpAL/qtrXWuUwwcprGbRvoEkG6aTIRGDxkv6Z5", - "7sbPngDNeg6+wvnd5+WYQTVIpeJxwdcRtSseGrTsy1EG1oxH3RArDKbMhvZWs9xFv+F4boYyulidJWY4", - "GXncfuhHzXXRhE7waK7K7wdbfZ/G96Vv0G4tvmOp8783GiQJh4ov90CmY+TaNnE4hPtkqPhwNqaekbNr", - "LXdfoRIFFWd/q9fqIbpJQK05AWScYGocRA0QQGh8f1p8u+sNWBeu3wN0nE2QDZsNiUG2xKF5OWlzUVgE", - "Ba8zNJpvIIzen/bQ22y1P0mkFZYZcQEJUyzRiBCGUjA9w23YNXdxcQGphEtTVbtb24mJXdiAJ0puv/XQ", - "L/OExNjaoTQpxFjRADydRrSyH7iOzEHZN2HMilawRlarZX7bb8iESiUqXtuo/eb50c7OzpOq/XL7Ube/", - "1d169Harf9DX//+P5g7eXz88wzfWYZm3WN+xIvc5endyvG2NpeV51Mdd/GT/+hqrJ3v0Sj75GI/E5I8d", - "fCcBHH5Wdpw7vaF2KonoOjapscrn6lbwKKtxZbuxh9otOZzl/rPL2hpIvNUtbyMyxefzbD1u148dqTLM", - "lV7Thc0tavLzBPTOnEoKEpx1Tgyo1w3zmMrLp4Lgy5BfMc+9HeMJkUNzn/n9GVJpnGzItbVuCM7VWJp3", - "07LVc2v38e7+zt7ufr/vCchYRHge0GGgb6BGC3h9dIIiPCcCQR/UhgevEI0iPioj+qOdvf3H/Sdb203X", - "YZ54msEhU7xcL9S2EPmbC+5zX0qL2t5+vLezs9Pf29vebbQqay9utChnWy6JJI93Hu9u7W/vNoKCT6B/", - "5gJkqgJ86HNd0PqTeWzsyoQEdEwDBCE2SHdA7RiuMJK9VpVpcoTDoTWe+O8OhWkkl3pMmMlsS2Noi9NI", - "0SQi5hscSCNbNOz8GEbyeaNQxogYZvFDa4xkw4pWegi4vWRNUCk8rAS6UypBCsmFJ0qi8MBQ6Eo+B6eZ", - "L+xDHR7YPTTEhpdadepGZEaiIhKYq0svNuaCoAxPzKGVdkXZDEc0HFKWpF6UqAXl81SALGoGRXjEU2We", - "GeHAipOA0zLoHmPNrpvpuc+5uFzp/qlv4qFIGdPDrLQKHYIhfWxNNXCLY2R7uwiDgtCXPQeaR1P7XaI3", - "poexEOU/J6lClCmutVMWjuYdmMlakhgSRCoOnNQaDO0wTaVLv9wCxlLn/mHmy3nnHfm+dMfGXeDrathi", - "QtRQKqxWSiwaU95C+3No3tibXHdcaUhpAHdGru4C6OBu39Vo25UMJ7cD8WXOaJmtIW8Et7CgIekhoC7w", - "inHhfRVKO1c8SUiY2X96A3ZuSCX7SZoXFN3RwEFNCRWICzqh5YnLBrbb9GpbBxUdNt0YHYsdFyVU+Aju", - "G/VEj8eKCANBF7lcDD+yh9DqtCzsW52W5URl0LgfPRDJXS0Xlvji7N26vmmJ4GMaebYLvhD2q9XMnNfW", - "y93+eXfr/zMemBrfQESjzPhPxDwkvUpyAGjf7OZ5cfburG5NWWYGVFzdwp4yjxcP58j8GhxE7KOSfZW0", - "GoxDf32xZJPksvcTnyw7Fjgmo3Q8JmIYe4xrz/V3ZBoY1ybK0OnTsjyr5eamWvNZ6XBAbR7jwAbWN4O+", - "xyBX2UanAM0P/uN6Q8w1XBeOp49K2DY2Iq+HXmW5MNCLs3cS5V5KHktd+Xhr/eXPpnNJAxyZEU10LWVF", - "AxsgZ2MJ+SzvaE2RHjk59sqGjhBQezZJUiDD8zfdk9fvN+OQzDqlNYFn0ZRHRK97o8AtZi4oL3fuLzGJ", - "WZ2lwyCGbEpABVhlFNwYSAV69UBHcYWjoYy4z1njrf6I4CNqv39ugqb0CjooKR2l/r0AhRJ+73kpRnOk", - "umnPYcKqybRE4F7dsZxCxphXCtsrTeojlV8IjkzmnDI+5/Hd7uD5Zfmg+eVK6rWD+OY9cY7hDYK3jk6P", - "jcAQcKYwZUSgmChs8/QUXFxAHGp1Wl19R4WYxOBqN/77cu+WGhN8MRqr1oh7tJB241YMuDXh4m+MC0KI", - "YszomEhlw8VLM8sp3n60d2CSWoRkvPtor9frrRuj8iwPSml0FJvGhb8QrtKT0y87h1sIRWmyl0+ts8O3", - "v7QOWpupFJsRD3C0KUeUHRT+nf0z/wB/mH+OKPOGsDTKg0LHC/lPyk+a+s4yvx/onTDrEqZxiYMCv/KJ", - "qUafAc8GiJvzhgsrPNH6icG4L40LvnHmkDx9lSpkDCk6hDbIHkI/LreEOsEI2tg5U6ZolCdWWbSB3ig1", - "jlyaPWAhc0BCWJYvIIrMXwFnM00VvuQBJQbuvn3R+4H1chmG1IPJv1ltzzhJQFTVanprbeIkWY22fkEx", - "439Nk6bY0GbPTXTvXP8mb2zl2V9P/ufP/1+ePf5j68+X79//7+zF/xy/ov/7Pjp7/UURVMuj2u81NP2r", - "RaPDw1IpJL0pKp1iFXgEqimXqgbC9gtS3Phr9tARKH4HA9ZFL6kiAkcHaNCquAgPWqhNrnGgTC/EGdJD", - "2UiHDd35zJh/dOdPTrf8XB0jtCENwh5IFskk01HIY0zZxoANmB0LuY1IeNPXf4UowIlKBdGnp2XYaI5G", - "Agd5KEM+eQd9wknyeWPAQMMl10roHSRYqCwNh5sBkMKuyvgM2OYkdIHhRkMesOxeyuLCjY2mlxlBwDZf", - "9bj0A8WrvnBRDsXZ7/si6MHrSx9kRKUi4JidYbZGo8wdDe33S6xiv7/fXyngZzi0BP2AEhaTZDqkbEBL", - "BoFhasO4wUOtgS1d8yZDI+iXt2/PNBj0f8+RGyiHRXbERskzPoDS2AhVJAvefxstn+nbnG7DDRkjGXSL", - "GkQNPTPuoW9fniNFROwc9tuBBueYBnp/8PxPpUw1KlKMDo9On230GmT5BNhm619yjm+zHVaDO6zRrM4W", - "mGG8hm8HnRyDe66l0FyAA7ea51ygyDCYnK4P0DtJyr6ucFTmVd+cZDTPLW/mBhi0NtyISZVTHKA3mdyI", - "s6VkjpY5Mrghc7qEYe3Di/H5WRi94pcL3kxWL7KsDTx8sMqcxPWNW88KlpO/B+JA89avu2DTXI+2i8ZQ", - "PZkfNfKzv3VpZWddHXXdBA3lGMpC/G2Wo6F5coXbSFKwqK9dUzWsfYRH+rN9cndayftTNMWS/aTgY0U3", - "2dp53Chbpp616fN18eGaj82SMqpyAZnZs6sJTb2kUWS8GSSdMByhJ6h9fvLi15OXLzdQF71+fVo9imU9", - "fOfTIFeDQ+0XZ+8g2gXLoXsBqnd6xLnjMLmmUsnFeNVGD6nLc0P8Usrf4A0A3viKSR3c6/PCNu4iXcN9", - "uvV9e6kiliZ3+NIMDVbYvaUEDbXM1ZfcoMxnzc9fN9XCrSynFPvj4w9FmcD5XN84t0GnRT3+podSs0AS", - "opOzPMVhbpRyw1f29GS7t7W339vq93tb/SYmuhgHS+Y+PTxqPnl/2xgiDvDoIAgPyPgLTIQWsY3whqMr", - "PJdo4MTrQcvI8wVBvkC2VgRv9Py6mELiZhkjqgLFqpwQ6+SAaJbc4Usj6pclOj4vpzhuLOQ9+scXZUMm", - "Ta926/tgew3XsX4TFPA0CrUgNdKka/QyElr1URKVZ48Gan/HLhm/YuWtGyOoZgB/pkTM0fvT05LJXJCx", - "TY7bYOPgM1FzDjxZ6xi2V8jaK1dzwzwLd5Fbocp2C9fdV8+kULTZOR9Mg6ENbHe5+Ol9N6fMHI3GkyV7", - "qlhdQjIbpqlPqtKfXOTFu3cnxyXkwHhva7+//6S7P9ra6+6G/a0u3trZ624/wv3xTvB4pyY9fXO/mZu7", - "wpSpuT7SCQAPFkwTyBYeaHrLfFlGqUKZn5sm5CMtnqKCHGziesCocMKoghyOlE30MKDjWzHZBGiaNJOU", - "UQUZASAfDWV6y2BM0YNY76UD9ALawiccQ7yRW4RWjsp2BBzOjR1VMwY3dQL/Wr7k82mqtNwGfeQ0VUj/", - "C7atwWDVleVDGB5zgF5x6COckynjVb3HNAfnrcXmVR2pbd2KnPspTGYZ5gF6njHJjM1attqWxP5peLf1", - "jAav742S75098ZbGlvzkCm5lnZaBaKvTcoAC97NFRzS7Lm+MRREVfQ8MBEfAQnNHn1TRyCY5gJ1QqWhg", - "tEYMh1tHyTahFwmHRgSoey403iNWTMg6OUbx/hS1IZzxb8gqlfpfG9nTYpEqd7ef7D7Ze7z9ZK9R0EK+", - "wNUM/gh8mxYXt5LbB0k6dJU/arZ+dPYO7j59r8o0NlYCu/eCj2gieKClVcpQXkokn/xJ70kxViPk6Sgq", - "WJ1sYBcEBDSp+1LzPvYnjWZ0PGZ/fgwut/8QNN663pPbI69yl03kl4RPipbSBbWRjLomCaPfnR4QSsja", - "iJM3RMIO0DlRCPCni3AAl3TmkmRRzsWlWIh7EWt3Z2dn//Gj7UZ4ZVdXIJwh6K+Lqzy1KyiQGLRE7Tfn", - "52izgHBmTOenCfkhmBXg/HSGbD7mfsmFU+tOOz4sqZGXcqyxY8/iWpC/t0KQ3ZQFOnhWZQLSApV7ob2z", - "03+8+2j/UTMythrbUFwv5zAuJ4cBj01jUjz5NljX3x6eIT26GOOgrKFsbe/sPtp7vL/WqtRaq4IUPCZ1", - "xhoL23+892h3Z3urWeiUz4JugwJLBFvmXR6i8yCF5zQ8oFhkvZ2628IneBoEe0OCCNP4MHDeL5Xbx6TI", - "GArTLD+EJheDNRIsXFwN+jZS0Sple4xowAVKWZaYqbfaHHoz62Y9mzb3wWo2vihDR5hpcFkff5OJ8Qaw", - "SwSZUZ7KrzAQVyTQyDSOOBdr9a1zJ3pDZBopY4KkEr0//QmYiEYuJBVJyq7yFv2WRELccHNrEXAJJ/xY", - "XQesRqfR5OiXbbhTQ6adZW6wJfKvDTgKNatK2eqn6yMcBSnkHsPZeepdQegATxU8tM+Nk0cUcc5QMMVs", - "QiCXu8l0yCYIoymPwl7L/1QShcOx9wmDX6GIm1QJl4QkNi2XWYTupmUWOiOo/YLnBeUMKlXS6z6KDVex", - "iZfK2PgorinOKX2Og1mAkoYnVrwQxW+6lLT5iE8kaIEK3Fd61eQxCRbGKwUzk2ZuFhvlsRx5ta1ve88S", - "K9zbd4Waq5OPrUZrZQzFM0jiQHApEYnoBFKavT8tL3OZ/2FMGY01n139HF1ebAPUlQln0pcXBe402Tgb", - "pe9C9Dh2fcmVCDgM/pvepwPzkG9d8VGMWQqJugqITK4TKgx6NHscn3Kphlk0yZqLlWoISZhSQfKQM3df", - "TsF/f25YHLTx3ouOtd0EXNZr4ka9F7DKP1TdAut5qheifmh1Mhz0ofFiPM3SEJ48JqgaALJOxFeetYdK", - "GJUWgo1Qm3FVYkuFzDMbTR6q/DqqnqeuoOvL3f5502Cs5bFXZ1hNT9iYe1I7rmHwtx7tznchISKmkIYM", - "hYRREjrlMbP8W9sW+MhHkqAwJRZyRiAV2AIcG/KGFI7MGcUom1R4fXXCJmZ4s4blOZpgXtuwyZOj9HtW", - "vxUpwMo4CUiEcx/rRh4PVA79luLFgQWZpBEWqBpwuGTJch5HlF02GV3O4xGPaIB0h+pzzphHEb8a6k/y", - "Z9jLRqPd6Q7D3EWw8jxjFmcdRM2BVObNt/Cz3uVGxT0dTC+bpv8mVOxu8oLr9Rt6TiNiY/LeMXpdQPRy", - "EpPd7X5d5ELNoKWYhcV4znU5t0VZH8W7UMvDrOiBxz/NeABVXiXKhsjSfn27BRezZXEai6YY1HaPwi5J", - "TBmuhWQtjSwhzbzcqu4PbjWbkgTl2Xf3Hz3ea5gt54tsnUtqGn+BZXMWL7Fo1pzUaROz2f6j/SdPdnYf", - "Pdley0DlPGVqzqfOW6Z4PpXaJhWj2aM+/N9aizK+Mv4l1fjLlBdUqlNy4wV9XkK6eZR0zbNHbZruqHiS", - "7p2lbAFtZmNcIi0dlkSuQimuNhmPCSiVQwO3br6Yind9ozUEOMEBVXOPwQRfmYztWZNKtG8Ta1p5sR6Q", - "2rFtwgbNuWQ6yh06225y9FdjWq/gwn7jpFsyHdWZ8V9XZzVG/NwGVHwiavBCk9cFWDQXZPu5wrLk1aH/", - "DiDlcl5qreo/ZFo0rwntcD0rC517Rvoi1v0loIvHXznOgtm3JCRXIb7sCq0nwbV0aM+N7Ksyudort8If", - "7AV4s17DUTEd3tJ8g6Xcefmtu/68zYrELfYzN9j68xVcQNfpWM0MBvho12BBno/dKaFEDTYpLlYnhL6F", - "/D7Gp+BGGX6sO8KdJPmxP99KYp+F4zgnyrU91xp9Gi3J58wUETPsMUy5IZBrUrajGk7cQdbEh7bijUql", - "wd2pX1azEbgN3ce0FDhMBBnT6yXYYhqY67rsPi4tBMJyzm+J2jG+RruPUTDFQlbWzuhkqqJ52ci664me", - "+LISykRp0bl5dvT8NF3HxRcNe5zF0X0ke16IdfBnbSfhcFmc+lHWzNmMEzwH2bJWEXy8s9vv72z3bxSo", - "/rWSyRfGqfMILfSzxpzS02NxhMz/czHj4JWgpiaZA5NUguD4ALypEhwQFJExhHFlmV5X6vQLUy9fvH0k", - "tY7/Gf67g3KFR62dxbI4xhkIHm4cG+PvttFyr7TlWI/i98VlLwkWy9hMsBA1VvVf3ev2d7r9vbdbOweP", - "9g62tm4jsj0DUp0Lz+OPW1ePo2083o3254//3Jo+nmzHO95wgVuoW1ApA1gpY2D3kBBRTSVZTcEqSUQZ", - "6crM7W21A/ISXmBeklbS/3rWB7ODpcLCeXmTRZkBqxw41YpqdxHYZFe/1IRSXf7J8fJl38iPrLoQP4JV", - "lwL41GwxkGll60vTeqSs4b3zrtCw8c2z1Ldx1d3jc/oG0vaecg3EffhcYowlClt2Yy/eah4VbsIFVdN4", - "+fWQNcuSBMBj+EepwnIgTQ+dTBgkji3+nL19FGs7686tTiv6uFumGft785AqGxSfIaA96qIY0OBtAPIS", - "L4cCNMlVC2HcE7AgAIift7pbT+CFPvq4+3O/+6SHfit4CnQMtIrg23KtS7/2m8AwY5Qgd5qX860naz2j", - "O3guw6Bf7b1UdxHbcHmL43nWTndXOLfp0gHnnxfOuBJVdEt1gj4v2bETnNeLY3e9PKJJzy+bPHnb31o7", - "6856V0TvCzyKv0jVa6beRViqOrn6JZYq08eQSK103YEKNaxSqN5mBbehAODaCbH2B0hWNHyXDl3mQo+z", - "AnTQhCuUBwGslHJg+SJlXnworz+vMgzZHWoRYnsFQjRbUxbIR5eR7skxSgQP0yD3gI1g0WkQECnHKRRN", - "7jWVaFe/Md6mMg+u5VqjX63M12nvq8NMyXX9eb8i16owpUbY+qPe6q8+6luxAHRaaRKu5mGmUTMOtlYm", - "jhU+lR57RBnsFSmosJkPDTj6myIEFxU8qLeEAi0OpIkrAKhxahGTpKfsH74eenMEHJOIKOIbBJmKnNbt", - "g8qci65mqVt7+36TGb4eBhCQuLCQXwlJtJwec2krZMaYzb0LqybBRu2+KwApEQzfNYm4LLTKi3u8Ugqp", - "Parm+cQrFl0T8VVM355l6Pi6ycRtz5WlHm5PUHlrVaW6jDJVX8+i3fGw+w9jZ0TD3sHmz3/7f7sf/voX", - "f2mVkiIlieiGZAwPYJdk3jU1Y7XS1iunI4VsNy2psC1IogiOwYoQXBJjtYjxdXG9j/oZJc1f4XhhC/By", - "GFOW/Xvlhv72l/p3twIY3wHzWHmOt5HYVXHHYtsxEROXPd15e0ExbRbNNaQlKmSPs9e0o8qfZNalWLb1", - "wog2PSg2PKKQhFMOmNZScBCQRJGwZ7NoUViL4EBR1eLFNoud887WtIYhdaeNlqkkqfrkLRF80GLkqmtm", - "CLsadXYf7dl67UVIbi2ckO/MTMB1XX1DDWWPFeAllRBN4JxmC41Rm8SJmrscrc6tcWO9APDDbEDvS+ZX", - "zn7Vf/I1cnW+W5qc8zusrlmMz3cLWhmZv3D+tRnx/H5Rx9VEO4YmbcWwcmKYisYjVbfebSrWV/QQvP8W", - "XZz0N+NZaLNRTtJqWu7NmKlNm/vWF80QQkHgpb6kOZW5YPUudFrtIrlUSCzsrLCS+rM5dbJQtUhzPYDO", - "NGiupkSQwkFAhzyB55ogs35+DWJkTIbKhIhutaScqXogKDgOZmqsA0HmC7po11qeoOYUX2czgE0Uy4WX", - "A9hHnqpt68VTqGTyxpUWo2M3BCyjIqn6s82UsahJcfTFwyhi1eK+TXsv4VletYT71dFWBTnzOUqo6cPH", - "3zBVz7kA2bY+IuXWk9aA3BwSASG51ZQ0jfK50JiEQ56q5fRvU7zbcJQQjciYC1LIfuvkeAxIbCsOr+AF", - "LmYiX8MHn9wgSZAKquZa8bMS5YhgQcRhaggeAAkTwc/5xJBM9vNnsICNPQ5oLwgjggbo8OwE6BGq5mvi", - "eH+KIjomwTzQ+jPkAl1InwFC3uujE6s7uYRt8B5IFaCeKwJ8eHYCNUWF0R9a/d52rw/EnBCGE9o6aO30", - "tqDCqkY42OIm5J6HP61vuQkro5ydhFYOemqa6F4Cx0QRIVsHv3t8tBURJpe9BKkTTwpif4KpsHJ/EoHn", - "uEEVqvtC9iJ3lR6Y+7hjAN7Y9CPV3PrRkeS1PdYPGhMM1cAWt/t9o2YxZS9enNea3PzDxtvl8zaS5wA8", - "nlQ+C3K9kyktyD93Wrv9rbXWs7I8pG/adwynasoF/UhgmY/WBMKNJj1hxrkXmTQR1n2hSGeAQkUK+/2D", - "Pi+ZxjEWcweuHFYJl3XCMJEIQ4E6U0jhDz7qIWvYhuylcsrTSHMTZDyXSagvLKx5Sm/yEWERTOmMDJi9", - "p02pTywgu3OM9P1s1JYyaZipzelnMWVPeTivQDcbblMP13XmzBzA1SyJkgwh2dOwrkpKbsikjEG1RUls", - "SsmsXMCijwSUx5UB99YFJgwzlVdbNXVxL8nc2kq9AzbKyqIZHhwLgTLsWbbw7Q1/OAIkv/RH8hxn35AF", - "b1mcYPDAEERpmMtczkMWixGOIm/Y/iTiIxzZ8sGXxCOivoAWFijFPKFOuGE8JCbnYzJXU87M3+koZSo1", - "f48Ev5JEaBHI5n62sLa1My3qQh13GkP+ZVNZQs+5aZa4+emSzD/3BuwwjF3VEGlLv0eS27rKJvsNlci5", - "VBrc9WcnrXmtP0ql4rFFKVYsA2mWyVOVpMq+VEqibMJqaA5VQuWUhAOmOPokTFH4+efNT/mMn0F3ITjU", - "eFJoYra0+YmGn+tWLYdY734ITT3aHwEADFr6dhm09N8TgbXuksopmDIkmC8mxSNtZ6HUWi7cqEI4wAwl", - "PDFh6IBUplx0aQxI/o+jCCkgJddXS5twkjX7sZElvkp2NqzExAFUyAhq2hWIqb+776cnSQJBfAaO/zl/", - "/QrBVaXPwDTLzUYAI8r0LYrCFCR5mL03YM9wMEVGboJUY4MWDQetTLsIN2CtqbR+r90uiLg/66X9bKbp", - "0PDnXk8PZaTnA/T7JzPKgaalJB4qfknYoPW5gwofJlRN01H27YMfoHXe+eclRoDahvdvuNItkCUgvwbN", - "vYFZiLjltdEcYZRzoKIdZUQZFkvrznhAbyGoVXk8kUVgfBqAAXTQOhg4E+ig1Rm0CJvBb9ZOOmh99kPA", - "CtH1ea1M6R0na2dItNfvb6wOm7Pw9YjQpYaa/D4vSF/bX03wsELXouBhNueS8ukTNEWUjLh1B5LPUxy6", - "tPw/RLwVIp61XBSEN+hfvAcM+kbEKLgVCUzrs5GTwJZqJwYtICslaBwuyNUoHNRJcDnyFtWPqjq/qFbs", - "1lFZAEuMHP7t3gH+wbx5IXKY98ldzYsjSDGZleV9WOgIh+UQsePXiF8Q9S1gXP+uWKnNh3mf+PtQ8OcF", - "sXJfDrQKN9skM/fe5A/lBxd+aUcxjbWueg5r6p4TptAz+LVn/+s0HkhMexHxycUBMiCM+ARFlNnXuMJr", - "kb4ULSyhk/Hiz/pZp36XR6lt7s9///NfsCjKJv/+57+0NG3+AnLfNMktIO/qxZRgoUYEq4sD9CshSRdH", - "dEbcZiAzIpkRMUc7fRAzEwGfPKUe5YAN2BuiUsEKr5YmpZG0A4LqwWA/lKVE2igI3ZCObb4FY2D2qPCO", - "lg0o75SiO4suo2YHhQ3oW9HhAATQUpN81upfLb/1zOy5ZD+r2soXLKar+Ysi18pgb9cscE0GAyD20R18", - "sJtG7fPzZxs9BDqGwQrIqQEScz6MFZ57P3jSap5kOEqZoQCUDW8qVPmutf8e2zbNDMB2xO/JAlxXtrze", - "BGxMHkSQ0MHrh67QxBzsh5szDfvss8cu9q3eQHvz/RancM5AjRThr3fODvcWYW6+FEB2Hyowajs/a1dx", - "7+zoxJV22bg3pL+TW0Pv1BZEyK4OxE2dvztTy444G0c0UKjr1gI5+WOSqWplBHko7OCNXTXCbl/V7HXF", - "+22zlIyl9qbL8rLkV97t3x6VSde5RvIMezmu/bhJVqHOMZUB130L2NINcGLrDRrxJaPTIhatMkgZt+3s", - "ylkqLln2fHLsCPLuTFN26pRV74Y7YIrHFYZ4j4ywUkOtkJPyIWHzu+wUXZz/EsvVt4Wa/buTgu7aiuVD", - "84dkxgorYNNc0GTmrb1AXxD1i2lxiwdtZ/Bs/JwIR9UuhTDsOtuW6YqCKQkuzYbgQXq57ntimjRTfc14", - "35PmC+BZR2KxIP8hojRQdnNYLVNwT2xduNvTb2GGtdTbr/fOaxHMA2RwNhk5i7UpuYblnAUb39VT753c", - "ZgbYD/IyO0ujyL14zIhQKCu/XLwDNj+BW9Jq2d5R29Lr4N2bl13CAg5+aJkPlV+Isl++soRvDsxs5Qea", - "NNEJTcAtdfdZnYTzBedv3AVRVt77v7af2wLf/7X93JT4/q+dQ1Pke+PWkKV/V6z5riXuB4x8WuCmZaAB", - "a2JQ6nOVhJq1aiikuvbflZxqNr2WpJrB9Yew2kRYLYJrqbxqj+JWJVYzxz09yWTI5oM2fHL+id+ZpHq3", - "Vj6LkS79LpXlZw9bX4ULsPPCJ8pQKskDdKCkGcYVr42G5uqcIJdeHw51T447AMiOBh3kA7IBIndkvHbr", - "uHPh1s5795brw3hEJylPZTH2JMYqmBJpg5UiUmbAD03szq/nWsH7G8bS/l1eHXcuV//A+1uS+KsHapi3", - "eYFaJfO7Vk1lfttey/y29r2JXXvjaurbNEcbNU6FLoi6KRqXYs0XnR196/LpIuidVlRydQGBBnEwYP+t", - "9Y/fFcHxh59dkEza72/vwe+EzT787OJk2KlDFcKUoDb35uGrY3j2m0D0OaTnzEPyqusw+fwB9VwCm/84", - "BSl/+WyuITks/KEhNdKQCuBariHZs7hdFamcw+rOdSSHbz6A2yQmP7Sku9CSZDoe04ASpvJyVwtOYrZa", - "3gOMLWP2fajg3FG6aBtrSRlRrhBA82Trd+7Yk01+98qRy+v+MH3kuYmKCZ06kl+G9frIt4YP/btlznev", - "hzxkFDMC/yLoEi1T+qooQqbHOFXglJhnCAGvTySM1J6N2EN58UKZJgkXSppskSAAQzUrNdUCsC+zZDlZ", - "pC87JOS1pUR2Bgzyv+vPJpZ/85LMTS5IyvOi/NlObf5HX+xVOZXmvZLR15ex/HlCG8lYd0zGNh3y/clY", - "98Y67kTSOillmW9nhAEK5YhklMyz4D76kbLJxoPyQDXMKttbIZ+RR9TaHNt6gn4V6DkXl02Zgqe+zQPg", - "DcUdfoPal14e5E+6fyUM1BNDPxpp7pxvLBQtuk+ndVrlJEGUhpp1OBbiLt+x4PHQ/mgyfGqqsPkTQakL", - "7Kj3zWT07HegYr/iCtE4iYiWe0iIugab9GlaYcmlyaayUOJrPSaoyaYYQmDSd0lXJsQWQoPnCHdgbXiZ", - "XDwuL9eM+GR12oBschcj78kbMGAmjTdxOb8vUMZkkeJIkogECl1NaTCFHAL6N1NZEML7cZJcZEmDNg7Q", - "C6DUYu4kmLwtidCiY8CZ5BExqQFmcXxxsJjj8v3pKXQy6QNMNsuLA+TyWmYXhNStijkBsvojr2ymg7bG", - "JMGjyJzohZazC/vbsNkC8qROA+bLHMDIlR2QjtFFIYnARU0WAcdQX/KJvC9RtlOfis/sRXEkAHAGNwkL", - "W3Wmaxr58wds9b21GRrmMjDLuOVUBguLecknWRrAEirjJGmKvnaZgMWzOF6Cw6hdKBIoVchT9TepQiIE", - "dLbYXYfcqI0D8w+FLzWi2iIfWZlFQD/vA43Jy+UFlWaqhYoW5l+zOG51WnY9hXxea1gYVuSEqA64+JCg", - "T6aQ+OGHLWGdlA5lZl/I6VC5OWwZ6nqR21bX/u4tWhZQ4fegl5ZfAPJVUOZEFThbnlfEeVCx4abwelUW", - "M+V7fDTidtmVhUJ+zR4EFkoAfgNK66p3gqyeW1Zs7q4fDBZX8JDDBuTCbqBQPFvrJeGbR6SvdyQLW22C", - "IT9wc/0nh0aImaRL6vpBWUKJsKt1B5lwgynnsoD2IzLFM8qFzVltayZlmAkmC6M9Wn+jC42qF7Ys2oUV", - "zw+srQnh4ic7Rw+6Wy8lfw/3Ke/xvKBtZxy/40RqyJsnEUYjQckYJTiVREtLaUyQqclgUx8THExd1dre", - "gL2dEmRr1RUMCFlpUyrRxVZ80UGjVKEIiwloO+aj8T0SJOBxTFho6k8O2JTgGdWqmkARVoQF864kUI90", - "RvKSD1p1t286CJ50soqHHeQKZYKB4aJQBvMCJYIAEhl1mZVqTg6YSNnfTa4/PeyFW+gFIlLhUUTlNMuu", - "H+CQsMCbSO/822ZjX9+Ie07UYqXIe3nluREvvc9nn6ItM6vV+028CD0w1xYuXEXABmx+idAr61XDsq/Y", - "eV4d8z+QpM1e3R7v6WUmA/EyKv42nmRK5bF/PMsoS5JhaqYj5RLS3+1bS8ZQUMpKzy3WJnvTB5csd3wG", - "5rV43uYn9+fJDWxk3wgn7Cwp1O6fIN/0t8ByLVRvxHPvyThobUkFq9g9smC7qPsTn7gocLlvgg0bgsu4", - "cZHnKIFBpzJF7X8w49Lbt3EPuCkzdhbXhQfwAnumrJtEuI4vW+NsLQOulFD/D/MXrCkQ/9lywvtkfPmL", - "wJ0xu5OMvRmGl+B5xPH3/i4TcCFMCJwt4PpwUjAVbIGFB6Y2WNw6GYfoOP/796enG3VcQqilPEKoB8wh", - "yoUgg9hT3+71jAhBQ1ds7+j02JbmoxKJlPXQ65hCBbxLQhIorUF5Kk1Z/l6xQv1i2bBKCXrClJgnnDK1", - "chV509tZzOcbFRu7Yz5pk9B994/HYIV/eEwKeIcWV+wGlmuRCqtaZzznnEaZqRCopS084qkefaGCPhrT", - "iMi5VCQ2nnnjNAIigjSltoqN7Wdi8DqIKok0PXQgZikhIqZSUs7kgNmC2QkRem7dHcql5k5GXuO9whnX", - "PDOs79twYIOi+uCzhVUd1Mr19HGSuHr6Picpu7wvWNJz8EhDch6PeEQDFFF2KVE7opdG6UAziSL9x8ZS", - "l7Yh9PvaNXpuTlka0idszL1lDAzOZsj8fYRtlNmae0R8cGztBSkSi+M/cNB+tiZX8jVBcNRVNCZZuDBK", - "FY3oR8Pq9CBUKhqY2s95sBqUrbXxagN2SpTQbbAgKOBRRALljCubieDB5iDt93eChEJehx0CiwOGV/85", - "hhmPzt5BO1NatzNg+h8w8NvDM/MSO8bWRlBYKCPqiotLdLL5eoWT7zmA6T/YS85scGnUmPfAfzzfrR8L", - "WktDsoZEebJMAeLJd+/GaSW4H9aCh2ktgGD8bDfticABCMVymqqQXzG/ZWDGozTW/zB/nKxK6aBwMH0P", - "Tb8ZadcsZ+U0boMPgijtnkJiyqzcywOFAdhD9S/VgHNbACGm5LnnvQUO1feI3V/fKF+E4zf4NGkh6koY", - "fTO0ddc3n12Dy1RUhMdDIXODaW4nii+3Pl1hWm99ehrx4FKilCkagcGkIGliyJyof8wz3dmHPxATIDrS", - "FV9G5DqhAnJ+gE23MBLRO5YII0VETBmONmHPZhDI2eesWHjGKQQpBxGFMDEaEpTwKIK8JFdTwpDeDRiq", - "3ACFd1ppc+YX2xSfGBVHIxLwmLg8hhs+1e03TNVzLspJCb8Vvvi2AH+9H71Vvc8VeRjrZ/yivIyn+Brc", - "msPUPhO7FbVf8PxHYwrqIDibQWunLwetDhq0tuNBS5/AEQYTKlboEYopSxWRPXRs7FsQhrrXR5IEnIXS", - "pVN0FrydvqwLSjVoWRPhuAf97lLssVgFoHxjJ/GxB90O6f4QYIPaRYKzNBl2gOhCxFMFDtyOrmyrkCgw", - "j2zc+QtsgUZ+6PZNOPlvlnxLPApOWbPLwtEbzp4l3FtpdXNBFVMu8zx9KMAJDqiadxCOIh7k1oNUZq8D", - "3WwpI0HwpdahegP2Jkv1ZwMh0NHZu44zmqGQykszgrWL9dDrGREyHWWLQ8ANjAUPDoOEA6Y4CnAUpJHG", - "WzIekwBiGCIaUyVr7GrZUm6zcFw+iefg3UcLuodmTPLjBJxejhaygnGb5qg3BQkiTOOiUakKHBB94UkX", - "zL4jPSjX1/A4ss9bgeBSIjtUl0R0QkeRfayRPfRWixw4JgOWRJgxIlAqjd+RXno3EUTK1ATG6AGgMqfB", - "qA7KE50kgitrJo44F9JYdjWGvz9FUpFkCZq9MSOfwp5vKbGqGdzOdE8KQ2UN9deSbYL0gRhMMQDXeKSv", - "6Xtw9jELuu8ErA+F8N8KOpkQoakCGyZrnkYNWTtwGqIvRXrUZhU/z1o1yyqejVrw5i54Oi9NVDF0DYcg", - "QK/zAuuZ/JLW5jKxn9aLvvhVd2o4d9nL378I++kLd/m9FGs6LzhXN81FnmP4Q0sLXlh5iVRLAQqr0xE0", - "jki4zQiBxnkH7i3dwEPOMoBLYQd16QS+PUTo32103F0nJn7YuFXKElAqRVITKrU6fec3gYG3k7fznqND", - "b5C385uKV4K8i/cXN/pNRSqV7ICu3MJ3n5nztgKUTHpOSGNRF6BkuJ51JFiqKL23bZqpSXbE70mCt2/P", - "a8jvDuw/tP4GKkMBWH6TnYmNdnlbSJyouXtc5OPKA6CkHyEYw5f4IfMhuL18Czd4Xv966OHwtPZx/UcF", - "ojt7v8/LtJ4cP/yyQ0WaK10sm/rW6WIRTOmM1BvdyxRsQZQI0k14Ao8roQGYhYe7yxQWvclHZIe3uars", - "vxB1KY5JiEIqSKCiOaJMceAIZo6fJBJcawLwnYu5z5hepNzngseHdjcr7kNLU9YYlr/5xvNuiBXuzhy3", - "WWJC+4KXdve2rRkeogy9eIra5FoJk3EXjbXmg+g4Aym5DggJJeDkRnHBW/0ayyb9SIaTUZNVLsmd/Nrm", - "pkZBKhWP3dmfHKM2ThXvTgjTZ6FF/TFIsongMxqa0o05UGc8MlDdqgHounZXLVTY8L5cuTCLuxcZpsmF", - "NPlIkzJbMK4LrYPWiDIMi1uZpbhMUyagSs+HKYQ15LTjMKf14wqzml/bKTsaE7WS44CoODep8TZ+XHMP", - "+ZorOqa6O6102zUrrtfMV7WhC+ltJMzN/Jjv1mz9/ttxr6TyQXpWWtP5LFNI68zm3xYK9u/ufrhrc/n7", - "B+yO/4I45btgKocB9Ig+hHnJAxyhkMxIxBOou2fatjqtVEStg9ZUqeRgczPS7aZcqoP9/n6/9fnD5/8b", - "AAD//4ZlynVGYgEA", + "H4sIAAAAAAAC/+y9/XIbOZI4+CqIutkYaYakqA/LtjY6fqeWbLe2rbbOsj232/RRYBVIolUFVAMoSrTD", + "/+4DzCPOk1wgAdQniizJlmyNvTsxI7PwmchMZCby42MQ8iTljDAlg4OPgQznJMHw56FSOJy/43GWkNfk", + "z4xIpX9OBU+JUJRAo4RnTI1TrOb6XxGRoaCpopwFB8EZVnN0NSeCoAWMguScZ3GEJgRBPxIFvYBc4ySN", + "SXAQbCVMbUVY4aAXqGWqf5JKUDYLPvUCQXDEWbw000xxFqvgYIpjSXq1aU/10AhLpLv0oU8+3oTzmGAW", + "fIIR/8yoIFFw8Ht5G+/zxnzyBwmVnvwwU/xcYRZNlmc8puGyudmXlGXXMBvCmeIJVjRE0vRBKXRCEyxJ", + "hDhDOFR0QRBlE56xCL05OkMhZ4yEejA5YnwiiViQCE0FT5CaEzTnUkEbJXB4iRSexGQwYkGvdh6E6S/R", + "eij9Y07UnAjPYqlEdhQ05QKpOZWIMv01JIPygSmRkSZkewGNYjJWNCE8U01A/cKvUMzZDLblxkVJJhWa", + "4wVBH4jg6M8Mx3S6pGzWDqQJmXJB0C/LlCSYoTTGIZGIKkSZ4m43BkYFjj1KfMhFZ4wLMo6IVJRhPf44", + "5cJQRHX1r+APHKNSW1gatEdqjpXDcsYVuiQkrW4UX+HLKhh/39npPR0Oh+97AVUkMWSFr2mSJcHB/qNH", + "u496QUKZ+fd2vnrKFJkRoZdvf8FC4GVpO5JnIiTjkEZi1U7CmBKm0NHJ8etbbiDYHg7g/7eeBL1g++nO", + "YHv/Cfx7ez8ob6sB+OrKP60mvXOFVSabPMhQ09giyriEJM1d/5YlEyIQn6IwE4IwFS8RkBSJOiBdZdtD", + "31GEnE3pLBOOBH0kVwHnHEuEmWEa/Rq/KAbrRHehZmIRv2JjQRJMmYZxYxGv3SekKRRZItJLCjlTgsex", + "ZgpKkSRV0lFRT7NxhnCaxjQE1lMhqr1kKINewLI41h9rKyxOm8R0RqFBJ9BQWTok1xcpjghTROQU3gU0", + "FbbYNnEBbu9pFHyxOxeUlIX+7bI6zBPN4QUJzXbzG6ACkQkJeUKQHrp6AjvDnf3+cK8/3H+z/fhguHcw", + "fPQ/QS+YcpFgFRwEEVakrw+8yzGt5t9HBZR0Q2QbFleVB3aDGg/uhi4xliqnaiByqpZj7FnTG5oQqXCS", + "asLWaygBs42s3YD1c3CQXwng7c8CMCPXamwh5N2PDz/IdUpCfcVwR575ja3H6yE6RRjlPECjq2GMKzfy", + "9LM2IgiWesFa7tC30+9BxmSW6ruQROM0xkqPq4UUQINxQqXUXfMfIioNYfYCh+RjxtVYZIyZhoyoKy4u", + "yy3tKGOaBr1gjuV4MUuzoLfqHqgiNUxBYpxKGM+euBgTIbgIjKy5HE+5cIekL7EChCuGakBI5neWB0JB", + "L6gAIOePbi9u3fmpehcHswAuCSOmG7kaNtNceHms5nLzpa3mlIYtG6nUHTOynWWVA0QUzxiXioayE9+E", + "21gfb8IjD+s8zodDNCJM0SklwgqqBImMwbXmBkF6EEQZymSNDnJZekwWWvkZL/bGKkybQKlpCuXDK132", + "xRVTuuby488pZQ2SVvfu1UQWmAJNHpMFNVdLVRiyRzOOBF0Q4WHf+Y1qWKFphzY0rWsWwjgjmxVIsQWN", + "KO7CDiJY05h6sOfs6ASZz+jkGG3MyXV1kp3HkydB+5AMJx5c+CVLMOtrgtDLcuND2/LYL/e8Mj9Pkmw8", + "EzxLmyOfvDo9fYvgI2IgMpZHfLLjE/3SkI5xFAkipX//7mN5bcPhcHiAdw6Gw8HQt8oFYREXrSA1n/0g", + "3R5GZMWQnUBqx2+A9Ld3J8cnh+iIi5QLUILWEk4ZPOV9ldGmeio+/P85o3HUxPqJ/pmIcX6J+AB24sSo", + "k2MnJ9h+6N0p2tA8JCKTbDajbLbZBd9DrsGhrzrfJQ5LRbaNVhOVk1Jufd+GguA10+kWnSZrklpmTnKc", + "yLbRXRPNURMax1SSkLNIluegTO3vtW+mRDDmhmpM9Uz/jBIiJZ4RtAEmFVA/DDPVgs0U05hEm92E2bbN", + "/MEnpSukgt6AFn08Cbd3dr28I8EzMo7ozNrE6leU/l2jmB5HIWjt3whc5t32AVMKMm3O9xxYN0wiyJQI", + "onH8M6dLBV8Qhq328heYN/i/tgpj4Za1FG4BMM+K5p96wZ8Zycg45ZKaFTY4l/2i0QhAjaCHf83wadVZ", + "lzBKKixW0we0+AKUWMh1a2FjzRZatMGztV3e6DZ13gmsMZclSlyglUU+00KNRzrgTNkPNfMln6GYMqNx", + "aNHOnAXIVcuU/BRzYIlfCA45+JvEr9d9C+ZlfmgZTX/r5QJ4zGdlaM4JFmpCKsBsucLsQMXqWsF/ViGf", + "2l2FJRmv5iBnlDESgb3YErZpqcVYr5oBVHRJ1XhBhPTSHCzrV6qQbdE6VMzDyymNyXiO5dwa2KKIGmPh", + "WWUnHmmtYojHoI+7AUGKAP31/JfDnUf7yE7ggaG1XOoGzZ2UeuvhTVuksJjgOPbiRju63fyObmKIHwMK", + "Y2Xb3ZNjoENMw+kCe5pWT87k3PwFvFuvCu4+zQY0esX67/eeTR8BkzBaQuvrjV8GzC3Ds5hrmC5Rxuif", + "WUXAHqCTKRiI9UVBIxL1EIYPYHfQ+t+MMCI0nyosQyUhGG2QwWzQQyMtF/a1FNzHO/3hsD8cBVUxNt7r", + "G/U+xUoRoRf4//2O+x8O+/8z7D99X/w5HvTf//0vPgToKpk7qdDuc8PRfg+5xZbF9fpC14nyt+b+5eX7", + "OI456hPNJ2560kcnTcHB7DXi4SURA8q3YjoRWCy32Iyy64MYKyJVdeer235RWMA+VgCBzTSYbgiGmtID", + "aLwR8ysiQs2BY6IRT/Y0E6ZK9hDWejMwL6Rvyf9EIWaaFoxwwQUiLEJXVM0RhnZVaCXLPk5pn5qlBr0g", + "wdcvCZupeXCwv9vAc43kG/aP/vu/uZ82/48X1UUWEw+Sv+aZomyG4HP5Wc+tIX+iWXUiDrpZDGJeQtmJ", + "6bbdfIP6vBN2G1l10kaZaz1qzYRyE9mahTTfd7WylXhUh1cLIgSN3LV8dHqMNmJ6SSy9IJExNMqGw90Q", + "GsCfxP4S8iTBLDK/bQ7Qq4QqfR1mxS1vnmxrr2sknHMQVOKY3+Q5DSRFUHBwvPIeXwUaL7SP8nGbt/4v", + "XKp+ghmeEVBHbUM0EfyS6IWaNwFKJLokSy3lLNFMD9pfUAkvPIQt0AIbq8NgxN7MuSSmifskwbZPFwQl", + "PLw0T79zDpr8AscZkT10NdciB9gECY7tz8g8jI3YXC9ShjwlkVZCTDPYGrogbHGBEpwCmWNBgMZRghUR", + "FMf0g3nCh1cGElF9w40YAcJAKdY0H4ZcRPDCxhHB4bwEhb9KdGEElgsY/oIyjdYXhjBrj9Ufg1dv3/z8", + "6u1vx+NXZ89+OzwZ//rsv/XPplNw8PvHwLhq5JLKzwQLItBfPsJ+PxnxNiIiOAgOMzXngn4w1ppPvUDD", + "QGr8wikd8JQwTAchT4Je8LfyP99/eu8EMmPGXmgy8Czsk1cYMnephyUdO2ugRNbC5N42NMg0i3px9nZL", + "384pllLNBc9m8yphWNHgRiQRUXk5pnw8SX1rovISnWy9QlpwQTHVBJoLKtvD4enPW3IU6H88cv/YHKBj", + "Q7WwfM2DuLDyk5xr9Mm9Po7O3iIcxzy0NpRp2wOvm8rH4AlTYply6lPiasypaNrkUf1+8fUGrGhrQtmW", + "1MfQD28Gd8CbW6sSz9iCCs4Src4tsKD6npZVWvnt1fGz8bPf3gUH+iKIstBaJc9evX4THAS7w+Ew8CGo", + "xqA1PPDF2Vvz6mnIRqVxNhtL+sEjShzm+0MJSbgwKrTtgzbmVUnD0C2CwxkFuy9+Nsi1/QLwyh2KfSPK", + "RzED1571Xvzsw5b5MiViQaXPzvZL/s2dfNPdp4Lb5pUsR1rA4kFJfwljnkX90pS9YEoFCcG9Qv/rT5Jo", + "QX7xofos5ennN391EmDXSKY4TikjK0TTb0REvOLiMuY46m9/YQnRPqh6XGPMh+r55i9rDiUaHmcTzKIr", + "Gqn5OOJXTC/Zw1ftF5Q3zpnrtd4Jjv/1v/98d1roWdsvJqnltNs7jz6T09Z4qx7aa0PJN5Kl/m28Tf2b", + "eHf6r//9p9vJ192EEURuJdTZ839mRqg7zVhfQmMObXkZzm/v3GFFcatQQ3fkcG/tM7CPUfMFETFelhiv", + "XVOwPQTuV1uVoOAliWw/zUYvke68hg3r0dwl/6Ku5O8M/YzWsyjPmn7WvMLeC11Wki9ke+fU/rnTXFLL", + "ii5pOgapeYxnuc13lUvo+SVNrSgOPcwxxrFhBFEGwvuEczUYMeOhos8ODphckxB4nlRYocOzE4muaByD", + "hQiYSvNq0YJ9ybUJmkul/1tkrIcmmdLSOlcEWb0JJslgLdB4QlDGsHsPr8nOdoNN9wIAyyURjMRjIxvL", + "jpAxnZDt1Aoc2OoUS+uiJlSWVuF1/OvpOdo4XjKc0BD9akY95VEWE3RuvAs2q9DrjVgqwE1BT6Lpmdp5", + "+RTxTPX5tK8EIW6JCQyW29jsY+3ixdlb+9wvNwcj9ppowBIWWUdfd+NYJ9CIs79qiiVRddjy/DWgt7l0", + "SIZTOedqnObO06u407ltXqji3Y0JvWARpln1SHd6rU6gCypUhmPNayvipPeB3zixe9QG4yNfVl8s3yuc", + "ZlX1ZbarxcWMDB7tXndZj+HESEqdDSclVb5hQnF65sdui10z/glzC1lpOCpUzc+Y69wM0nDeMT/33M5u", + "AaWTHCY1c9OXAc+hLKnmnZzPjQ+WkQgl2rjQ2rzFY62/X/TQxd8qP2jad6qFli+ukIEG8BOmfyqPXzdK", + "rDUX3Mjdu3w4WN7+PA5lq6cTWmwjJTCTxkdtjlMyQL8AE0eKJKnmZGyGqES5axdi/Oo/ETdCjes6Ynpp", + "0viJWHDkRiNJZ4yy2aYW8/XFhKPIWJammcqEbregsoBmFXWc9abh1WpWRww/hggJysI4iwi6cBaei6pc", + "2LT/NFVCaxBqaDgGJKDZgLKntpJM6en1hhOswrmGE8+UcRyzW6869dWsTOseVO1a8qe2W5z/ec4u6oEw", + "C4+KozdnH3nALFiyT7aZAa2g4jdRXpIlHLkzR+KGQbJsifTbCwWRPF4Qe+2WbZkTCPXhRnAqzJjGIGlt", + "kJr860EuPuvcuqPQ8OoM/qqq4AnxkarvNltgjJX+nU+440J6c2a+nlaMJQHgg+pxgEAcu+gZXYmABQIx", + "jSwxiqggoWoMT9lsxMCH5ML+MrCjXWgi1zLKFwmcgjgEENrLR4tKJ+vEPhhGb40nVCkS9aqywSUhqVy/", + "KS1eW8O1x7ouyJWgjpE5p+KO4hlhUy5Cklgl4fMUx2elwbxq3M2GaLp0GPiW1uziMyA6hUTGf8icB5hZ", + "K2Eb9ejFqKa1GReC6pQXOI4v0IZttIkE+QM88e1ZMc4KZH9zdOZQIH/2fnfa0xipucDFXKl0rP9LjjUV", + "X9QHs30dhReRZU+GoF/t7e3aU7VGN7Pg2rBV+5rXLaL9aJz43fqypvFCr9L6mXQR5Y+KLoUl9ZKyqOsA", + "v+q2rda5XDBymsZdG+hSQfpZOhMYXGy/pHnu1u+mAM12Dr4mjtfnJllECGZS8aTsb79Rc/GgVWeQKrAW", + "PO5HWGEwZXa0t5rlNh2Pk6UZyuhibZaY8Wzi8RuiHyAUYEZneLJU1feDbW803+c+Yru1+I6lzYHfaJAk", + "Giu+2oWZTpFr28Vj0cQbKD5eTClfHd5h/V8q8XfmOrJ6rR6in4bUmhNAxgnnxsPUAAGExnen5be7wYj1", + "4fo9QMf5BPmw+ZAYZEscmZeTDS5KizCBHGiy3EQYvTsdoDf5av8qkVZYFsRFNMyxRBNCGMrA9Ay3Yd/c", + "xeUFZBIuTVXvbm0nJvhhE54ouf02yGOOwUqTR1CDq9SE1vZjIifhoOybMGZlK1gnq9Uqx+/XZEalEjW3", + "b7Tx+vnR7u7u07r9cudRf7jd3370Znt4MNT/+Z/uHuJfPr7DN9ZhlbdY57My9zl6e3K8Y42l1XnUhz38", + "9Mn1NVZP9+mVfPohmYjZH7v4XiJA/KzsuPCaQxuZJKLv2KTGKp+vXMklrcUX7tYubnfksVY44K5qayDx", + "Rre8i9AWn9O0ddm9efBJnWGudbsuba6pyS9T0DsLKilJcNa7MaReP85jKi9/FgRfQshe895O8IzIsbnP", + "/P4MmTRONuTaWjcE52oqzbtp1eq5vfd478nu/t6T4dAT0dFEeB7ScahvoE4LeHV0gmK8JAJBH7QBD14R", + "msR8UkX0R7v7Tx4Pn27vdF2HeeLpBodc8XK90IaFyN9dnhL3pbKonZ3H+7u7u8P9/Z29Tquy9uJOi3K2", + "5YpI8nj38d72k529TlDwCfTPXIRNXYD3RVYemuh+/a++TElIpzREEKODdAe0kcAVRvLXqipNTnDk4k/9", + "d4fCNJYrPSbMZLalMbQlWaxoGhPzDQ6kky0adn4MI3kzZDCWx/vebCQbl7TWQ8DtJW+CKvFlFdCdmoDm", + "kvBESRwdGApdy+fgNIuFvW/DA7uHjtjwUqtO/ZgsSFxGAnN1mchaQVCOJ+bQKruibIFjGo0pSzMvSrSC", + "8nkmQBY1gyI84Zkyz4w2QLuYBLyeQfeYanbdTc99zsXlWv9RfRPncehrrUKHYEifWlMN3OIY2d4uRKEk", + "9OXPgebR1H6X6LXpYSxExc9pVs1q04OZrCWJIUGk4sBJrcHQDtNVuvTLLWAsde4fZr6Cd96T70t/atwF", + "vqyGLWYE8i+otRKLxpQ30P4cmnd2R9cd1xpSOsCdkav7ADr46/c12vYlw+ndQHyVM1puaygawS0saEQG", + "CKgLvGJcfGCN0s4VT1MS5fafwYhZf+78J2leUHRHAwc1J1QgLuiMVieuGtju0qvtJqjosOnW6Fju2JRQ", + "4SO4b7QTPZ4qk2vh0oVMkXL8kj2EoBec55kpLCeqguZ1nt2jAZHC1bKxxBdnb2/qm5YKPqW+fEPgC2G/", + "Ws3MeW293Bue97f/H+OBqfENRDTKjP9EwqNaIgnbvtvN8+Ls7VnbmvLUDqi8usaeco+XVcmtHETso5J9", + "lbQajEN/fbHkkxSy91OfLDsVOCGTbDolYpx4jGvP9XdkGhjXJsrQ6c9VeVbLzV215rPK4YDaPMWhjczv", + "Bn2PQa62jV4Jmu/9x/WamGu4LZ5PH5WwbWxI3wD9lifTQC/O3kpUeCl5LHXV4231lz+bLyUNcWxGNOG5", + "lJUNbICcnSXks6KjNUV65GR/DhZHCGhjMUszIMPz1/2TV++2kogsepU1gWfRnMdEr3uzxC0WLqqvcO6v", + "MIlFm6XDIIbsSkAlWOUU3BlIJXr1QEdxheOxjLnPWeON/ojgI9p499xEXekV9FBaOUr9ewkKFfze91KM", + "5kht057DhHWTaYXAvbpjNRumMa+UtleZ1EcqvxAcmySgVXxuJkDil9WD5pfrk+6YQXzznjjH8JpS4wve", + "Ojo9NgJDyJnClBGBEqKwTTlacnEBcSjoBX19R0WYJOBqN/3P1d4tLSb4cjRWqxH3qJG3404MuC3x5q+N", + "C0KEEszolEhl480rM8s53nm0f2CyYkRkuvdofzAY3DRG5VkRlNLpKLaMC38pXGUg5593DncQitJlLx+D", + "s8M3vwQHwVYmxVbMQxxvyQllB6V/5/8sPsAf5p8TyrwhLJ0SqdBpI4FK9UlT31nm94NSzkuX369TXju/", + "PgOeDRA35403Vnim9RODcZ8bWHzr1CNF/itVSjlSdgjtkH6EflhtCXWCEbSxc2ZM0bjIzNK0gd4qt45c", + "mX6gkXogJSxPOBDH5q+Qs4WmCl/2gQoDd98+6/3AermMI+rB5H9Ybc84SUBU1Xp6C7Zwmq5HW7+gmPO/", + "rllXbGy05yb66lz/Nm9s1dlfzf7rz/9Xnj3+Y/vPl+/e/ffixX8d/0b/+1189uqzIqhWh8V/1dj2LxbO", + "Dg9LlZj2rqh0ilXoEajmXKoWCNsvSHHjrzlAR6D4HYxYH72kiggcH6BRUHMRHgVog1zjUJleiDOkh7KR", + "Dpu685kx/+jOH51u+ak+RmRDGoQ9kDySSWaTiCeYss0RGzE7FnIbkfCmr/+KUIhTlQmiT0/LsPESTQSk", + "9bbqeTF5D33Eafppc8RAwyXXSugdpFioPI+HmwGQwq7K+AzY5iRygeFGQx6x/F7K48KNjWaQG0HANl/3", + "uPQDxau+cFENxXky9EXQg9eXPsiYSkXAMTvHbI1GuTsaejKssIonwyfDtQJ+jkMr0A8ooZnv3yFlB1oy", + "CAxTG8YNHmodbOmaNxkaQb+8eXOmwaD/9xy5gQpY5EdslDzjAyiNjVDFsuT9txl4s43C6XbckDGSQbe4", + "Q9TQM+Me+ublOVJEJM5hfyPU4JzSUO8Pnv+plJlGRYrR4dHps81Bh4IFANt8/SvO8U2+w3pwhzWatdkC", + "c4zX8O2hk2Nwz7UUWghw4FbznAsUGwZT0PUBeitJ1dcVjsq86puTjJeF5c3cAKNg042Y1jnFAXqdy404", + "X0qlSELVmFfQJQxrH16Mz09j9F4j/bhwepFlbeDhg1XuJK5v3HZWsJr8PRAHmrd+3SWb5s1ou2wM1ZP5", + "UaM4+y+dNeXLizu7N1Vyb5rhoRqEWQrgzZM8dM/OcBdZDpoK3zVV49ZXfKQ/2zd7p9a8O0VzLNlfFXys", + "KTfbu4875evUs3Z9/y6/fPOpWVJOli6iM3+3NbGtlzSOjTuEpDOGY/QUbZyfvPj15OXLTdRHr16d1o9i", + "VQ/f+XRI9uBo48XZWwiXwXLsnpDavSZx4XlMrqlUshnw2ukldnVyiV8qCSC8EcSbXzArhHu+bmzjPvI9", + "fE2/wG8v18TK7BCfm+LBSst3lOGhlbn6siNU+az5+cvmariT5awtL1IWKpzT9q2TI/QC6nFYPZSaBZII", + "nZwVSRYLq5YbvrYnW6tnezgcbA+72PgSHK6Y+/TwqPvkwx1jyTjAk4MwOiDTz7AxWsQ20h+Or/BSopGT", + "z0eBUQhKmkCJbK0M3+n9tpmD4nYpJ+oCxbqkEjdJItEtO8TnhuSvSrV8Xk2y3FnI+4xKJJ1cKNzVbp0n", + "bK/xTcznBIU8iyMtSE006RrFjkRW/5REFfmrgdrfskvGr1h168aKqhnAnxkRS/Tu9LRicxdkatPzdtg4", + "OF20nANPb3QMO2tk7bWruWWihvtIzlBnu6Xr7ounYigb/ZwTp8HQDsa/Qvz0PrxTZo5G48mKPdXMNhFZ", + "jLPMJ1XpTy504+3bk+MKcmC8v/1k+ORp/8lke7+/Fw23+3h7d7+/8wgPp7vh492WBPndHW9u70tTpeb2", + "UCkAPJhATSRcdKDpLXeGmWQK5Y5ympCPtHiKSnKwCQwCq8QJowqSQFI208OAkcCKySbC0+SppIwqSCkA", + "CW0o01sGa4wexLo/HaAX0BY+4QQCltwitHJUNUTgaGkMsZoxuKlT+NfqJZ/PMyj3A33kPFMIykPpbWsw", + "WHVl9RCGxxyg3zj0Ec5LlfG63mOag02g2byuI21YvyTnvwqTWYZ5gJ7nTDJns5atbkhi/zS827pWg9v4", + "ZsV5z554oLGlOLmSX1ovMBANeoEDFPivNT3Z7Lq8QRplVPS9UBAcAwstPIUyRWObJQF2QqFAEmwEw+G2", + "UbLNCEaisREB2t4bjfuJFRPyTo5RvDtFGxAP+XdklUr9r838bbJMlXs7T/ee7j/eebrfKeqhWOB6Bn8E", + "zlHNxa3l9mGajV3tkZatH529NSUOQ85klhgrgd17yck0FTzU0iplqChmUkz+dPC0HOwR8cwUdrJLspFh", + "n0rly1ZWnml5YPuTxgs6nbI/P4SXO38Immxf78udiVe5K+qkeSXhk7KptaE2kknfZHH0++MDQgnZGrLy", + "mkjYATonCgH+9BEO4ZLOfZosyrnAFgtxL2Lt7e7uPnn8aKcTXtnVlQhnDPprc5WndgUlEoOWaOP1+Tna", + "KiGcGdM5ekKCCWYFOD+dIZvQeVitBDrYHu76sKRFXiqwxo69SFpB/s4KQXZTFujgmpULSA0q90J7d3f4", + "eO/Rk0fdyNjV3RPXqzmMS+phwGPzoJRPfgPM828Oz5AeXUxxWNVQtnd29x7tP35yo1WpG60KcviY3Bs3", + "WNiTx/uP9nZ3trvFXvlM8DaqsEKwVd7lIToPUnhOwwOKJuvttd0WPsHTINhrEsaYJoehc5+p3T4mx8ZY", + "mGbFIXS5GKyRoHFxdejbSUWrFQ4yogEXqFRxcbDeHHo762Y7mzb3wXo23pShY8w0uGyQgEnleAvYpYIs", + "KM/kFxiIKxJqZJrGnIsb9W3zR3pNZBYrY4KkEr07/SswEY1cSCqSVn3tLfqtCKW45eZuRMAVnPBjdRuw", + "Op1Gl6NfteFeC5n2VvnRVsi/NWIp0qwqY+vfvo9wHGaQvAzn56l3BbEHPFPwUr80XiJxzDlD4RyzGYFk", + "8CZVIpshjOY8jgaB/6kkjsZT7xNGXmGeF/XL3SJ0N1d/f+MFL0raGVSq5ed9lBiuYjM3DTrUki9q4rZE", + "OGl4YsVLaQBMl4o2H/OZBC1Qgf/LoJ59JsXCuLVgZvLULRKjPFZDt3b0be9ZYo17+65Qc3XyqdVorYyh", + "eA5JHAouZVGY+91pdZmrHBjzevbr37Ori+2AujLlTBJ/mXhbE76Twcd3IXo8wz7nSgQcBgfQVTWgbR7D", + "BLMMMn2VEJlcp1QY9Oj2OD7nUo3zcJQbLlaqMWRxygQpYtbcfTmHAIClYXHQxnsvOtZ2G3Dl5Y1v0buB", + "Vf6h2hbYzlO9EPVDq5fjoA+NmwE5K2OAiqCiegTJTULGirQ/VMKotBSthDYYVxW2VEpds9nlocqvo+p5", + "2krKvtwbnneN5lodvHWG1fyETbknN+QNDP7WJd75LqREQPlxzlBEGCWRUx5zy7+1bYGTfSwJijJiIWcE", + "UoEtwLEhb8gByZxRjLJZjdfXJ+xihjdrWJ3kCea1Dbs8OUq/a/YbkQGsjJOARLhw0u7k8UDl2G8pbg4s", + "yCyLsUD1iMUVS5bLJKbsssvocplMeExDpDvUn3OmPI751Vh/kj/BXjY77U53GBc+hrXnGbM462FqDqQ2", + "b7GFn/QuN2v+7WB62TL9t3T/Ti+4Xr+h5zQmNqjvLaPXJUSvZkHZ2xm2hT60DFoJemgGhN6Uc1uU9VG8", + "i9U8zKsmeNzxjQdQ7VWiaois7Ne3W3AxWxXo0TTFoA33KOyyzFThWsr20skS0s3Lre7+4FazJUlYnX3v", + "yaPH+x3T7XyWrXNFVeXPsGwukhUWzZaTOu1iNnvy6MnTp7t7j57u3MhA5TxlWs6nzVumfD614ig1o9mj", + "IfzfjRZlfGX8S2rxl6kuqFLo5NYL+rSCdIsw65Znj9Y833H5JN07S9UC2s3GuEJaOqyIXKVaXhtkOiWg", + "VI4N3PrFYmru+Z3WEOIUh1QtPQYTfGVSvudNauHCXaxp1cV6QGrHthkfNOeS2aRw6Nxwk6O/GdN6DRee", + "dM7aJbNJmxn/VX1WY8QvbEDlJ6IOLzRFYYGmuSDfzxWWFa8O/XcIOZuLWm11/yHTontVaofreWHqwjPS", + "F/LuL0JdPv7acZbMvhUhuQ7xVVdoOwneSIf23Mi+MpXrvXJr/MFegLfrNZ6U8+mtTFhYSb5X3Lo3n7db", + "lblmP3OD3Xy+kgvoTTrWU4sBPto1WJAXY/cqKNGCTYqL9Rml7yBBkPEpuFWKIOuOcC9ZguzPd5IZqHEc", + "50S5tudao8/iFQmhmSJigT2GKTcEck2qdlTDiXvImvjQdrJZK1W4N/fLajaEt6P7mJYCx6kgU3q9AltM", + "A3NdV93HpYVAVE0aLtFGgq/R3mMUzrGQtbUzOpureFk1su55oic+r4gzUVp07p5evThN17H5omGPszy6", + "j2TPS7EO/rTvJBqvCnQ/yps5m3GKlyBbtiqCj3f3hsPdneGtIt2/VDb60jhtHqGlftaYU3l6LI+Q+382", + "UxZeCWqKmjkwSSUITg7AmyrFIUExmUIcWJ4qdq1O35h69eLtI6l1/M/x3x2Uq1xq7SyWxTHOQPBw49gk", + "AW4bgXulrcZ6lL83l70iWCxnM2Ejaqzuv7rfH+72h/tvtncPHu0fbG/fRWh8DqQ2F57HH7avHsc7eLoX", + "P1k+/nN7/ni2k+x6wwXuoPBBrY5grQ6C3UNKRD0XZT2HqyQxZaQvc7e39Q7IK3iBeUlaS/83sz6YHawU", + "Fs6rmyzLDFgVwKmXZLuPwCa7+pUmlPryT45XL/tWfmT1hfgRrL4UwKdui4FULdufmxckYx3vnbelhp1v", + "npW+jevuHp/TN5C295RbIO7D5wpjrFDYqhu7eat5VLgZF1TNk9XXQ94szzIAj+EfpIqqgTQDdDJjkHm2", + "/HP+9lEuDq07B70g/rBXpRn7e/eQKhtVnyOgPeqyGNDhbQASG6+GAjQpVAth3BOwIACIn7b720/hhT7+", + "sPfTsP90gP5R8hToGWiVwbftWld+HXaBYc4oQe40L+fbT2/0jO7guQqDfrX3UttFbOPtLY4XaT/dXeHc", + "pisHXHxunHEtquiOCg19WrFjJzjfLG2P6+URTQZ+2eTpm+H2jdP23OyKGHyGR/FnqXrd1LsYS9UmV7/E", + "UuX6GBKZla57UOKG1Srd27TiNhQAXDsh1v4AyZqG7/Kpy0LocVaAHppxhYoggLVSDixfZMyLD9X1F2WK", + "IT1EK0LsrEGIbmvKA/noKtI9OUap4FEWFh6wMSw6C0Mi5TSDqsuDrhLt+jfGu1TmwbVca/Trlfk27X19", + "mCm5bj/v38i1Kk2pEbb9qLeH64/6TiwAvSBLo/U8zDTqxsFulIljjU+lxx5RBXtNCipt5n0Hjv66DMGm", + "ggcFm1CoxYEsdRUENU41MUl66gbi67E3R8AxiYkivkGQKelp3T6oLLjoepa6vf/EbzLD1+MQAhIbC/mV", + "kFTL6QmXtsRmgtnSu7B6Fm20MXQVJCWC4fsmk5eFVnVxj9dKIa1H1T0hec2iayK+yvnf8wwdXzYbue25", + "tlbE3Qkqb6yq1JZRpu7rWbY7Hvb/x9gZ0XhwsPXT3//v/vu//cVfm6WiSEki+hGZwgPYJVn2TdFZrbQN", + "qvlMIdtNIBW2FU0UwQlYEcJLYqwWCb4ur/fRMKek5W84aWwBXg4TyvJ/r93Q3//S/u5WAuNbYB5rz/Gz", + "sx/dRWpZxR2P3kiImLn87c5dDMp5s3ipj0qiUv46e887sv6rzLuUC8deGNloAOWOJxTSgMoR02oODkOS", + "KhINbB4vCmsRHEiyXj7Z5tFz7t2aWDEkD7XhNrU0WR+9RYoPAkau+maGqK9xb+/Rvq0YX4bkduOIfYdu", + "IrbbKixqKHvMCC+phHAE53Vbaow2SJKqpcsS6/wiN28WQX6YD+h9Cv3C6bOGT79EttC3K9ODfof1PcsB", + "/m5Ba0P7G+ffmpPP71h1XM/UY2jS1iyrZpapqUxS9dv9rhJ9x4/BfbDpI6W/GddEmw9zltUTg28lTG3Z", + "7Lu+cIgIShKvdEYtqMxFu/eh03ofy5VSZmlnpZW0n82pE6bqZaLbAXSmQXM1J4KUDgI6FClEbwgy6yjY", + "IcjG5MhMiejXi9qZuguCgudhrgc7EOTOpE3D2OoMN6f4Op8BjKpYNp4eYB9FrrftFz9DLZXXrrgZnboh", + "YBk1UdefrqaKRV3KszcPo4xVzX2b9l7Cs7xqBfdro60achZzVFDTh4//wFQ95wKE4/aQljvPegOCd0QE", + "xPTWc9p0SghDExKNeaZW079NMm/jWSI0IVMuSCn/rlMEMCCxrXm8hhe4oItiDe99coMkYSaoWmrN0Yqk", + "E4IFEYeZIXgAJEwEPxcTQzrbT5/AhDb1eLC9IIwIGqLDsxOgR6jbr4nj3SmK6ZSEy1Ar4JCNtJF/A4S8", + "V0cnVvlyGd/gQZEqQD1Xhvjw7ASqmgqjgATDwc5gCMScEoZTGhwEu4NtqPGqEQ62uAXZ7+FP65xu4tIo", + "ZyeRlYN+Nk10L4ETooiQwcHvHidvRYTJpi9B6sSzkt6QYiqs4pDG4HpuUIXqvpD+yF2lB+Y+7hmAd7Yd", + "SbW0jngkfWWP9b3GBEM1sMWd4dDoaUzZixcX1S63/rABe8W8neQ5AI8nF1BDrncypQX5p16wN9y+0XrW", + "Fqj0TfuW4UzNuaAfCCzz0Q2BcKtJT5jxDkYmz4T1fyjTGaBQmcJ+f6/PS2ZJgsXSgauAVcplmzBMJMJQ", + "Is+UcviDTwbIWsYh/amc8yzW3AQZ12cS6QsLa54ymH1AWIRzuiAjZu9pU2wUC8gvnSB9Pxu1pUoaZmpz", + "+nlQ2s88Wtagmw+3pYfrO3toAeB6mkVJxpAtatxWp6WwhFLGoN6jJDYnZV6woOlkAQV6Zci9lYkJw0wV", + "9V5NZd5LsrTGVu+AndK6aIYHx0KgEHyer3xn0x/PANkz/aFAx/k3ZMFbFScYvFCEcRYVMpdzscViguPY", + "G/c/i/kEx7aA8SXxiKgvoIUFSjnRqBNuGI+ISRqZLtWcM/N3NsmYyszfE8GvJBFaBLLZpy2sbfVOi7pQ", + "SZ4mkAHa1LbQc26ZJW59vCTLT4MRO4wSV7dE2uLzseS2srNJn0Mlcj6ZBnf96U1bnvuPMql4YlGKlQtR", + "mmXyTKWZsk+dkiibMhuaQ51SOSfRiCmOPgpTln75aetjMeMn0F0IjjSelJqYLW19pNGntlXLMda7H0NT", + "j/ZHAACjQN8uo0D/PRNY6y6ZnIMpQ4L5YlY+0o08FlvLhZt1CIeYoZSnJo4dkMoUrK6MAeUHcBwjBaTk", + "+mppE06yZT82NMVXS8/GpZhAghoZQVW9EjEN95746UmSUBCfgeO/zl/9huCq0mdgmhVmI4ARZfoWRVEG", + "kjzMPhixZzicIyM3Qa6yUUCjUZBrF9EmrDWT1nG23wcR9ye9tJ/MND0a/TQY6KGM9HyAfv9oRjnQtJQm", + "Y8UvCRsFn3qo9GFG1Tyb5N/e+wHa5t5/XmEEaMPw/k1XPAbSDBTXoLk3MIsQt7w2XiKMCg5UtqNMKMNi", + "ZeUbD+gtBLUqj2eyDIyPI7CgjoKDkbOhjoLeKCBsAb9ZQ+so+OSHgBWi2xNjmeI/TtbOkWh/ONxcH3dn", + "4esRoSsNNfl9akhfO19M8LBCV1PwMJtzWf30CZoyTkbcugfJ52ccucIAP0S8NSKetVyUhDfoX74HDPrG", + "xCi4NQlM67Oxk8BWaicGLSCtJWgcLkrWKBzUSXAF8pbVj7o631Qr9tqoLIQlxg7/9u4B/2DeohQ6zPv0", + "vubFMeSozAsDPyx0hMNyiNjza8QviPoWMG54X6zUJtT8mvj7UPDnBbFyXwG0GjfbIgv33uTPBQAxANKO", + "YhprXfUc1tQ/J0yhZ/DrwP6v03ggs+1FzGcXB8iAMOYzFFNmX+NKr0X6UrSwhE4mDCDvZ6MCXCKmDXN/", + "/ut//wmLomz2r//9p5amzV9A7lsmOwYkbr2YEyzUhGB1cYB+JSTt45guiNsMpFYkCyKWaHcIYmYq4JOn", + "2KQcsRF7TVQmWOnV0uREknZAUD0Y7IeyjEgbRqEb0qlN2GAMzB4V3tGyAeW9UnSv6XNqdlDagL4VHQ5A", + "BC412Wut/hX4rWdmzxX7Wd1W3rCYrucvilwrg719s8AbMhgAsY/u4IPdNNo4P3+2OUCgYxisgKQcIDEX", + "w1jhefCDJ63nSYajVBkKQNnwplKd8Vb777Ft080AbEf8nizAbYXT203AxuRBBIkcvH7oCl3MwX64OdOw", + "zz577ILn2g20t99veQrnTdRJEf5y5+xwrwlz86UEsq+hAqMN56jtav6dHZ242jCbXw3p7+XW0Du1FRXy", + "qwNxU2nw3tSyI86mMQ0V6ru1QFL/hOSqWhVBHgo7eG1XjbDbVz39Xfl+26pkc2m96fLELsWVd/e3R23S", + "m1wjRYq+Atd+3CTrUOeYypDrviVs6Yc4tQULjfiS02kZi9YZpIzfd37lrBSXLHs+OXYEeX+mKTt1xup3", + "wz0wxeMaQ/yKjLBWhK2U1PIhYfPb/BRdooAVlqtvCzWH9ycF3bcVy4fmD8mMFdXAprmgSe3beoG+IOoX", + "0+IOD9rO4Nn4ORGOql0OYth1vi3TFYVzEl6aDcGD9Grd98Q06ab6mvG+J80XwHMTicWC/IeI0kHZLWC1", + "SsE9sYXl7k6/hRlupN5+uXdei2AeIIOzycRZrE3NNiyXLNz8rp567+U2M8B+kJfZWRbH7sVjQYRCef3m", + "8h2w9RHcktbL9o7aVl4Hb1+/7BMWcvBDy32o/EKU/fKFJXxzYGYrP9Cki05oInapu8/aJJzPOH/jLojy", + "+uD/sfPcVgj/j53npkb4f+wemirhm3eGLMP7Ys33LXE/YOTTAjetAg1YE4Naoesk1LxVRyHVtf+u5FSz", + "6RtJqjlcfwirXYTVMrhWyqv2KO5UYjVzfKUnmRzZfNCGT84/8TuTVO/Xymcx0uXvpbL67GELtHABdl74", + "RBnKJHmADpQ0x7jytdHRXF0Q5Mrrw6HuyXEPANnToIOEQjZA5J6M124d9y7c2nnv33J9mEzoLOOZLMee", + "JFiFcyJtsFJMqgz4oYndxfXcKnh/w1g6vM+r497l6h94f0cSf/1ADfM2L1DrZH7XqqvMb9trmd8Wzzex", + "a69dUX6bJ2mzxanQBVF3ReNKrHnT2dG3Lp8ugt5qRaVQFxBoEAcj9n+0/vG7Ijh5/5MLksmGw519+J2w", + "xfufXJwMO3WoQpgS1CbvPPztGJ79ZhB9Dvk9i5C8+jpMQQBAPZfA5t9OQSpePrtrSA4Lf2hInTSkErhW", + "a0j2LO5WRaomwbp3Hcnhmw/gNonJDy3pPrQkmU2nNKSEqaJeVsNJzJbbe4CxZcy+D5WcOyoXbWctKSfK", + "NQJoka393h178snvXzlyieEfpo88N1ExkVNHisuwXR/51vBheL/M+f71kIeMYkbgb4Iu1TKlrwwjZHpM", + "MgVOiUWGEPD6RMJI7fmIA1RUP5RZmnKhpMkWCQIwlMNScy0A+zJLVpNF+rJDQmJcSmRvxCCBvP5sYvm3", + "LsnS5IKkvKjqn+/U5n/0xV5Vc3F+VTL68jKWP9FoJxnrnsnY5lP+ejLWV2Md9yJpnVTS1G/khAEK5YTk", + "lMzz4D76gbLZ5oPyQDXMKt9bKZ+RR9TaggJsNrvulswLvbZdtKUEu7Y84b/hjdvcpE9qd7loSwBEEcUz", + "xqWioQvcrSfy/nFDd76hV0PWi81TW17Tr9A/5+Ky6xXnKff0AG668g6/QVuCXh5kA/v6JgVQts1toJHm", + "3m/BRg2vrxmCQev3Yhhnkb4I3YXoRMmp4MnY/mjy1WqqsNlAwUQR2lG/NrPRs9+Dweg3rhBN0phoKZ5E", + "qG+wSZ+mFf1d0ncqSxXvbsYMNdmUA2JMMjrpquZYFgmPa+7ANuCdvXlcXq4Z89n6JBj55C7jgycLxoiZ", + "pPTEZbC/QDmTRYojSWISKnQ1p+EcMmLo30yhTUhWgdP0Ik+BtXmAXgClljOBweQbkgitCIWcSR4Tk+hi", + "kSQXB82Mre9OT6GTSYZhcrNeHCCXpTW/IKRuVc5wkZfj+c3m7djQmCR4HJsTvdBaY2l/mzb3RZGibMR8", + "eTAYubID0im6KKXEuGjJieEY6ks++2rSVq89saTZi+JIAOAMbhIWBW0PMTT2Z8PYHnpLlXTMzGGWcceJ", + "ORqLeclneVLLCirjNO2KvnaZgMWLJFmBw2ijVDNTqohn6u9SRUQI6Gyxuw250QYOzT8UvtSIamve5FVH", + "Af28z40my5wXVJqplgq8mH8tkiToBXY9pex0N5De12Q4qQ/YfBbTJ1NKY/JD7r5JgpIqsy9lKKndHLYq", + "e7vIbYvNf/f2WQuo6HuwslTfs4pVUOZEFThbXhSIelCZDuAgG7KYKU/koxG3y74s1bXs9rzVqIj5DSit", + "61698vKGee3F+37+aq7gIQfByMZuplzUw+PXvYt984j05Y6ksdUuGPIDN29unuuEmGm2oswlVOmUYOeD", + "0o+Q1zmccy5LaD8hc7ygXNgM7NbqmmMmmCyM9mi95y40ql5Y++2FFc8PrK0J4fInO8cAulufO38P96no", + "8bykbeccv+dEasgCKRFGE0HJFKU4k0RLS1lCkKkwYhN5ExzOXRHnwYi9mRNkSzeWDAh5pV8q0cV2ctFD", + "k0yhGIsZaDvmo/GkEyTkSUJYZMqxjtic4AXVqppAMVaEhcu+JFCed0GKAiZadbcvlAgeKPMCoD3k6saC", + "geGiVBX2AqWCABIZdZlVSrCOmMjYf5rMlXrYC7fQC0SkwpOYynleKyLEEWGhNy3k+bfNxr68EfecqGbh", + "1K/yZnkrXvo1HzHLtsy8dPU38b75wBy1uHD1LTuw+RVCr2xXDauej+dFsdh/Q5I2e3V7/EovMzmIV1Hx", + "t/EkU6kW/+NZRlmSjDIzHalWVP9u31pyhoIyVnlusTbZ2z645JUQcjDfiOdtfXR/ntzCRvaNcMJeq2Lf", + "lnO72PS3wHItVG/Fc7+ScdDakkpWsa/Igu2ivp74xEWJy30TbNgQXM6NyzxHCQw6FWc/mHGdGVv3gNsy", + "Y2dxbTyAl9gzZf00xm18uagd72fA1iDwb+r9WttdiRF+dcZXvAjcG7M7ydmbYXgpXsYcf+/vMiEXwgR0", + "2nLEDyehWMkWWHpg2gCLWy/nED0XTfLu9HSzjUsItZJHCPWAOUS1rGmYeKo1vloQIWjkSkcenR5b71Uq", + "kcjYAL1KKNRzvCQkhUIxlGcSQWTuQO/PhbY2i+BVYlh7AWFKLFNOmVq7iqLp3Szm061K590zn7QpFb/7", + "x2Owwj88JgW8Q4srdgOrtUiFVasznnNOo8zUu9TSFp7wTI+uOYsrtDuDu21KYyKXUpHEeOZNsxiICJLu", + "2ppMtp+JKO0hqiTS9NCDCLyUiIRKSTmTI2bLv6dE6Ll1dyj+WzgZeY33Cudc88ywvm/DgU0vxvhsYdUG", + "NUgtAHVAg4NgC6fpFpSL9jtJ2eV9xpKeg0cakstkwmMaopiyS4k2YnpplA60kCjWf2yudGkbQ78vXXHq", + "9pSlIX3CptxblMPgbI7M30cQUpWtuUfEB8fWXpAysTj+AwftZ2tyLV8TBMd9RROSB7+jTNGYfjCsTg9C", + "paKhiaspQi+hCLONvhyxU6KEboMFQSGPYxIqZ1zZSgUPt0bZcLgbphSylOwSWBwwvPbPCcx4dPYW2plC", + "0b0R0/+Agd8cnpmX2Cm2NoLSQhlRV1xcopOtV2ucfM8BTP/GXnJmgytjIL0H/uP57uaRza00JFtIlKer", + "FCCefvdunFaC+2EteJjWAkgtke9mYyZwCEKxnGcq4lfMbxlY8DhL9D/MHyfrEpQoHM7fQdNvRto1y1k7", + "jdvggyBKu6eImKJBX+WBwgDsofqXasC5LYAQU/Hc894Ch+p7xO4vb5Qvw/EbfJq0EHUFub4Z2rrvm8+u", + "weXdKsPjoZC5wTS3E8VXW5+uMG23Pv0c8/BSoowpGleSGmi9DfKA6h+LvI324Q/EBIiOdKXEEblOqYAM", + "NrX0CIjoHUuEkSIioQzHW7BnMwhkoHRWLLzgFIKUw5hCmBiNCEp5HEOWnas5YUjvBgxVboDSO620FSDK", + "bcpPjIqjCQl5QlxWzk2f6vYPTNVzLqopNr8VvvimBH+9H71Vvc81WUXbZ/ysLKOn+BrcmqPMPhO7FW28", + "4MWPxhTUQ3A2o2B3KEdBD42CnWQU6BM4wmBCxQo9QgllmSJygI6NfQvCUPeHSJKQs0i65KDOgrc7lG1B", + "qQYtWyIc96HffYo9FqsAlK/tJD72oNsh3R8CbNBGmeAsTUY9ILoI8UyBA7ejK9sqIgrMI5v3/gJbopEf", + "un0XTv4PS74VHgWnrNll6egNZ8/TR661urmgijmXRdZJFOIUh1QtewjHMQ8L60Em89eBfr6UiSD4UutQ", + "gxF7nSeutIEQ6Ojsbc8ZzVBE5aUZwdrFBujVggiZTfLFIeAGxoIHh0GiEVMchTgOs1jjLZlOSQgxDDFN", + "qJItdrV8KXdZBrGYxHPw7mOetuZhGZP8OAGnV6CFrGHcljnqLUHCGNOkbFSqAwdEX3jSBbPvRA/K9TU8", + "je3zVii4lMgO1ScxndFJbB9r5AC90SIHTsiIpTFmjAiUSeN3pJfeTwWRMjOBMXoAqDNrMKqHikQnqeDK", + "moljzoU0ll2N4e9OkVQkXYFmr83Ip7DnO0oTbAa3M30lhaG2hvZryTZB+kAMphiAazzS1/RXcPYxC/ra", + "6YQfCuG/EXQ2I0JTBTZM1jyNGrJ24DREX4n0aM2Rf5636pYjPx+15M1d8nRemahi7BqOQYC+yQusZ/JL", + "2prLxH66WfTFr7pTx7mrXv7+RdhPn7nL76X02HnJubprZv0Cwx9akvvSyiukWglQWJ+OoHNEwl1GCHTO", + "O/DV0g085CwDuBJ20JZO4NtDhOH9Rsfdd5rth41blSwBlcI6LaFS69N3fhMYeDd5O79ydOgt8nZ+U/FK", + "kHfx68WNflORShU7oCse8t1n5ryrACWTnhPSWLQFKBmuZx0JVipK72ybbmqSHfF7kuDt2/MN5HcH9h9a", + "fweVoQQsv8nOxEa7vC0kSdXSPS7yae0BUNIPEIzhS/yQ+xDcXb6FWzyvfzn0cHja+rj+o57Wvb3fF0WH", + "T44ffhGtMs1VLpYtfev0sQjndEHaje5VCrYgSgXppzyFx5XIAMzCw91lCovB7AOyw9tcVfZfiLoUxyRC", + "ERUkVPESUaY4cAQzx18lElxrAvCdi6XPmF6m3OeCJ4d2N2vuQ0tT1hhWvPkmy36EFe4vHLdZYUL7jJd2", + "97atGR6iDL34GW2QayVMxl001ZoPotMcpOQ6JCSSgJOb5QVvD1ssm/QDGc8mXVa5InfyK5ubGoWZVDxx", + "Z39yjDag2MKMMH0WWtSfgiSbCr6gkSlEWgB1wWMD1e0WgN7U7qqFirxShlMuzOK+igzT5UKafaBplS0Y", + "14XgIJhQhmFxa7MUV2nKBFTp+TCFsIaCdhzmBD+uMKv5bThlR2OiVnIcEBXnJjXe5o9r7iFfc2XHVHen", + "VW67bqUiu/mqdnQhvYuEubkf8/2ard99O+6VVD5Iz0prOl/kCmmb2fzbQsHh/d0P920uf/eA3fFfEKd8", + "l0zlMIAe0YcwL3mIYxSRBYl5ClUkTdugF2QiDg6CuVLpwdZWrNvNuVQHT4ZPhsGn95/+/wAAAP//tsVd", + "yWFyAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go new file mode 100644 index 00000000..f03066ef --- /dev/null +++ b/lib/providers/auto_standby_linux.go @@ -0,0 +1,112 @@ +//go:build linux + +package providers + +import ( + "context" + "log/slog" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" + "go.opentelemetry.io/otel" +) + +type autoStandbyRuntimeManager interface { + GetAutoStandbyRuntime(ctx context.Context, id string) (*autostandby.Runtime, error) + SetAutoStandbyRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error + SubscribeLifecycleEvents(consumer instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) +} + +type autoStandbyInstanceStore struct { + manager instances.Manager + runtimeManager autoStandbyRuntimeManager +} + +func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { + insts, err := s.manager.ListInstances(ctx, nil) + if err != nil { + return nil, err + } + + out := make([]autostandby.Instance, 0, len(insts)) + for _, inst := range insts { + runtime, err := s.runtimeManager.GetAutoStandbyRuntime(ctx, inst.Id) + if err != nil { + return nil, err + } + + out = append(out, autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + Runtime: runtime, + }) + } + return out, nil +} + +func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string) error { + _, err := s.manager.StandbyInstance(ctx, id, instances.StandbyInstanceRequest{}) + return err +} + +func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { + return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime) +} + +func (s autoStandbyInstanceStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { + src, unsub := s.runtimeManager.SubscribeLifecycleEvents(instances.LifecycleEventConsumerAutoStandby) + dst := make(chan autostandby.InstanceEvent, 32) + go func() { + defer close(dst) + for event := range src { + dst <- autostandby.InstanceEvent{ + Action: autostandby.InstanceEventAction(event.Action), + InstanceID: event.InstanceID, + Instance: toAutoStandbyInstance(event.Instance), + } + } + }() + return dst, unsub, nil +} + +func toAutoStandbyInstance(inst *instances.Instance) *autostandby.Instance { + if inst == nil { + return nil + } + return &autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + } +} + +// ProvideAutoStandbyController provides the Linux auto-standby controller. +func ProvideAutoStandbyController(instanceManager instances.Manager, log *slog.Logger) *autostandby.Controller { + if instanceManager == nil || log == nil { + return nil + } + + runtimeManager, ok := instanceManager.(autoStandbyRuntimeManager) + if !ok { + return nil + } + + return autostandby.NewController( + autoStandbyInstanceStore{manager: instanceManager, runtimeManager: runtimeManager}, + autostandby.NewConntrackSource(), + autostandby.ControllerOptions{ + Log: log.With("controller", "auto_standby"), + Meter: otel.GetMeterProvider().Meter("hypeman/autostandby"), + Tracer: otel.GetTracerProvider().Tracer("hypeman/autostandby"), + }, + ) +} diff --git a/lib/providers/auto_standby_unsupported.go b/lib/providers/auto_standby_unsupported.go new file mode 100644 index 00000000..f1180785 --- /dev/null +++ b/lib/providers/auto_standby_unsupported.go @@ -0,0 +1,15 @@ +//go:build !linux + +package providers + +import ( + "log/slog" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" +) + +// ProvideAutoStandbyController is unavailable on non-Linux platforms. +func ProvideAutoStandbyController(instances.Manager, *slog.Logger) *autostandby.Controller { + return nil +} diff --git a/lib/scopes/scopes.go b/lib/scopes/scopes.go index e92ecde0..f41aafb1 100644 --- a/lib/scopes/scopes.go +++ b/lib/scopes/scopes.go @@ -243,6 +243,7 @@ var RouteScopes = map[string]Scope{ "POST /instances/{id}/start": InstanceWrite, "GET /instances/{id}/stat": InstanceRead, "GET /instances/{id}/stats": InstanceRead, + "GET /instances/{id}/auto-standby/status": InstanceRead, "GET /instances/{id}/wait": InstanceRead, "POST /instances/{id}/stop": InstanceWrite, "PATCH /instances/{id}": InstanceWrite, diff --git a/openapi.yaml b/openapi.yaml index dae2115b..b1884bac 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -203,6 +203,103 @@ components: $ref: "#/components/schemas/CreateInstanceRequestCredentialInject" minItems: 1 + AutoStandbyPolicy: + type: object + description: | + Linux-only automatic standby policy based on active inbound TCP connections + observed from the host conntrack table. + properties: + enabled: + type: boolean + description: Whether automatic standby is enabled for this instance. + default: false + example: true + idle_timeout: + type: string + description: | + How long the instance must have zero qualifying inbound TCP connections + before Hypeman places it into standby. + example: "5m" + ignore_source_cidrs: + type: array + description: Optional client CIDRs that should not keep the instance awake. + items: + type: string + example: ["10.0.0.0/8", "192.168.0.0/16"] + ignore_destination_ports: + type: array + description: Optional destination TCP ports that should not keep the instance awake. + items: + type: integer + minimum: 1 + maximum: 65535 + example: [22, 9000] + + AutoStandbyStatus: + type: object + required: [supported, configured, enabled, eligible, status, reason, active_inbound_connections, tracking_mode] + properties: + supported: + type: boolean + description: Whether the current host platform supports auto-standby diagnostics. + example: true + configured: + type: boolean + description: Whether the instance has any auto-standby policy configured. + example: true + enabled: + type: boolean + description: Whether the configured auto-standby policy is enabled. + example: true + eligible: + type: boolean + description: Whether the instance is currently eligible to enter standby. + example: true + status: + type: string + enum: [unsupported, disabled, ineligible, active, idle_countdown, ready_for_standby, standby_requested, error] + example: idle_countdown + reason: + type: string + enum: [unsupported_platform, policy_missing, policy_disabled, instance_not_running, network_disabled, missing_ip, has_vgpu, active_inbound_connections, idle_timeout_not_elapsed, observer_error, ready_for_standby] + example: idle_timeout_not_elapsed + active_inbound_connections: + type: integer + description: Number of currently tracked qualifying inbound TCP connections. + example: 0 + idle_timeout: + type: string + nullable: true + description: Configured idle timeout from the auto-standby policy. + example: "5m0s" + idle_since: + type: string + format: date-time + nullable: true + description: When the controller most recently observed the instance become idle. + example: "2026-04-06T17:04:05Z" + last_inbound_activity_at: + type: string + format: date-time + nullable: true + description: Timestamp of the most recent qualifying inbound TCP activity the controller observed. + example: "2026-04-06T17:01:05Z" + next_standby_at: + type: string + format: date-time + nullable: true + description: When the controller expects to attempt standby next, if a countdown is active. + example: "2026-04-06T17:09:05Z" + countdown_remaining: + type: string + nullable: true + description: Remaining time before the controller attempts standby, when applicable. + example: "4m0s" + tracking_mode: + type: string + description: Diagnostic identifier for the runtime tracking mode in use. + example: conntrack_events_v4_tcp + UpdateInstanceRequest: type: object properties: @@ -217,6 +314,8 @@ components: are accepted. Use this to rotate real credential values without restarting the VM. example: OUTBOUND_OPENAI_KEY: new-rotated-key-456 + auto_standby: + $ref: "#/components/schemas/AutoStandbyPolicy" CreateInstanceRequest: type: object @@ -323,6 +422,8 @@ components: snapshot_policy: description: Snapshot compression policy for this instance. Controls compression settings applied when creating snapshots or entering standby. $ref: "#/components/schemas/SnapshotPolicy" + auto_standby: + $ref: "#/components/schemas/AutoStandbyPolicy" skip_kernel_headers: type: boolean description: | @@ -766,6 +867,8 @@ components: example: cloud-hypervisor snapshot_policy: $ref: "#/components/schemas/SnapshotPolicy" + auto_standby: + $ref: "#/components/schemas/AutoStandbyPolicy" PathInfo: type: object @@ -2925,6 +3028,39 @@ paths: schema: $ref: "#/components/schemas/Error" + /instances/{id}/auto-standby/status: + get: + summary: Get auto-standby diagnostic status + operationId: getAutoStandbyStatus + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Instance ID or name + responses: + 200: + description: Current auto-standby diagnostic status for the instance + content: + application/json: + schema: + $ref: "#/components/schemas/AutoStandbyStatus" + 404: + description: Instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /instances/{id}/stats: get: summary: Get instance resource utilization stats diff --git a/stainless.yaml b/stainless.yaml index c88f36cc..8f9645cd 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -72,6 +72,8 @@ resources: instances: models: + auto_standby_policy: "#/components/schemas/AutoStandbyPolicy" + auto_standby_status: "#/components/schemas/AutoStandbyStatus" snapshot_policy: "#/components/schemas/SnapshotPolicy" snapshot_schedule: "#/components/schemas/SnapshotSchedule" snapshot_schedule_retention: "#/components/schemas/SnapshotScheduleRetention" @@ -100,6 +102,9 @@ resources: # Subresources define resources that are nested within another for more powerful # logical groupings, e.g. `cards.payments`. subresources: + auto_standby: + methods: + status: get /instances/{id}/auto-standby/status volumes: methods: attach: post /instances/{id}/volumes/{volumeId}