From cb66913f4088b7a95e97cc52847730a1b3f5de34 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Sat, 4 Apr 2026 11:35:58 -0400 Subject: [PATCH 01/13] Add Linux auto-standby controller and e2e coverage --- cmd/api/api/auto_standby.go | 62 ++ cmd/api/api/instances.go | 19 +- cmd/api/api/instances_test.go | 112 ++++ cmd/api/auto_standby_linux.go | 60 ++ cmd/api/auto_standby_unsupported.go | 15 + cmd/api/main.go | 3 + lib/autostandby/README.md | 45 ++ lib/autostandby/classifier.go | 57 ++ lib/autostandby/classifier_test.go | 77 +++ lib/autostandby/conntrack_linux.go | 79 +++ lib/autostandby/conntrack_linux_test.go | 33 ++ lib/autostandby/conntrack_unsupported.go | 21 + lib/autostandby/controller.go | 145 +++++ lib/autostandby/controller_test.go | 129 +++++ lib/autostandby/policy.go | 109 ++++ lib/autostandby/policy_test.go | 45 ++ lib/autostandby/types.go | 70 +++ lib/instances/auto_standby.go | 33 ++ .../auto_standby_integration_linux_test.go | 191 +++++++ lib/instances/create.go | 6 + lib/instances/fork.go | 3 + lib/instances/fork_test.go | 11 + lib/instances/types.go | 12 +- lib/instances/update.go | 38 +- lib/instances/update_test.go | 15 +- lib/oapi/oapi.go | 537 ++++++++++-------- openapi.yaml | 38 ++ 27 files changed, 1698 insertions(+), 267 deletions(-) create mode 100644 cmd/api/api/auto_standby.go create mode 100644 cmd/api/auto_standby_linux.go create mode 100644 cmd/api/auto_standby_unsupported.go create mode 100644 lib/autostandby/README.md create mode 100644 lib/autostandby/classifier.go create mode 100644 lib/autostandby/classifier_test.go create mode 100644 lib/autostandby/conntrack_linux.go create mode 100644 lib/autostandby/conntrack_linux_test.go create mode 100644 lib/autostandby/conntrack_unsupported.go create mode 100644 lib/autostandby/controller.go create mode 100644 lib/autostandby/controller_test.go create mode 100644 lib/autostandby/policy.go create mode 100644 lib/autostandby/policy_test.go create mode 100644 lib/autostandby/types.go create mode 100644 lib/instances/auto_standby.go create mode 100644 lib/instances/auto_standby_integration_linux_test.go diff --git a/cmd/api/api/auto_standby.go b/cmd/api/api/auto_standby.go new file mode 100644 index 00000000..96e20fed --- /dev/null +++ b/cmd/api/api/auto_standby.go @@ -0,0 +1,62 @@ +package api + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/oapi" + "github.com/samber/lo" +) + +func toDomainAutoStandbyPolicy(policy *oapi.AutoStandbyPolicy) (*autostandby.Policy, error) { + if policy == nil { + return nil, nil + } + + out := &autostandby.Policy{} + if policy.Enabled != nil { + out.Enabled = *policy.Enabled + } + if policy.IdleTimeout != nil { + out.IdleTimeout = *policy.IdleTimeout + } + if policy.IgnoreSourceCidrs != nil { + out.IgnoreSourceCIDRs = append([]string(nil), (*policy.IgnoreSourceCidrs)...) + } + if policy.IgnoreDestinationPorts != nil { + out.IgnoreDestinationPorts = make([]uint16, 0, len(*policy.IgnoreDestinationPorts)) + for _, port := range *policy.IgnoreDestinationPorts { + if port < 0 || port > 65535 { + return nil, fmt.Errorf("auto_standby.ignore_destination_ports must be between 1 and 65535") + } + out.IgnoreDestinationPorts = append(out.IgnoreDestinationPorts, uint16(port)) + } + } + + return out, nil +} + +func toOAPIAutoStandbyPolicy(policy *autostandby.Policy) *oapi.AutoStandbyPolicy { + if policy == nil { + return nil + } + + out := &oapi.AutoStandbyPolicy{ + Enabled: lo.ToPtr(policy.Enabled), + } + if policy.IdleTimeout != "" { + out.IdleTimeout = lo.ToPtr(policy.IdleTimeout) + } + if len(policy.IgnoreSourceCIDRs) > 0 { + out.IgnoreSourceCidrs = lo.ToPtr(append([]string(nil), policy.IgnoreSourceCIDRs...)) + } + if len(policy.IgnoreDestinationPorts) > 0 { + ports := make([]int, 0, len(policy.IgnoreDestinationPorts)) + for _, port := range policy.IgnoreDestinationPorts { + ports = append(ports, int(port)) + } + out.IgnoreDestinationPorts = &ports + } + + return out +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index b7248207..90f3457a 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -278,6 +278,13 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if request.Body.Cmd != nil { cmd = *request.Body.Cmd } + autoStandby, err := toDomainAutoStandbyPolicy(request.Body.AutoStandby) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_auto_standby", + Message: err.Error(), + }, nil + } domainReq := instances.CreateInstanceRequest{ Name: request.Body.Name, @@ -302,6 +309,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Cmd: cmd, SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, + AutoStandby: autoStandby, } if request.Body.SnapshotPolicy != nil { snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) @@ -924,9 +932,17 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst if request.Body.Env != nil { env = *request.Body.Env } + autoStandby, err := toDomainAutoStandbyPolicy(request.Body.AutoStandby) + if err != nil { + return oapi.UpdateInstance400JSONResponse{ + Code: "invalid_auto_standby", + Message: err.Error(), + }, nil + } result, err := s.InstanceManager.UpdateInstance(ctx, inst.Id, instances.UpdateInstanceRequest{ - Env: env, + Env: env, + AutoStandby: autoStandby, }) if err != nil { switch { @@ -1057,6 +1073,7 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { oapiPolicy := toOAPISnapshotPolicy(*inst.SnapshotPolicy) oapiInst.SnapshotPolicy = &oapiPolicy } + oapiInst.AutoStandby = toOAPIAutoStandbyPolicy(inst.AutoStandby) // Convert volume attachments if len(inst.Volumes) > 0 { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b3f74ea2..c31dd400 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" @@ -276,6 +277,7 @@ func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, re Name: "updated-instance", Image: "docker.io/library/alpine:latest", Env: req.Env, + AutoStandby: req.AutoStandby, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -297,6 +299,7 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances HotplugSize: req.HotplugSize, OverlaySize: req.OverlaySize, Vcpus: req.Vcpus, + AutoStandby: req.AutoStandby, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -477,6 +480,49 @@ func TestCreateInstance_MapsNetworkEgressEnforcementMode(t *testing.T) { assert.Equal(t, instances.EgressEnforcementModeHTTPHTTPSOnly, mockMgr.lastReq.NetworkEgress.EnforcementMode) } +func TestCreateInstance_MapsAutoStandbyPolicy(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + enabled := true + idleTimeout := "5m" + ignoreSourceCidrs := []string{"10.0.0.0/8", "192.168.0.0/16"} + ignoreDestinationPorts := []int{22, 9000} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-auto-standby", + Image: "docker.io/library/alpine:latest", + AutoStandby: &oapi.AutoStandbyPolicy{ + Enabled: &enabled, + IdleTimeout: &idleTimeout, + IgnoreSourceCidrs: &ignoreSourceCidrs, + IgnoreDestinationPorts: &ignoreDestinationPorts, + }, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.AutoStandby) + assert.True(t, mockMgr.lastReq.AutoStandby.Enabled) + assert.Equal(t, "5m", mockMgr.lastReq.AutoStandby.IdleTimeout) + assert.Equal(t, []string{"10.0.0.0/8", "192.168.0.0/16"}, mockMgr.lastReq.AutoStandby.IgnoreSourceCIDRs) + assert.Equal(t, []uint16{22, 9000}, mockMgr.lastReq.AutoStandby.IgnoreDestinationPorts) + + instance := oapi.Instance(created) + require.NotNil(t, instance.AutoStandby) + require.NotNil(t, instance.AutoStandby.Enabled) + assert.True(t, *instance.AutoStandby.Enabled) + assert.Equal(t, idleTimeout, *instance.AutoStandby.IdleTimeout) +} + func TestUpdateInstance_MapsEnvPatch(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -524,6 +570,72 @@ func TestUpdateInstance_MapsEnvPatch(t *testing.T) { assert.Equal(t, "rotated-key-456", mockMgr.lastReq.Env["OUTBOUND_OPENAI_KEY"]) } +func TestUpdateInstance_MapsAutoStandbyPatch(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{ + Manager: origMgr, + result: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-auto-standby", + Name: "inst-update-auto-standby", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "10m0s", + }, + }, + State: instances.StateStopped, + }, + } + svc.InstanceManager = mockMgr + + enabled := true + idleTimeout := "10m" + ignoreDestinationPorts := []int{22} + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-auto-standby", + Name: "inst-update-auto-standby", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + AutoStandby: &oapi.AutoStandbyPolicy{ + Enabled: &enabled, + IdleTimeout: &idleTimeout, + IgnoreDestinationPorts: &ignoreDestinationPorts, + }, + }, + }) + require.NoError(t, err) + updated, ok := resp.(oapi.UpdateInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.AutoStandby) + assert.Equal(t, resolved.Id, mockMgr.lastID) + assert.True(t, mockMgr.lastReq.AutoStandby.Enabled) + assert.Equal(t, "10m", mockMgr.lastReq.AutoStandby.IdleTimeout) + assert.Equal(t, []uint16{22}, mockMgr.lastReq.AutoStandby.IgnoreDestinationPorts) + + instance := oapi.Instance(updated) + require.NotNil(t, instance.AutoStandby) + require.NotNil(t, instance.AutoStandby.Enabled) + assert.True(t, *instance.AutoStandby.Enabled) +} + func TestUpdateInstance_RequiresBody(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/cmd/api/auto_standby_linux.go b/cmd/api/auto_standby_linux.go new file mode 100644 index 00000000..6a901e03 --- /dev/null +++ b/cmd/api/auto_standby_linux.go @@ -0,0 +1,60 @@ +//go:build linux + +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" + "golang.org/x/sync/errgroup" +) + +type autoStandbyInstanceStore struct { + manager instances.Manager +} + +func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { + insts, err := s.manager.ListInstances(ctx, nil) + if err != nil { + return nil, err + } + + out := make([]autostandby.Instance, 0, len(insts)) + for _, inst := range insts { + out = append(out, autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + }) + } + return out, nil +} + +func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string) error { + _, err := s.manager.StandbyInstance(ctx, id, instances.StandbyInstanceRequest{}) + return err +} + +func startAutoStandbyController(grp *errgroup.Group, ctx context.Context, logger *slog.Logger, manager instances.Manager) bool { + if grp == nil || ctx == nil || logger == nil || manager == nil { + return false + } + + controller := autostandby.NewController( + autoStandbyInstanceStore{manager: manager}, + autostandby.NewConntrackSource(), + logger.With("controller", "auto_standby"), + 5*time.Second, + ) + grp.Go(func() error { + return controller.Run(ctx) + }) + return true +} diff --git a/cmd/api/auto_standby_unsupported.go b/cmd/api/auto_standby_unsupported.go new file mode 100644 index 00000000..b568b7ab --- /dev/null +++ b/cmd/api/auto_standby_unsupported.go @@ -0,0 +1,15 @@ +//go:build !linux + +package main + +import ( + "context" + "log/slog" + + "github.com/kernel/hypeman/lib/instances" + "golang.org/x/sync/errgroup" +) + +func startAutoStandbyController(*errgroup.Group, context.Context, *slog.Logger, instances.Manager) bool { + return false +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 3430b9fe..8696d39e 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -495,6 +495,9 @@ func run() error { logger.Info("starting guest memory controller") return app.GuestMemoryController.Start(gctx) }) + if startAutoStandbyController(grp, gctx, logger, app.InstanceManager) { + logger.Info("auto-standby controller enabled") + } // Run the server grp.Go(func() error { diff --git a/lib/autostandby/README.md b/lib/autostandby/README.md new file mode 100644 index 00000000..c5b90718 --- /dev/null +++ b/lib/autostandby/README.md @@ -0,0 +1,45 @@ +# Auto Standby + +This feature automatically puts a Linux VM into `Standby` after it has stopped serving inbound TCP traffic for a configured amount of time. + +## What counts as activity + +The feature looks at host-side conntrack state, not ingress configuration and not TAP byte counters. + +A VM is considered active when there is at least one tracked TCP flow where: + +- the original destination is the VM's private IP +- the VM is the server/responding side of the connection +- the flow is still in an active TCP state + +That means: + +- inbound client connections keep the VM awake +- replies to outbound guest requests do not keep the VM awake +- same-host clients count by default + +## Idle behavior + +When the active inbound TCP connection count reaches zero, Hypeman starts an idle timer for that instance. + +- if a new inbound TCP connection appears before the timer expires, the timer is cleared +- if the count stays at zero for the full `idle_timeout`, Hypeman places the VM into `Standby` + +The timer is in-memory. After a Hypeman restart, idle timers begin from controller startup time instead of being reconstructed from the past. This avoids immediate standby caused only by restarting the control plane. + +## Exclusions + +Instances can ignore some traffic when deciding whether they are active: + +- `ignore_source_cidrs` excludes matching client source ranges +- `ignore_destination_ports` excludes matching VM destination ports + +This is intended for probes, internal callers, or ports that should not keep a VM warm. + +## Limits + +- Linux only +- TCP only +- Wake-on-traffic is not part of this feature + +Wake-on-traffic would require a separate host-owned listener or forwarding layer that can accept a connection while the VM is asleep, trigger restore, and then hand traffic through once the VM is running. diff --git a/lib/autostandby/classifier.go b/lib/autostandby/classifier.go new file mode 100644 index 00000000..1091bf91 --- /dev/null +++ b/lib/autostandby/classifier.go @@ -0,0 +1,57 @@ +package autostandby + +import ( + "fmt" + "net/netip" + "time" +) + +// ActiveInboundCount returns the number of active inbound TCP connections for an instance +// and the compiled idle timeout that should be applied to it. +func ActiveInboundCount(inst Instance, conns []Connection) (int, time.Duration, error) { + compiled, err := compilePolicy(inst.AutoStandby) + if err != nil { + return 0, 0, err + } + if compiled == nil { + return 0, 0, nil + } + + instanceIP, err := netip.ParseAddr(inst.IP) + if err != nil { + return 0, 0, fmt.Errorf("parse instance IP %q: %w", inst.IP, err) + } + + count := 0 + for _, conn := range conns { + if matchesInboundConnection(instanceIP, compiled, conn) { + count++ + } + } + + return count, compiled.idleTimeout, nil +} + +func matchesInboundConnection(instanceIP netip.Addr, policy *compiledPolicy, conn Connection) bool { + if policy == nil { + return false + } + if !conn.TCPState.Active() { + return false + } + if !conn.OriginalDestinationIP.IsValid() || conn.OriginalDestinationIP != instanceIP { + return false + } + if _, ignored := policy.ignorePorts[conn.OriginalDestinationPort]; ignored { + return false + } + if !conn.OriginalSourceIP.IsValid() { + return false + } + for _, prefix := range policy.ignoreSourceCIDRs { + if prefix.Contains(conn.OriginalSourceIP) { + return false + } + } + return true +} diff --git a/lib/autostandby/classifier_test.go b/lib/autostandby/classifier_test.go new file mode 100644 index 00000000..bc778069 --- /dev/null +++ b/lib/autostandby/classifier_test.go @@ -0,0 +1,77 @@ +package autostandby + +import ( + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestActiveInboundCountCountsOnlyQualifyingInboundTCP(t *testing.T) { + t.Parallel() + + inst := Instance{ + ID: "inst-1", + IP: "192.168.100.10", + State: StateRunning, + AutoStandby: &Policy{ + Enabled: true, + IdleTimeout: "5m", + IgnoreSourceCIDRs: []string{"10.0.0.0/8"}, + IgnoreDestinationPorts: []uint16{22}, + }, + } + + count, idleTimeout, err := ActiveInboundCount(inst, []Connection{ + { + OriginalSourceIP: netip.MustParseAddr("1.2.3.4"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationIP: netip.MustParseAddr("8.8.8.8"), + OriginalDestinationPort: 443, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("10.1.2.3"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("5.6.7.8"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 22, + TCPState: TCPStateEstablished, + }, + { + OriginalSourceIP: netip.MustParseAddr("9.9.9.9"), + OriginalDestinationIP: netip.MustParseAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateTimeWait, + }, + }) + require.NoError(t, err) + + assert.Equal(t, 1, count) + assert.Equal(t, 5*time.Minute, idleTimeout) +} + +func TestActiveInboundCountRejectsInvalidInstanceIP(t *testing.T) { + t.Parallel() + + _, _, err := ActiveInboundCount(Instance{ + IP: "not-an-ip", + AutoStandby: &Policy{ + Enabled: true, + IdleTimeout: "5m", + }, + }, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "parse instance IP") +} diff --git a/lib/autostandby/conntrack_linux.go b/lib/autostandby/conntrack_linux.go new file mode 100644 index 00000000..28cc9999 --- /dev/null +++ b/lib/autostandby/conntrack_linux.go @@ -0,0 +1,79 @@ +//go:build linux + +package autostandby + +import ( + "context" + "net/netip" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// ConntrackSource reads current IPv4 TCP conntrack entries from the host. +type ConntrackSource struct { + listFlows func(table netlink.ConntrackTableType, family netlink.InetFamily) ([]*netlink.ConntrackFlow, error) +} + +// NewConntrackSource creates a conntrack-backed connection source. +func NewConntrackSource() *ConntrackSource { + return &ConntrackSource{ + listFlows: netlink.ConntrackTableList, + } +} + +// ListConnections returns normalized TCP flows from the host conntrack table. +func (s *ConntrackSource) ListConnections(context.Context) ([]Connection, error) { + flows, err := s.listFlows(netlink.ConntrackTable, netlink.InetFamily(unix.AF_INET)) + if err != nil { + return nil, err + } + return connectionsFromFlows(flows), nil +} + +func connectionsFromFlows(flows []*netlink.ConntrackFlow) []Connection { + result := make([]Connection, 0, len(flows)) + for _, flow := range flows { + conn, ok := connectionFromFlow(flow) + if !ok { + continue + } + result = append(result, conn) + } + return result +} + +func connectionFromFlow(flow *netlink.ConntrackFlow) (Connection, bool) { + if flow == nil || flow.Forward.Protocol != unix.IPPROTO_TCP { + return Connection{}, false + } + + tcpInfo, ok := flow.ProtoInfo.(*netlink.ProtoInfoTCP) + if !ok || tcpInfo == nil { + return Connection{}, false + } + + srcIP, ok := addrFromIP(flow.Forward.SrcIP) + if !ok { + return Connection{}, false + } + dstIP, ok := addrFromIP(flow.Forward.DstIP) + if !ok { + return Connection{}, false + } + + return Connection{ + OriginalSourceIP: srcIP, + OriginalDestinationIP: dstIP, + OriginalDestinationPort: flow.Forward.DstPort, + TCPState: TCPState(tcpInfo.State), + }, true +} + +func addrFromIP(ip []byte) (netip.Addr, bool) { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Addr{}, false + } + return addr.Unmap(), true +} diff --git a/lib/autostandby/conntrack_linux_test.go b/lib/autostandby/conntrack_linux_test.go new file mode 100644 index 00000000..44ba938b --- /dev/null +++ b/lib/autostandby/conntrack_linux_test.go @@ -0,0 +1,33 @@ +//go:build linux + +package autostandby + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +func TestConnectionFromFlowNormalizesTCPConntrackEntry(t *testing.T) { + t.Parallel() + + conn, ok := connectionFromFlow(&netlink.ConntrackFlow{ + Forward: netlink.IPTuple{ + Protocol: unix.IPPROTO_TCP, + SrcIP: net.ParseIP("1.2.3.4").To4(), + DstIP: net.ParseIP("192.168.100.10").To4(), + DstPort: 8080, + }, + ProtoInfo: &netlink.ProtoInfoTCP{State: uint8(TCPStateEstablished)}, + }) + require.True(t, ok) + + assert.Equal(t, mustAddr("1.2.3.4"), conn.OriginalSourceIP) + assert.Equal(t, mustAddr("192.168.100.10"), conn.OriginalDestinationIP) + assert.Equal(t, uint16(8080), conn.OriginalDestinationPort) + assert.Equal(t, TCPStateEstablished, conn.TCPState) +} diff --git a/lib/autostandby/conntrack_unsupported.go b/lib/autostandby/conntrack_unsupported.go new file mode 100644 index 00000000..8f929d7c --- /dev/null +++ b/lib/autostandby/conntrack_unsupported.go @@ -0,0 +1,21 @@ +//go:build !linux + +package autostandby + +import ( + "context" + "fmt" +) + +// ConntrackSource is unavailable on non-Linux platforms. +type ConntrackSource struct{} + +// NewConntrackSource creates an unsupported conntrack source. +func NewConntrackSource() *ConntrackSource { + return &ConntrackSource{} +} + +// ListConnections reports that conntrack-backed auto-standby is unsupported. +func (*ConntrackSource) ListConnections(context.Context) ([]Connection, error) { + return nil, fmt.Errorf("conntrack-backed auto-standby is only supported on Linux") +} diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go new file mode 100644 index 00000000..14c9499c --- /dev/null +++ b/lib/autostandby/controller.go @@ -0,0 +1,145 @@ +package autostandby + +import ( + "context" + "errors" + "log/slog" + "time" +) + +const defaultPollInterval = 5 * time.Second + +// InstanceStore supplies the controller with instance state and standby actions. +type InstanceStore interface { + ListInstances(ctx context.Context) ([]Instance, error) + StandbyInstance(ctx context.Context, id string) error +} + +// ConnectionSource lists current TCP flows that may keep an instance awake. +type ConnectionSource interface { + ListConnections(ctx context.Context) ([]Connection, error) +} + +// Controller decides when eligible instances should transition to standby. +type Controller struct { + store InstanceStore + source ConnectionSource + log *slog.Logger + now func() time.Time + pollInterval time.Duration + idleSince map[string]time.Time +} + +// NewController creates a new auto-standby controller. +func NewController(store InstanceStore, source ConnectionSource, log *slog.Logger, pollInterval time.Duration) *Controller { + if log == nil { + log = slog.Default() + } + if pollInterval <= 0 { + pollInterval = defaultPollInterval + } + return &Controller{ + store: store, + source: source, + log: log, + now: time.Now, + pollInterval: pollInterval, + idleSince: make(map[string]time.Time), + } +} + +// Run starts the controller loop and blocks until the context is canceled. +func (c *Controller) Run(ctx context.Context) error { + c.log.Info("auto-standby controller started", "poll_interval", c.pollInterval) + if err := c.Poll(ctx); err != nil { + c.log.Warn("auto-standby poll failed", "error", err) + } + + ticker := time.NewTicker(c.pollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if err := c.Poll(ctx); err != nil { + c.log.Warn("auto-standby poll failed", "error", err) + } + } + } +} + +// Poll runs a single reconciliation pass. +func (c *Controller) Poll(ctx context.Context) error { + instances, err := c.store.ListInstances(ctx) + if err != nil { + return err + } + conns, err := c.source.ListConnections(ctx) + if err != nil { + return err + } + + now := c.now().UTC() + seen := make(map[string]struct{}, len(instances)) + var reconcileErrs []error + + for _, inst := range instances { + seen[inst.ID] = struct{}{} + + if !eligible(inst) { + delete(c.idleSince, inst.ID) + continue + } + + count, idleTimeout, err := ActiveInboundCount(inst, conns) + if err != nil { + delete(c.idleSince, inst.ID) + reconcileErrs = append(reconcileErrs, err) + continue + } + + if count > 0 { + delete(c.idleSince, inst.ID) + continue + } + + start, ok := c.idleSince[inst.ID] + if !ok { + c.idleSince[inst.ID] = now + continue + } + if now.Sub(start) < idleTimeout { + continue + } + + if err := c.store.StandbyInstance(ctx, inst.ID); err != nil { + c.log.Warn("auto-standby standby attempt failed", "instance_id", inst.ID, "instance_name", inst.Name, "error", err) + } else { + c.log.Info("instance entered standby due to inbound inactivity", "instance_id", inst.ID, "instance_name", inst.Name, "idle_timeout", idleTimeout) + } + + // Reset the timer after every attempt to avoid retrying every poll interval + // when the standby transition fails for a transient reason. + c.idleSince[inst.ID] = now + } + + for id := range c.idleSince { + if _, ok := seen[id]; !ok { + delete(c.idleSince, id) + } + } + + return errors.Join(reconcileErrs...) +} + +func eligible(inst Instance) bool { + if inst.State != StateRunning { + return false + } + if !inst.NetworkEnabled || inst.IP == "" || inst.HasVGPU { + return false + } + return inst.AutoStandby != nil && inst.AutoStandby.Enabled +} diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go new file mode 100644 index 00000000..07c70273 --- /dev/null +++ b/lib/autostandby/controller_test.go @@ -0,0 +1,129 @@ +package autostandby + +import ( + "context" + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeInstanceStore struct { + instances []Instance + standbyIDs []string + standbyErr error +} + +func (f *fakeInstanceStore) ListInstances(context.Context) ([]Instance, error) { + return append([]Instance(nil), f.instances...), nil +} + +func (f *fakeInstanceStore) StandbyInstance(_ context.Context, id string) error { + f.standbyIDs = append(f.standbyIDs, id) + return f.standbyErr +} + +type fakeConnectionSource struct { + connections []Connection + err error +} + +func (f *fakeConnectionSource) ListConnections(context.Context) ([]Connection, error) { + if f.err != nil { + return nil, f.err + } + return append([]Connection(nil), f.connections...), nil +} + +func TestControllerWaitsFullIdleTimeoutFromStartup(t *testing.T) { + t.Parallel() + + store := &fakeInstanceStore{ + instances: []Instance{{ + ID: "inst-idle", + Name: "inst-idle", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.10", + AutoStandby: &Policy{ + Enabled: true, + IdleTimeout: "5m", + }, + }}, + } + source := &fakeConnectionSource{} + controller := NewController(store, source, nil, 0) + + now := time.Date(2026, 4, 3, 12, 0, 0, 0, time.UTC) + controller.now = func() time.Time { return now } + + require.NoError(t, controller.Poll(context.Background())) + assert.Empty(t, store.standbyIDs) + + now = now.Add(4 * time.Minute) + require.NoError(t, controller.Poll(context.Background())) + assert.Empty(t, store.standbyIDs) + + now = now.Add(1 * time.Minute) + require.NoError(t, controller.Poll(context.Background())) + assert.Equal(t, []string{"inst-idle"}, store.standbyIDs) +} + +func TestControllerClearsIdleTimerWhenTrafficReturns(t *testing.T) { + t.Parallel() + + store := &fakeInstanceStore{ + instances: []Instance{{ + ID: "inst-busy", + Name: "inst-busy", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &Policy{ + Enabled: true, + IdleTimeout: "1m", + }, + }}, + } + source := &fakeConnectionSource{} + controller := NewController(store, source, nil, 0) + + now := time.Date(2026, 4, 3, 13, 0, 0, 0, time.UTC) + controller.now = func() time.Time { return now } + + require.NoError(t, controller.Poll(context.Background())) + now = now.Add(30 * time.Second) + source.connections = []Connection{{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalDestinationIP: mustAddr("192.168.100.20"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }} + require.NoError(t, controller.Poll(context.Background())) + + now = now.Add(70 * time.Second) + source.connections = nil + require.NoError(t, controller.Poll(context.Background())) + assert.Empty(t, store.standbyIDs) +} + +func TestControllerSkipsIneligibleInstances(t *testing.T) { + t.Parallel() + + store := &fakeInstanceStore{ + instances: []Instance{ + {ID: "stopped", State: "Stopped", NetworkEnabled: true, IP: "192.168.1.10", AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}}, + {ID: "vgpu", State: StateRunning, NetworkEnabled: true, IP: "192.168.1.11", HasVGPU: true, AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}}, + }, + } + controller := NewController(store, &fakeConnectionSource{}, nil, 0) + + require.NoError(t, controller.Poll(context.Background())) + assert.Empty(t, store.standbyIDs) +} + +func mustAddr(raw string) netip.Addr { + return netip.MustParseAddr(raw) +} diff --git a/lib/autostandby/policy.go b/lib/autostandby/policy.go new file mode 100644 index 00000000..f763b290 --- /dev/null +++ b/lib/autostandby/policy.go @@ -0,0 +1,109 @@ +package autostandby + +import ( + "fmt" + "net/netip" + "slices" + "strings" + "time" +) + +// NormalizePolicy validates and canonicalizes a policy for storage. +func NormalizePolicy(policy *Policy) (*Policy, error) { + if policy == nil { + return nil, nil + } + + if !policy.Enabled { + return &Policy{Enabled: false}, nil + } + + idleTimeout := strings.TrimSpace(policy.IdleTimeout) + if idleTimeout == "" { + return nil, fmt.Errorf("auto_standby.idle_timeout is required when enabled") + } + + parsedTimeout, err := time.ParseDuration(idleTimeout) + if err != nil { + return nil, fmt.Errorf("auto_standby.idle_timeout must be a valid duration: %w", err) + } + if parsedTimeout <= 0 { + return nil, fmt.Errorf("auto_standby.idle_timeout must be positive") + } + + normalized := &Policy{ + Enabled: true, + IdleTimeout: parsedTimeout.String(), + } + + if len(policy.IgnoreSourceCIDRs) > 0 { + seen := make(map[string]struct{}, len(policy.IgnoreSourceCIDRs)) + for _, raw := range policy.IgnoreSourceCIDRs { + cidr := strings.TrimSpace(raw) + if cidr == "" { + continue + } + prefix, err := netip.ParsePrefix(cidr) + if err != nil { + return nil, fmt.Errorf("auto_standby.ignore_source_cidrs contains invalid CIDR %q: %w", cidr, err) + } + canonical := prefix.Masked().String() + if _, ok := seen[canonical]; ok { + continue + } + seen[canonical] = struct{}{} + normalized.IgnoreSourceCIDRs = append(normalized.IgnoreSourceCIDRs, canonical) + } + slices.Sort(normalized.IgnoreSourceCIDRs) + } + + if len(policy.IgnoreDestinationPorts) > 0 { + seen := make(map[uint16]struct{}, len(policy.IgnoreDestinationPorts)) + for _, port := range policy.IgnoreDestinationPorts { + if port == 0 { + return nil, fmt.Errorf("auto_standby.ignore_destination_ports must not contain 0") + } + if _, ok := seen[port]; ok { + continue + } + seen[port] = struct{}{} + normalized.IgnoreDestinationPorts = append(normalized.IgnoreDestinationPorts, port) + } + slices.Sort(normalized.IgnoreDestinationPorts) + } + + return normalized, nil +} + +func compilePolicy(policy *Policy) (*compiledPolicy, error) { + normalized, err := NormalizePolicy(policy) + if err != nil { + return nil, err + } + if normalized == nil || !normalized.Enabled { + return nil, nil + } + + idleTimeout, err := time.ParseDuration(normalized.IdleTimeout) + if err != nil { + return nil, fmt.Errorf("parse normalized auto-standby idle timeout: %w", err) + } + + compiled := &compiledPolicy{ + idleTimeout: idleTimeout, + ignorePorts: make(map[uint16]struct{}, len(normalized.IgnoreDestinationPorts)), + } + + for _, raw := range normalized.IgnoreSourceCIDRs { + prefix, err := netip.ParsePrefix(raw) + if err != nil { + return nil, fmt.Errorf("parse normalized auto-standby source CIDR %q: %w", raw, err) + } + compiled.ignoreSourceCIDRs = append(compiled.ignoreSourceCIDRs, prefix) + } + for _, port := range normalized.IgnoreDestinationPorts { + compiled.ignorePorts[port] = struct{}{} + } + + return compiled, nil +} diff --git a/lib/autostandby/policy_test.go b/lib/autostandby/policy_test.go new file mode 100644 index 00000000..451c5eea --- /dev/null +++ b/lib/autostandby/policy_test.go @@ -0,0 +1,45 @@ +package autostandby + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePolicyCanonicalizesValues(t *testing.T) { + t.Parallel() + + normalized, err := NormalizePolicy(&Policy{ + Enabled: true, + IdleTimeout: "300s", + IgnoreSourceCIDRs: []string{"10.0.0.0/8", " 10.0.0.0/8 ", "192.168.1.1/24"}, + IgnoreDestinationPorts: []uint16{9000, 22, 9000}, + }) + require.NoError(t, err) + require.NotNil(t, normalized) + + assert.True(t, normalized.Enabled) + assert.Equal(t, "5m0s", normalized.IdleTimeout) + assert.Equal(t, []string{"10.0.0.0/8", "192.168.1.0/24"}, normalized.IgnoreSourceCIDRs) + assert.Equal(t, []uint16{22, 9000}, normalized.IgnoreDestinationPorts) +} + +func TestNormalizePolicyRejectsInvalidValues(t *testing.T) { + t.Parallel() + + _, err := NormalizePolicy(&Policy{ + Enabled: true, + IdleTimeout: "0s", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be positive") + + _, err = NormalizePolicy(&Policy{ + Enabled: true, + IdleTimeout: "5m", + IgnoreDestinationPorts: []uint16{0}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain 0") +} diff --git a/lib/autostandby/types.go b/lib/autostandby/types.go new file mode 100644 index 00000000..49ef8a68 --- /dev/null +++ b/lib/autostandby/types.go @@ -0,0 +1,70 @@ +package autostandby + +import ( + "net/netip" + "time" +) + +const ( + StateRunning = "Running" +) + +// Policy configures per-instance automatic standby behavior. +type Policy struct { + Enabled bool `json:"enabled"` + IdleTimeout string `json:"idle_timeout,omitempty"` + IgnoreSourceCIDRs []string `json:"ignore_source_cidrs,omitempty"` + IgnoreDestinationPorts []uint16 `json:"ignore_destination_ports,omitempty"` +} + +// Instance is the minimal instance view needed by the auto-standby controller. +type Instance struct { + ID string + Name string + State string + NetworkEnabled bool + IP string + HasVGPU bool + AutoStandby *Policy +} + +// Connection is the normalized network view used by activity classification. +type Connection struct { + OriginalSourceIP netip.Addr + OriginalDestinationIP netip.Addr + OriginalDestinationPort uint16 + TCPState TCPState +} + +// TCPState is the conntrack TCP state for a flow. +type TCPState uint8 + +const ( + TCPStateNone TCPState = 0 + TCPStateSynSent TCPState = 1 + TCPStateSynRecv TCPState = 2 + TCPStateEstablished TCPState = 3 + TCPStateFinWait TCPState = 4 + TCPStateCloseWait TCPState = 5 + TCPStateLastAck TCPState = 6 + TCPStateTimeWait TCPState = 7 + TCPStateClose TCPState = 8 + TCPStateListen TCPState = 9 + TCPStateIgnore TCPState = 11 +) + +// Active reports whether the TCP state should keep a VM awake. +func (s TCPState) Active() bool { + switch s { + case TCPStateSynRecv, TCPStateEstablished, TCPStateFinWait, TCPStateCloseWait, TCPStateLastAck: + return true + default: + return false + } +} + +type compiledPolicy struct { + idleTimeout time.Duration + ignoreSourceCIDRs []netip.Prefix + ignorePorts map[uint16]struct{} +} diff --git a/lib/instances/auto_standby.go b/lib/instances/auto_standby.go new file mode 100644 index 00000000..52f197e7 --- /dev/null +++ b/lib/instances/auto_standby.go @@ -0,0 +1,33 @@ +package instances + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/autostandby" +) + +func cloneAutoStandbyPolicy(policy *autostandby.Policy) *autostandby.Policy { + if policy == nil { + return nil + } + + cloned := &autostandby.Policy{ + Enabled: policy.Enabled, + IdleTimeout: policy.IdleTimeout, + } + if len(policy.IgnoreSourceCIDRs) > 0 { + cloned.IgnoreSourceCIDRs = append([]string(nil), policy.IgnoreSourceCIDRs...) + } + if len(policy.IgnoreDestinationPorts) > 0 { + cloned.IgnoreDestinationPorts = append([]uint16(nil), policy.IgnoreDestinationPorts...) + } + return cloned +} + +func normalizeAutoStandbyPolicy(policy *autostandby.Policy) (*autostandby.Policy, error) { + normalized, err := autostandby.NormalizePolicy(policy) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return normalized, nil +} diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go new file mode 100644 index 00000000..61212404 --- /dev/null +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -0,0 +1,191 @@ +//go:build linux + +package instances + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +const autoStandbyE2EManualEnv = "HYPEMAN_RUN_AUTO_STANDBY_E2E" + +func requireAutoStandbyE2EManualRun(t *testing.T) { + t.Helper() + if os.Getenv(autoStandbyE2EManualEnv) != "1" { + t.Skipf("set %s=1 to run auto-standby end-to-end integration tests", autoStandbyE2EManualEnv) + } +} + +type integrationAutoStandbyStore struct { + manager *manager +} + +func (s integrationAutoStandbyStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { + insts, err := s.manager.ListInstances(ctx, nil) + if err != nil { + return nil, err + } + + out := make([]autostandby.Instance, 0, len(insts)) + for _, inst := range insts { + out = append(out, autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + }) + } + return out, nil +} + +func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id string) error { + _, err := s.manager.StandbyInstance(ctx, id, StandbyInstanceRequest{}) + return err +} + +func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { + requireAutoStandbyE2EManualRun(t) + requireKVMAccess(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + mgr, _ := setupCompressionTestManagerForHypervisor(t, hypervisor.TypeCloudHypervisor) + require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) + require.NoError(t, mgr.systemManager.EnsureSystemFiles(ctx)) + createNginxImageAndWait(t, ctx, mgr.imageManager) + + connSource := autostandby.NewConntrackSource() + if _, err := connSource.ListConnections(ctx); err != nil { + if errors.Is(err, unix.EPERM) || errors.Is(err, unix.EACCES) { + t.Skipf("conntrack access unavailable for auto-standby e2e test; rerun as root or with CAP_NET_ADMIN: %v", err) + } + require.NoError(t, err) + } + + inst, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "auto-standby-e2e", + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeCloudHypervisor, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "3s", + }, + }) + require.NoError(t, err) + + t.Cleanup(func() { + logInstanceArtifactsOnFailure(t, mgr, inst.Id) + _ = mgr.DeleteInstance(context.Background(), inst.Id) + }) + + inst, err = waitForInstanceState(ctx, mgr, inst.Id, StateRunning, 30*time.Second) + require.NoError(t, err) + require.NoError(t, waitForVMReady(ctx, inst.SocketPath, 10*time.Second)) + require.NoError(t, waitForExecAgent(ctx, mgr, inst.Id, 30*time.Second)) + require.NoError(t, waitForLogMessage(ctx, mgr, inst.Id, "start worker processes", 45*time.Second)) + + conn, err := dialGuestPortWithRetry(inst.IP, 80, 15*time.Second) + require.NoError(t, err) + defer conn.Close() + + require.Eventually(t, func() bool { + conns, err := connSource.ListConnections(ctx) + if err != nil { + t.Logf("conntrack read while waiting for inbound activity failed: %v", err) + return false + } + + count, _, err := autostandby.ActiveInboundCount(autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(StateRunning), + NetworkEnabled: true, + IP: inst.IP, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "3s", + }, + }, conns) + if err != nil { + t.Logf("active inbound count failed: %v", err) + return false + } + return count > 0 + }, 10*time.Second, 200*time.Millisecond, "host->guest TCP connection never appeared in conntrack") + + controllerCtx, controllerCancel := context.WithCancel(ctx) + controllerDone := make(chan error, 1) + controller := autostandby.NewController( + integrationAutoStandbyStore{manager: mgr}, + connSource, + slog.Default(), + 250*time.Millisecond, + ) + go func() { + controllerDone <- controller.Run(controllerCtx) + }() + t.Cleanup(func() { + controllerCancel() + select { + case err := <-controllerDone: + if err != nil { + t.Logf("auto-standby controller exited with error during cleanup: %v", err) + } + case <-time.After(2 * time.Second): + t.Log("timed out waiting for auto-standby controller shutdown") + } + }) + + time.Sleep(5 * time.Second) + + current, err := mgr.GetInstance(ctx, inst.Id) + require.NoError(t, err) + require.Equal(t, StateRunning, current.State, "instance should remain running while inbound TCP connection is open") + + require.NoError(t, conn.Close()) + conn = nil + + inst, err = waitForInstanceState(ctx, mgr, inst.Id, StateStandby, 45*time.Second) + require.NoError(t, err) + require.Equal(t, StateStandby, inst.State) +} + +func dialGuestPortWithRetry(ip string, port int, timeout time.Duration) (net.Conn, error) { + deadline := time.Now().Add(timeout) + address := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) + var lastErr error + + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", address, 1*time.Second) + if err == nil { + return conn, nil + } + lastErr = err + time.Sleep(250 * time.Millisecond) + } + + if lastErr == nil { + lastErr = fmt.Errorf("timed out dialing %s", address) + } + return nil, lastErr +} diff --git a/lib/instances/create.go b/lib/instances/create.go index ca4be904..a58cd611 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -345,6 +345,7 @@ func (m *manager) createInstance( SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), + AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), } // 12. Ensure directories @@ -570,6 +571,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { return err } } + normalizedAutoStandby, err := normalizeAutoStandbyPolicy(req.AutoStandby) + if err != nil { + return err + } + req.AutoStandby = normalizedAutoStandby // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 2c75c94e..ec3e8142 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -482,6 +482,9 @@ func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { if src.NetworkEgress != nil { dst.NetworkEgress = cloneNetworkEgressPolicy(src.NetworkEgress) } + if src.AutoStandby != nil { + dst.AutoStandby = cloneAutoStandbyPolicy(src.AutoStandby) + } if src.Credentials != nil { dst.Credentials = cloneCredentialPolicies(src.Credentials) } diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index cb1a2299..2cc4fc02 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -295,6 +296,12 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { StoppedAt: &stoppedAt, HypervisorPID: &pid, ExitCode: &exitCode, + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "5m", + IgnoreSourceCIDRs: []string{"10.0.0.0/8"}, + IgnoreDestinationPorts: []uint16{22}, + }, } cloned := cloneStoredMetadataForFork(src) @@ -308,6 +315,8 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { cloned.Cmd[0] = "printf" *cloned.HypervisorPID = 4321 *cloned.ExitCode = 42 + cloned.AutoStandby.IgnoreSourceCIDRs[0] = "192.168.0.0/16" + cloned.AutoStandby.IgnoreDestinationPorts[0] = 443 now := time.Now() *cloned.StartedAt = now *cloned.StoppedAt = now @@ -320,6 +329,8 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { require.Equal(t, "echo", src.Cmd[0]) require.Equal(t, 1234, *src.HypervisorPID) require.Equal(t, 17, *src.ExitCode) + require.Equal(t, "10.0.0.0/8", src.AutoStandby.IgnoreSourceCIDRs[0]) + require.Equal(t, uint16(22), src.AutoStandby.IgnoreDestinationPorts[0]) require.Equal(t, startedAt, *src.StartedAt) require.Equal(t, stoppedAt, *src.StoppedAt) } diff --git a/lib/instances/types.go b/lib/instances/types.go index fb8f355d..141a6b9d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -3,6 +3,7 @@ package instances import ( "time" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/tags" @@ -139,6 +140,9 @@ type StoredMetadata struct { // Snapshot policy defaults for this instance. SnapshotPolicy *SnapshotPolicy + // Automatic standby policy driven by host-observed inbound TCP activity. + AutoStandby *autostandby.Policy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -220,6 +224,7 @@ type CreateInstanceRequest struct { SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance + AutoStandby *autostandby.Policy // Optional automatic standby policy } // StartInstanceRequest is the domain request for starting a stopped instance @@ -228,11 +233,10 @@ type StartInstanceRequest struct { Cmd []string // Override cmd (nil = keep previous/image default) } -// UpdateInstanceRequest is the domain request for updating a running instance. -// Currently supports updating env vars referenced by credential policies -// to enable secret/key rotation without instance restart. +// UpdateInstanceRequest is the domain request for updating mutable instance properties. type UpdateInstanceRequest struct { - Env map[string]string // Updated environment variables (merged with existing) + Env map[string]string // Updated environment variables (merged with existing) + AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil } // ForkInstanceRequest is the domain request for forking an instance. diff --git a/lib/instances/update.go b/lib/instances/update.go index 74ae3384..b0690349 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -14,10 +14,9 @@ type updateInstanceRulesService interface { UpdateInstanceRules(ctx context.Context, instanceID string, rules []egressproxy.HeaderInjectRuleConfig) error } -// updateInstance updates mutable properties of a running instance. -// Currently supports updating env vars referenced by credential policies, -// which causes the egress proxy header inject rules to be recomputed -// with the new secret values — enabling key rotation without restart. +// updateInstance updates mutable instance properties. +// Env updates recompute egress proxy header inject rules with the new secret +// values. Auto-standby updates only change persisted metadata. func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInstanceRequest) (*Instance, error) { log := logger.FromContext(ctx) @@ -32,14 +31,34 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta if err != nil { return nil, fmt.Errorf("get instance: %w", err) } - - if inst.State != StateRunning && inst.State != StateInitializing { - return nil, fmt.Errorf("%w: instance must be running or initializing to update (current state: %s)", ErrInvalidState, inst.State) + normalizedAutoStandby, err := normalizeAutoStandbyPolicy(req.AutoStandby) + if err != nil { + return nil, err } + req.AutoStandby = normalizedAutoStandby if err := validateUpdateInstanceRequest(meta, req); err != nil { return nil, err } + if len(req.Env) > 0 && inst.State != StateRunning && inst.State != StateInitializing { + return nil, fmt.Errorf("%w: instance must be running or initializing to update env (current state: %s)", ErrInvalidState, inst.State) + } + if req.AutoStandby != nil { + meta.AutoStandby = cloneAutoStandbyPolicy(req.AutoStandby) + } + if len(req.Env) == 0 { + if err := m.saveMetadata(meta); err != nil { + return nil, fmt.Errorf("save metadata: %w", err) + } + + log.InfoContext(ctx, "instance updated", "instance_id", id) + + updated, err := m.getInstance(ctx, id) + if err != nil { + return nil, fmt.Errorf("get updated instance: %w", err) + } + return updated, nil + } prevEnv := cloneEnvMap(meta.Env) nextEnv := cloneEnvMap(meta.Env) @@ -74,8 +93,11 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta } func validateUpdateInstanceRequest(meta *metadata, req UpdateInstanceRequest) error { + if len(req.Env) == 0 && req.AutoStandby == nil { + return fmt.Errorf("%w: request must include env and/or auto_standby", ErrInvalidRequest) + } if len(req.Env) == 0 { - return fmt.Errorf("%w: env must include at least one credential source env var", ErrInvalidRequest) + return nil } if meta == nil || len(meta.Credentials) == 0 || meta.NetworkEgress == nil || !meta.NetworkEgress.Enabled { return fmt.Errorf("%w: instance has no credential-backed env vars to update", ErrInvalidRequest) diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 338483be..135dd04e 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/egressproxy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,11 +23,11 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { }, } - t.Run("requires at least one env key", func(t *testing.T) { + t.Run("requires at least one update field", func(t *testing.T) { err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{}) require.Error(t, err) assert.ErrorIs(t, err, ErrInvalidRequest) - assert.Contains(t, err.Error(), "at least one credential source env var") + assert.Contains(t, err.Error(), "env and/or auto_standby") }) t.Run("rejects instances without credential backed envs", func(t *testing.T) { @@ -54,6 +55,16 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { }) require.NoError(t, err) }) + + t.Run("allows auto standby without env changes", func(t *testing.T) { + err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{ + AutoStandby: &autostandby.Policy{ + Enabled: true, + IdleTimeout: "5m", + }, + }) + require.NoError(t, err) + }) } type fakeUpdateInstanceRulesService struct { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index d4d336f0..6e619df7 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -194,6 +194,23 @@ type AttachVolumeRequest struct { Readonly *bool `json:"readonly,omitempty"` } +// AutoStandbyPolicy Linux-only automatic standby policy based on active inbound TCP connections +// observed from the host conntrack table. +type AutoStandbyPolicy struct { + // Enabled Whether automatic standby is enabled for this instance. + Enabled *bool `json:"enabled,omitempty"` + + // IdleTimeout How long the instance must have zero qualifying inbound TCP connections + // before Hypeman places it into standby. + IdleTimeout *string `json:"idle_timeout,omitempty"` + + // IgnoreDestinationPorts Optional destination TCP ports that should not keep the instance awake. + IgnoreDestinationPorts *[]int `json:"ignore_destination_ports,omitempty"` + + // IgnoreSourceCidrs Optional client CIDRs that should not keep the instance awake. + IgnoreSourceCidrs *[]string `json:"ignore_source_cidrs,omitempty"` +} + // AvailableDevice defines model for AvailableDevice. type AvailableDevice struct { // CurrentDriver Currently bound driver (null if none) @@ -332,6 +349,10 @@ type CreateIngressRequest struct { // CreateInstanceRequest defines model for CreateInstanceRequest. type CreateInstanceRequest struct { + // AutoStandby Linux-only automatic standby policy based on active inbound TCP connections + // observed from the host conntrack table. + AutoStandby *AutoStandbyPolicy `json:"auto_standby,omitempty"` + // Cmd Override image CMD (like docker run ). Omit to use image default. Cmd *[]string `json:"cmd,omitempty"` @@ -764,6 +785,10 @@ type IngressTarget struct { // Instance defines model for Instance. type Instance struct { + // AutoStandby Linux-only automatic standby policy based on active inbound TCP connections + // observed from the host conntrack table. + AutoStandby *AutoStandbyPolicy `json:"auto_standby,omitempty"` + // CreatedAt Creation timestamp (RFC3339) CreatedAt time.Time `json:"created_at"` @@ -1227,6 +1252,10 @@ type Tags map[string]string // UpdateInstanceRequest defines model for UpdateInstanceRequest. type UpdateInstanceRequest struct { + // AutoStandby Linux-only automatic standby policy based on active inbound TCP connections + // observed from the host conntrack table. + AutoStandby *AutoStandbyPolicy `json:"auto_standby,omitempty"` + // Env Environment variables to update (merged with existing). // Only keys referenced by the instance's existing credential `source.env` bindings // are accepted. Use this to rotate real credential values without restarting the VM. @@ -15316,258 +15345,262 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZIw+ioIntloaoakqItlWRMde2TJdmvbsnUs2312mv4osAok0aoCqgEUJdrh", - "v/MA84jzJF8gAdSNKLIoW5I19sbGtMzCNZGZyEzk5VMr4HHCGWFKtg4+tWQwJTGGPw+VwsH0PY/SmLwh", - "f6ZEKv1zInhChKIEGsU8ZWqYYDXV/wqJDARNFOWsddA6w2qKrqZEEDSDUZCc8jQK0Ygg6EfCVqdFrnGc", - "RKR10NqMmdoMscKtTkvNE/2TVIKySetzpyUIDjmL5maaMU4j1ToY40iSTmXaUz00whLpLl3ok4034jwi", - "mLU+w4h/plSQsHXwe3EbH7LGfPQHCZSe/HCGaYRHETkmMxqQRTAEqRCEqWEo6IyIRVAcme/RHI14ykJk", - "2qE2S6MI0TFinJGNEjDYjIZUQ0I30VO3DpRIiQcyIaxpSEPPCRydIPMZnRyj9pRclyfZfjzab9UPyXBM", - "Fgf9JY0x62rg6mW58aFtceyXu76RKY/jdDgRPE0WRz55fXr6DsFHxNJ4RERxxP3tbDzKFJkQoQdMAjrE", - "YSiIlP79u4/FtfX7/f4B3j7o93t93ypnhIVc1ILUfPaDdKsfkiVDNgKpHX8BpK/enxyfHKIjLhIuMPRd", - "mKmC2EXwFPdVRJvyqfjw/2lKo3AR60f6ZyKGlEmFWQ0OntiPGlx8jNSUINsPvT9F7TEXKCSjdDKhbLLR", - "BN81w4qIIuEQq8XpYKnItqGcIUVjIhWOk1anNeYi1p1aIVakq780mlAQvGI63aLRZIuklpqTHMaybnTX", - "BFGGYhpFVJKAs1AW56BM7e3Wb6ZAMEQI7uFQz/TPKCZS4glBbc02Ne9mSCqsUomoRGNMIxI2OiMfIpjN", - "/MFHiIaEKTqmZfo26NTFo2Bre8fLO2I8IcOQTuxNVB7+GH7XKKbHUQha+zeiCW3ebB8wpSDjxfmeA+uG", - "SQQZE0E0jn/hdIngM8I0tej5/gLztv6fzfyK3rT38yYA8yxv/rnT+jMlKRkmXFKzwgXOZb9oNAJQI+jh", - "XzN8WnbWBYySCovl9AEtvgIlmvU1gs25afq501J4srLLW92myjuBNdopS1yglkU+mxHmEZICzpT9UIbO", - "Sz5BEWUE2Rb2LDRP1BP8HHFgiV8JDhn4F4lfr/sGzMv8UDOa/tZpEZbGGpgRnxShOSVYqBEpAbPmCrMD", - "5aurBf9ZiXwqdxWWZLicg5xRxkiIdEtL2KYlSiVIqgvbByq6pGo4I0J6aQ6W9StVyLaoHSriweWYRmQ4", - "xXJqVozDEOgVR2elnXiktZL4ixPNBN2AIEVIpDg6/+Vw+9EeshN4YCh5KgKzgsWdFHrr4U1bpLAY4Sjy", - "4kY9uq1/Ry9iiB8DzjPCqLt7Mgx0iGk4Xcueph6+00pSOTV/Ae/Wq4K7T7MBjV6R/vuDZ9NHwCSMllCr", - "M/llwNeJOWw0ibiG6RyljP6ZlgTsHjrRuoJC+qKgIQk7CMMHzbJxqnh3QhgRmk+hseAxSFsFIRi1SW/S", - "66CBlgu7Wgru4u1uv9/tD1plMTba7U6SVIMCK0WEXuD/+R13Px52/9HvPvmQ/znsdT/87S8+BGgqmTup", - "0O6z7Wi/g9xii+J6daGrRPkbc//i8n0cxxz1ieYT65700cmi4GD2GvLgkoge5ZsRHQks5ptsQtn1QYQV", - "kaq88+VtvyosYB9LgMAmGkxrgqGi9AAatyN+RUSgOXBENOLJjmbCVMkOwlpvBuaF9C35dxRgpmnBCBdc", - "IMJCdEXVFGFoV4ZWPO/ihHapWWqr04rx9UvCJmraOtjbWcBzjeRt+0f3w1/dTxv/7UV1kUbEg+RveKoo", - "myD4bG71KZUoXwNVJF55Ig66aQRiXkzZiem2la0EC4HnX37CbiPLTtooc7VHHcQeyf/1jAhBQ3erHp0e", - "o3ZEL4lFdyRShgZpv78TQAP4k9hfAh7HmIXmt40eeh1TpW+zNL+kjTWoVzzu31skmHKQM6KI6w1loK4R", - "YnIYBoKAfoKjpdfwMhB7gXWUjbt4af/CperGmOEJAW3SNkQjwS+JXihKeEQDSiS6JHMtpMzRRA/anVFJ", - "NfkQNkMzbIwGvQF7O+WSmCbuk1ZEAkJnBMU8uERJhAMy5aCIz3CUEtlBV1MtMWhmLAiO7M9IkBhTNmBT", - "vUgZ8ISEWocwzWBr6IKw2QWKcQJUigUBEkUxVkRQHNGPJETcdIlJSPUFNWAE8BolWJNsEHChb199tgQH", - "0wIUfpLowsgbFzD8BWUaKy8MXfUGrHjyn1qv3719+vrdq+Ph67Nnrw5Phr8++1/9s+nUOvj9U8vYNzNB", - "4ynBggj0l0+w389GOg2JaB20DlM15YJ+NMaWz52WhoHU+IUT2uMJYZj2Ah63Oq2/Fv/54fMHJ0/pqQib", - "aTLwLOyzV5YxV6GHoxw7Y55E1kAEoh0GUy1wmBdn7zb15ZpgKdVU8HQyLROGvdnXIomQyssh5cNR4lsT", - "lZfoZPM10nIHiqgm0EzO2Or3T59uykFL/+OR+8dGDx0bqoXlaxbChRV/5FSjjxbCAWWOzt4hHEU8sCaQ", - "sdaVxnSSChL2KpY3GN3HnwlTYp5w6tPBKswpb7rIo7rd/OsarGhzRNmm1MfQDdaDO+DNjTWBZ2xGBWex", - "1sZmWFB9zcoyrbx6ffxs+OzV+9aB5uNhGlij4tnrN29bB62dfr/f8iGoxqAVPPDF2bsjOClDNiqJ0slQ", - "0o8eSeAw2x+KScyF0YBtH9SelgUFQ7cIDmfQ2nnx1CDX1gvAK3coIZXQ2o1iBi5jzPaLpz5smc4TImZU", - "+sxkv2Tf3MkXrnXD7su4LYmYEZEhLWBxr6B+BBFPw25hyk5rTAUJBNZo1+q0/iSxlsNnHzXq5Gv39PNb", - "rxrJnysESxwllJElkuU3IuFdcXEZcRx2t76ygMeI0mMvbvGV+VA+X4sTJEOJVmfBGsHCKxqq6TDkV0wv", - "2cNX7ReUNc6Y67XeCY7+/c9/vT/N1aStF6PEctqt7UdfyGkrvFUP7TWBZBtJE/823iX+Tbw//fc//+V2", - "cr+bMILIjYQ6e/7PzAjAsjWuh6VnSmPNLIPltylRUyIKt7dDFv2T0YehO3K4V9hKyTxafNNcYNR8RkSE", - "5wXGa9fU2uoD96usSlAFtGr7aTZ6iXTnFWxYj+Yu+RdVHX2772e0nkV51vRU8wp7LzRZSbaQre1T++f2", - "4pJqVnRJkyFIzUM8yUy2y16bzy9pYkVx6GGOMYoMIwhTEN5HnKvegP02JQzB2cEBk2sSAM+TCit0eHYi", - "0RWNIjDwAFNZvFq0YJ+zFdNcKv2/ImUdNEqVlta5IsjqTTBJCmuBxiOCUobdc3ZFdrYbrOKVBcslEYxE", - "QyMby4aQMZ2Q7VQLHNjqGEtFhOH2aVKG1/Gvp+eofTxnOKYB+tWMesrDNCLoPE00P9goQ68zYIkgM61C", - "sAkYG6mdl48RT1WXj7tKEOKWGMNgmYnMvrXOXpy9s6/1cqM3YG+IBixhIQlhze7GkUhNsUIhZz9piiVh", - "edji/BWg+2m505IMJ3LK1RA0v/kq7nRum5+Z1mvZAjqtWZCk5SPdrh7nK3iQ18CbUaFSHGleWxInve/z", - "xvPDozYYx5Ki+mL5XobdWJUfVpsaTMzI4AayKFT77R5GUmps9yio8gsWEKdnfmq22BXjnzC3kKV2n1zV", - "/IK5zs0gVRDZsTtuZzeA0kkGkzKs8NcBz6EsqOa1dvWQSEWZQSfdFlmJUKL2hdbmLR5r/f2igy7+WvpB", - "075TLbR8cYUMNICfMP1TcfyqUWKluaC5Ulg5HCxvfh6HstZRCc22kBKYSX23ahkrIT30CzBxpEicaE7G", - "JohKJA3zJSFi/OrviBuhxnUdML00adw8LDgyo5GkE0bZZEOL+fpiwmFoLEvjVKVCt5tRmUOzjDrOelPd", - "wFuzOmL4cZxKfSMHURoSdOEsPBdluXDR/rOoElqD0IKGY0ACmg0oe2ozTpWeXm84xiqYajjxVBm/L7t1", - "WV5A2cq06j3UriV7KbvB+Z9n7KIMVGtvqDB+vTn7RgNmwYJ9ss4MaAUVv4nykszhyJ05Ei8YJIuWSL+9", - "UBDJoxmx127RljnCwaW5SozrhTVjGoOktUFq8q+QqNc6t+ooNLwag7+sKiyiEpiA7WZzjLHSv7H/zjMu", - "pDdn5utoxVgSAD6oHgcIxLGLjtGVCFggENPIEqGQChKoheEpmwwYuIBc2F96drQLTeRaRvERoU/Z8cqC", - "BW3H9CkdLSqcrBP7YBi9NR5TpUjYKcsGl4QkcvWmtHhtDdce67ogV4I6RmYNRmFD8YywMRcBia2S8GWK", - "47PCYF41br0hFj0yDHwLa7b4hHCSRJSExv3HnAeYWaU9J7CxVl1+w4rWZjwAylNe4Ci6QG3baAMJovci", - "3VkxznJkf3t05lAge7V+f9rRGKm5wMVUqWSo/0cONRVfVAezfR2F6+H0nSTRfh/0q93dHXuq1uhmFlwZ", - "tmxf83o11B+NE7/rH8Z4rGnRuYk0EeWP8i65JfWSsrDpAL/qtrXWuUwwcprGbRvoEkG6aTIRGDxkv6Z5", - "7sbPngDNeg6+wvnd5+WYQTVIpeJxwdcRtSseGrTsy1EG1oxH3RArDKbMhvZWs9xFv+F4boYyulidJWY4", - "GXncfuhHzXXRhE7waK7K7wdbfZ/G96Vv0G4tvmOp8783GiQJh4ov90CmY+TaNnE4hPtkqPhwNqaekbNr", - "LXdfoRIFFWd/q9fqIbpJQK05AWScYGocRA0QQGh8f1p8u+sNWBeu3wN0nE2QDZsNiUG2xKF5OWlzUVgE", - "Ba8zNJpvIIzen/bQ22y1P0mkFZYZcQEJUyzRiBCGUjA9w23YNXdxcQGphEtTVbtb24mJXdiAJ0puv/XQ", - "L/OExNjaoTQpxFjRADydRrSyH7iOzEHZN2HMilawRlarZX7bb8iESiUqXtuo/eb50c7OzpOq/XL7Ube/", - "1d169Harf9DX//+P5g7eXz88wzfWYZm3WN+xIvc5endyvG2NpeV51Mdd/GT/+hqrJ3v0Sj75GI/E5I8d", - "fCcBHH5Wdpw7vaF2KonoOjapscrn6lbwKKtxZbuxh9otOZzl/rPL2hpIvNUtbyMyxefzbD1u148dqTLM", - "lV7Thc0tavLzBPTOnEoKEpx1Tgyo1w3zmMrLp4Lgy5BfMc+9HeMJkUNzn/n9GVJpnGzItbVuCM7VWJp3", - "07LVc2v38e7+zt7ufr/vCchYRHge0GGgb6BGC3h9dIIiPCcCQR/UhgevEI0iPioj+qOdvf3H/Sdb203X", - "YZ54msEhU7xcL9S2EPmbC+5zX0qL2t5+vLezs9Pf29vebbQqay9utChnWy6JJI93Hu9u7W/vNoKCT6B/", - "5gJkqgJ86HNd0PqTeWzsyoQEdEwDBCE2SHdA7RiuMJK9VpVpcoTDoTWe+O8OhWkkl3pMmMlsS2Noi9NI", - "0SQi5hscSCNbNOz8GEbyeaNQxogYZvFDa4xkw4pWegi4vWRNUCk8rAS6UypBCsmFJ0qi8MBQ6Eo+B6eZ", - "L+xDHR7YPTTEhpdadepGZEaiIhKYq0svNuaCoAxPzKGVdkXZDEc0HFKWpF6UqAXl81SALGoGRXjEU2We", - "GeHAipOA0zLoHmPNrpvpuc+5uFzp/qlv4qFIGdPDrLQKHYIhfWxNNXCLY2R7uwiDgtCXPQeaR1P7XaI3", - "poexEOU/J6lClCmutVMWjuYdmMlakhgSRCoOnNQaDO0wTaVLv9wCxlLn/mHmy3nnHfm+dMfGXeDrathi", - "QtRQKqxWSiwaU95C+3No3tibXHdcaUhpAHdGru4C6OBu39Vo25UMJ7cD8WXOaJmtIW8Et7CgIekhoC7w", - "inHhfRVKO1c8SUiY2X96A3ZuSCX7SZoXFN3RwEFNCRWICzqh5YnLBrbb9GpbBxUdNt0YHYsdFyVU+Aju", - "G/VEj8eKCANBF7lcDD+yh9DqtCzsW52W5URl0LgfPRDJXS0Xlvji7N26vmmJ4GMaebYLvhD2q9XMnNfW", - "y93+eXfr/zMemBrfQESjzPhPxDwkvUpyAGjf7OZ5cfburG5NWWYGVFzdwp4yjxcP58j8GhxE7KOSfZW0", - "GoxDf32xZJPksvcTnyw7Fjgmo3Q8JmIYe4xrz/V3ZBoY1ybK0OnTsjyr5eamWvNZ6XBAbR7jwAbWN4O+", - "xyBX2UanAM0P/uN6Q8w1XBeOp49K2DY2Iq+HXmW5MNCLs3cS5V5KHktd+Xhr/eXPpnNJAxyZEU10LWVF", - "AxsgZ2MJ+SzvaE2RHjk59sqGjhBQezZJUiDD8zfdk9fvN+OQzDqlNYFn0ZRHRK97o8AtZi4oL3fuLzGJ", - "WZ2lwyCGbEpABVhlFNwYSAV69UBHcYWjoYy4z1njrf6I4CNqv39ugqb0CjooKR2l/r0AhRJ+73kpRnOk", - "umnPYcKqybRE4F7dsZxCxphXCtsrTeojlV8IjkzmnDI+5/Hd7uD5Zfmg+eVK6rWD+OY9cY7hDYK3jk6P", - "jcAQcKYwZUSgmChs8/QUXFxAHGp1Wl19R4WYxOBqN/77cu+WGhN8MRqr1oh7tJB241YMuDXh4m+MC0KI", - "YszomEhlw8VLM8sp3n60d2CSWoRkvPtor9frrRuj8iwPSml0FJvGhb8QrtKT0y87h1sIRWmyl0+ts8O3", - "v7QOWpupFJsRD3C0KUeUHRT+nf0z/wB/mH+OKPOGsDTKg0LHC/lPyk+a+s4yvx/onTDrEqZxiYMCv/KJ", - "qUafAc8GiJvzhgsrPNH6icG4L40LvnHmkDx9lSpkDCk6hDbIHkI/LreEOsEI2tg5U6ZolCdWWbSB3ig1", - "jlyaPWAhc0BCWJYvIIrMXwFnM00VvuQBJQbuvn3R+4H1chmG1IPJv1ltzzhJQFTVanprbeIkWY22fkEx", - "439Nk6bY0GbPTXTvXP8mb2zl2V9P/ufP/1+ePf5j68+X79//7+zF/xy/ov/7Pjp7/UURVMuj2u81NP2r", - "RaPDw1IpJL0pKp1iFXgEqimXqgbC9gtS3Phr9tARKH4HA9ZFL6kiAkcHaNCquAgPWqhNrnGgTC/EGdJD", - "2UiHDd35zJh/dOdPTrf8XB0jtCENwh5IFskk01HIY0zZxoANmB0LuY1IeNPXf4UowIlKBdGnp2XYaI5G", - "Agd5KEM+eQd9wknyeWPAQMMl10roHSRYqCwNh5sBkMKuyvgM2OYkdIHhRkMesOxeyuLCjY2mlxlBwDZf", - "9bj0A8WrvnBRDsXZ7/si6MHrSx9kRKUi4JidYbZGo8wdDe33S6xiv7/fXyngZzi0BP2AEhaTZDqkbEBL", - "BoFhasO4wUOtgS1d8yZDI+iXt2/PNBj0f8+RGyiHRXbERskzPoDS2AhVJAvefxstn+nbnG7DDRkjGXSL", - "GkQNPTPuoW9fniNFROwc9tuBBueYBnp/8PxPpUw1KlKMDo9On230GmT5BNhm619yjm+zHVaDO6zRrM4W", - "mGG8hm8HnRyDe66l0FyAA7ea51ygyDCYnK4P0DtJyr6ucFTmVd+cZDTPLW/mBhi0NtyISZVTHKA3mdyI", - "s6VkjpY5Mrghc7qEYe3Di/H5WRi94pcL3kxWL7KsDTx8sMqcxPWNW88KlpO/B+JA89avu2DTXI+2i8ZQ", - "PZkfNfKzv3VpZWddHXXdBA3lGMpC/G2Wo6F5coXbSFKwqK9dUzWsfYRH+rN9cndayftTNMWS/aTgY0U3", - "2dp53Chbpp616fN18eGaj82SMqpyAZnZs6sJTb2kUWS8GSSdMByhJ6h9fvLi15OXLzdQF71+fVo9imU9", - "fOfTIFeDQ+0XZ+8g2gXLoXsBqnd6xLnjMLmmUsnFeNVGD6nLc0P8Usrf4A0A3viKSR3c6/PCNu4iXcN9", - "uvV9e6kiliZ3+NIMDVbYvaUEDbXM1ZfcoMxnzc9fN9XCrSynFPvj4w9FmcD5XN84t0GnRT3+podSs0AS", - "opOzPMVhbpRyw1f29GS7t7W339vq93tb/SYmuhgHS+Y+PTxqPnl/2xgiDvDoIAgPyPgLTIQWsY3whqMr", - "PJdo4MTrQcvI8wVBvkC2VgRv9Py6mELiZhkjqgLFqpwQ6+SAaJbc4Usj6pclOj4vpzhuLOQ9+scXZUMm", - "Ta926/tgew3XsX4TFPA0CrUgNdKka/QyElr1URKVZ48Gan/HLhm/YuWtGyOoZgB/pkTM0fvT05LJXJCx", - "TY7bYOPgM1FzDjxZ6xi2V8jaK1dzwzwLd5Fbocp2C9fdV8+kULTZOR9Mg6ENbHe5+Ol9N6fMHI3GkyV7", - "qlhdQjIbpqlPqtKfXOTFu3cnxyXkwHhva7+//6S7P9ra6+6G/a0u3trZ624/wv3xTvB4pyY9fXO/mZu7", - "wpSpuT7SCQAPFkwTyBYeaHrLfFlGqUKZn5sm5CMtnqKCHGziesCocMKoghyOlE30MKDjWzHZBGiaNJOU", - "UQUZASAfDWV6y2BM0YNY76UD9ALawiccQ7yRW4RWjsp2BBzOjR1VMwY3dQL/Wr7k82mqtNwGfeQ0VUj/", - "C7atwWDVleVDGB5zgF5x6COckynjVb3HNAfnrcXmVR2pbd2KnPspTGYZ5gF6njHJjM1attqWxP5peLf1", - "jAav742S75098ZbGlvzkCm5lnZaBaKvTcoAC97NFRzS7Lm+MRREVfQ8MBEfAQnNHn1TRyCY5gJ1QqWhg", - "tEYMh1tHyTahFwmHRgSoey403iNWTMg6OUbx/hS1IZzxb8gqlfpfG9nTYpEqd7ef7D7Ze7z9ZK9R0EK+", - "wNUM/gh8mxYXt5LbB0k6dJU/arZ+dPYO7j59r8o0NlYCu/eCj2gieKClVcpQXkokn/xJ70kxViPk6Sgq", - "WJ1sYBcEBDSp+1LzPvYnjWZ0PGZ/fgwut/8QNN663pPbI69yl03kl4RPipbSBbWRjLomCaPfnR4QSsja", - "iJM3RMIO0DlRCPCni3AAl3TmkmRRzsWlWIh7EWt3Z2dn//Gj7UZ4ZVdXIJwh6K+Lqzy1KyiQGLRE7Tfn", - "52izgHBmTOenCfkhmBXg/HSGbD7mfsmFU+tOOz4sqZGXcqyxY8/iWpC/t0KQ3ZQFOnhWZQLSApV7ob2z", - "03+8+2j/UTMythrbUFwv5zAuJ4cBj01jUjz5NljX3x6eIT26GOOgrKFsbe/sPtp7vL/WqtRaq4IUPCZ1", - "xhoL23+892h3Z3urWeiUz4JugwJLBFvmXR6i8yCF5zQ8oFhkvZ2628IneBoEe0OCCNP4MHDeL5Xbx6TI", - "GArTLD+EJheDNRIsXFwN+jZS0Sple4xowAVKWZaYqbfaHHoz62Y9mzb3wWo2vihDR5hpcFkff5OJ8Qaw", - "SwSZUZ7KrzAQVyTQyDSOOBdr9a1zJ3pDZBopY4KkEr0//QmYiEYuJBVJyq7yFv2WRELccHNrEXAJJ/xY", - "XQesRqfR5OiXbbhTQ6adZW6wJfKvDTgKNatK2eqn6yMcBSnkHsPZeepdQegATxU8tM+Nk0cUcc5QMMVs", - "QiCXu8l0yCYIoymPwl7L/1QShcOx9wmDX6GIm1QJl4QkNi2XWYTupmUWOiOo/YLnBeUMKlXS6z6KDVex", - "iZfK2PgorinOKX2Og1mAkoYnVrwQxW+6lLT5iE8kaIEK3Fd61eQxCRbGKwUzk2ZuFhvlsRx5ta1ve88S", - "K9zbd4Waq5OPrUZrZQzFM0jiQHApEYnoBFKavT8tL3OZ/2FMGY01n139HF1ebAPUlQln0pcXBe402Tgb", - "pe9C9Dh2fcmVCDgM/pvepwPzkG9d8VGMWQqJugqITK4TKgx6NHscn3Kphlk0yZqLlWoISZhSQfKQM3df", - "TsF/f25YHLTx3ouOtd0EXNZr4ka9F7DKP1TdAut5qheifmh1Mhz0ofFiPM3SEJ48JqgaALJOxFeetYdK", - "GJUWgo1Qm3FVYkuFzDMbTR6q/DqqnqeuoOvL3f5502Cs5bFXZ1hNT9iYe1I7rmHwtx7tznchISKmkIYM", - "hYRREjrlMbP8W9sW+MhHkqAwJRZyRiAV2AIcG/KGFI7MGcUom1R4fXXCJmZ4s4blOZpgXtuwyZOj9HtW", - "vxUpwMo4CUiEcx/rRh4PVA79luLFgQWZpBEWqBpwuGTJch5HlF02GV3O4xGPaIB0h+pzzphHEb8a6k/y", - "Z9jLRqPd6Q7D3EWw8jxjFmcdRM2BVObNt/Cz3uVGxT0dTC+bpv8mVOxu8oLr9Rt6TiNiY/LeMXpdQPRy", - "EpPd7X5d5ELNoKWYhcV4znU5t0VZH8W7UMvDrOiBxz/NeABVXiXKhsjSfn27BRezZXEai6YY1HaPwi5J", - "TBmuhWQtjSwhzbzcqu4PbjWbkgTl2Xf3Hz3ea5gt54tsnUtqGn+BZXMWL7Fo1pzUaROz2f6j/SdPdnYf", - "Pdley0DlPGVqzqfOW6Z4PpXaJhWj2aM+/N9aizK+Mv4l1fjLlBdUqlNy4wV9XkK6eZR0zbNHbZruqHiS", - "7p2lbAFtZmNcIi0dlkSuQimuNhmPCSiVQwO3br6Yind9ozUEOMEBVXOPwQRfmYztWZNKtG8Ta1p5sR6Q", - "2rFtwgbNuWQ6yh06225y9FdjWq/gwn7jpFsyHdWZ8V9XZzVG/NwGVHwiavBCk9cFWDQXZPu5wrLk1aH/", - "DiDlcl5qreo/ZFo0rwntcD0rC517Rvoi1v0loIvHXznOgtm3JCRXIb7sCq0nwbV0aM+N7Ksyudort8If", - "7AV4s17DUTEd3tJ8g6Xcefmtu/68zYrELfYzN9j68xVcQNfpWM0MBvho12BBno/dKaFEDTYpLlYnhL6F", - "/D7Gp+BGGX6sO8KdJPmxP99KYp+F4zgnyrU91xp9Gi3J58wUETPsMUy5IZBrUrajGk7cQdbEh7bijUql", - "wd2pX1azEbgN3ce0FDhMBBnT6yXYYhqY67rsPi4tBMJyzm+J2jG+RruPUTDFQlbWzuhkqqJ52ci664me", - "+LISykRp0bl5dvT8NF3HxRcNe5zF0X0ke16IdfBnbSfhcFmc+lHWzNmMEzwH2bJWEXy8s9vv72z3bxSo", - "/rWSyRfGqfMILfSzxpzS02NxhMz/czHj4JWgpiaZA5NUguD4ALypEhwQFJExhHFlmV5X6vQLUy9fvH0k", - "tY7/Gf67g3KFR62dxbI4xhkIHm4cG+PvttFyr7TlWI/i98VlLwkWy9hMsBA1VvVf3ev2d7r9vbdbOweP", - "9g62tm4jsj0DUp0Lz+OPW1ePo2083o3254//3Jo+nmzHO95wgVuoW1ApA1gpY2D3kBBRTSVZTcEqSUQZ", - "6crM7W21A/ISXmBeklbS/3rWB7ODpcLCeXmTRZkBqxw41YpqdxHYZFe/1IRSXf7J8fJl38iPrLoQP4JV", - "lwL41GwxkGll60vTeqSs4b3zrtCw8c2z1Ldx1d3jc/oG0vaecg3EffhcYowlClt2Yy/eah4VbsIFVdN4", - "+fWQNcuSBMBj+EepwnIgTQ+dTBgkji3+nL19FGs7686tTiv6uFumGft785AqGxSfIaA96qIY0OBtAPIS", - "L4cCNMlVC2HcE7AgAIift7pbT+CFPvq4+3O/+6SHfit4CnQMtIrg23KtS7/2m8AwY5Qgd5qX860naz2j", - "O3guw6Bf7b1UdxHbcHmL43nWTndXOLfp0gHnnxfOuBJVdEt1gj4v2bETnNeLY3e9PKJJzy+bPHnb31o7", - "6856V0TvCzyKv0jVa6beRViqOrn6JZYq08eQSK103YEKNaxSqN5mBbehAODaCbH2B0hWNHyXDl3mQo+z", - "AnTQhCuUBwGslHJg+SJlXnworz+vMgzZHWoRYnsFQjRbUxbIR5eR7skxSgQP0yD3gI1g0WkQECnHKRRN", - "7jWVaFe/Md6mMg+u5VqjX63M12nvq8NMyXX9eb8i16owpUbY+qPe6q8+6luxAHRaaRKu5mGmUTMOtlYm", - "jhU+lR57RBnsFSmosJkPDTj6myIEFxU8qLeEAi0OpIkrAKhxahGTpKfsH74eenMEHJOIKOIbBJmKnNbt", - "g8qci65mqVt7+36TGb4eBhCQuLCQXwlJtJwec2krZMaYzb0LqybBRu2+KwApEQzfNYm4LLTKi3u8Ugqp", - "Parm+cQrFl0T8VVM355l6Pi6ycRtz5WlHm5PUHlrVaW6jDJVX8+i3fGw+w9jZ0TD3sHmz3/7f7sf/voX", - "f2mVkiIlieiGZAwPYJdk3jU1Y7XS1iunI4VsNy2psC1IogiOwYoQXBJjtYjxdXG9j/oZJc1f4XhhC/By", - "GFOW/Xvlhv72l/p3twIY3wHzWHmOt5HYVXHHYtsxEROXPd15e0ExbRbNNaQlKmSPs9e0o8qfZNalWLb1", - "wog2PSg2PKKQhFMOmNZScBCQRJGwZ7NoUViL4EBR1eLFNoud887WtIYhdaeNlqkkqfrkLRF80GLkqmtm", - "CLsadXYf7dl67UVIbi2ckO/MTMB1XX1DDWWPFeAllRBN4JxmC41Rm8SJmrscrc6tcWO9APDDbEDvS+ZX", - "zn7Vf/I1cnW+W5qc8zusrlmMz3cLWhmZv3D+tRnx/H5Rx9VEO4YmbcWwcmKYisYjVbfebSrWV/QQvP8W", - "XZz0N+NZaLNRTtJqWu7NmKlNm/vWF80QQkHgpb6kOZW5YPUudFrtIrlUSCzsrLCS+rM5dbJQtUhzPYDO", - "NGiupkSQwkFAhzyB55ogs35+DWJkTIbKhIhutaScqXogKDgOZmqsA0HmC7po11qeoOYUX2czgE0Uy4WX", - "A9hHnqpt68VTqGTyxpUWo2M3BCyjIqn6s82UsahJcfTFwyhi1eK+TXsv4VletYT71dFWBTnzOUqo6cPH", - "3zBVz7kA2bY+IuXWk9aA3BwSASG51ZQ0jfK50JiEQ56q5fRvU7zbcJQQjciYC1LIfuvkeAxIbCsOr+AF", - "LmYiX8MHn9wgSZAKquZa8bMS5YhgQcRhaggeAAkTwc/5xJBM9vNnsICNPQ5oLwgjggbo8OwE6BGq5mvi", - "eH+KIjomwTzQ+jPkAl1InwFC3uujE6s7uYRt8B5IFaCeKwJ8eHYCNUWF0R9a/d52rw/EnBCGE9o6aO30", - "tqDCqkY42OIm5J6HP61vuQkro5ydhFYOemqa6F4Cx0QRIVsHv3t8tBURJpe9BKkTTwpif4KpsHJ/EoHn", - "uEEVqvtC9iJ3lR6Y+7hjAN7Y9CPV3PrRkeS1PdYPGhMM1cAWt/t9o2YxZS9enNea3PzDxtvl8zaS5wA8", - "nlQ+C3K9kyktyD93Wrv9rbXWs7I8pG/adwynasoF/UhgmY/WBMKNJj1hxrkXmTQR1n2hSGeAQkUK+/2D", - "Pi+ZxjEWcweuHFYJl3XCMJEIQ4E6U0jhDz7qIWvYhuylcsrTSHMTZDyXSagvLKx5Sm/yEWERTOmMDJi9", - "p02pTywgu3OM9P1s1JYyaZipzelnMWVPeTivQDcbblMP13XmzBzA1SyJkgwh2dOwrkpKbsikjEG1RUls", - "SsmsXMCijwSUx5UB99YFJgwzlVdbNXVxL8nc2kq9AzbKyqIZHhwLgTLsWbbw7Q1/OAIkv/RH8hxn35AF", - "b1mcYPDAEERpmMtczkMWixGOIm/Y/iTiIxzZ8sGXxCOivoAWFijFPKFOuGE8JCbnYzJXU87M3+koZSo1", - "f48Ev5JEaBHI5n62sLa1My3qQh13GkP+ZVNZQs+5aZa4+emSzD/3BuwwjF3VEGlLv0eS27rKJvsNlci5", - "VBrc9WcnrXmtP0ql4rFFKVYsA2mWyVOVpMq+VEqibMJqaA5VQuWUhAOmOPokTFH4+efNT/mMn0F3ITjU", - "eFJoYra0+YmGn+tWLYdY734ITT3aHwEADFr6dhm09N8TgbXuksopmDIkmC8mxSNtZ6HUWi7cqEI4wAwl", - "PDFh6IBUplx0aQxI/o+jCCkgJddXS5twkjX7sZElvkp2NqzExAFUyAhq2hWIqb+776cnSQJBfAaO/zl/", - "/QrBVaXPwDTLzUYAI8r0LYrCFCR5mL03YM9wMEVGboJUY4MWDQetTLsIN2CtqbR+r90uiLg/66X9bKbp", - "0PDnXk8PZaTnA/T7JzPKgaalJB4qfknYoPW5gwofJlRN01H27YMfoHXe+eclRoDahvdvuNItkCUgvwbN", - "vYFZiLjltdEcYZRzoKIdZUQZFkvrznhAbyGoVXk8kUVgfBqAAXTQOhg4E+ig1Rm0CJvBb9ZOOmh99kPA", - "CtH1ea1M6R0na2dItNfvb6wOm7Pw9YjQpYaa/D4vSF/bX03wsELXouBhNueS8ukTNEWUjLh1B5LPUxy6", - "tPw/RLwVIp61XBSEN+hfvAcM+kbEKLgVCUzrs5GTwJZqJwYtICslaBwuyNUoHNRJcDnyFtWPqjq/qFbs", - "1lFZAEuMHP7t3gH+wbx5IXKY98ldzYsjSDGZleV9WOgIh+UQsePXiF8Q9S1gXP+uWKnNh3mf+PtQ8OcF", - "sXJfDrQKN9skM/fe5A/lBxd+aUcxjbWueg5r6p4TptAz+LVn/+s0HkhMexHxycUBMiCM+ARFlNnXuMJr", - "kb4ULSyhk/Hiz/pZp36XR6lt7s9///NfsCjKJv/+57+0NG3+AnLfNMktIO/qxZRgoUYEq4sD9CshSRdH", - "dEbcZiAzIpkRMUc7fRAzEwGfPKUe5YAN2BuiUsEKr5YmpZG0A4LqwWA/lKVE2igI3ZCObb4FY2D2qPCO", - "lg0o75SiO4suo2YHhQ3oW9HhAATQUpN81upfLb/1zOy5ZD+r2soXLKar+Ysi18pgb9cscE0GAyD20R18", - "sJtG7fPzZxs9BDqGwQrIqQEScz6MFZ57P3jSap5kOEqZoQCUDW8qVPmutf8e2zbNDMB2xO/JAlxXtrze", - "BGxMHkSQ0MHrh67QxBzsh5szDfvss8cu9q3eQHvz/RancM5AjRThr3fODvcWYW6+FEB2Hyowajs/a1dx", - "7+zoxJV22bg3pL+TW0Pv1BZEyK4OxE2dvztTy444G0c0UKjr1gI5+WOSqWplBHko7OCNXTXCbl/V7HXF", - "+22zlIyl9qbL8rLkV97t3x6VSde5RvIMezmu/bhJVqHOMZUB130L2NINcGLrDRrxJaPTIhatMkgZt+3s", - "ylkqLln2fHLsCPLuTFN26pRV74Y7YIrHFYZ4j4ywUkOtkJPyIWHzu+wUXZz/EsvVt4Wa/buTgu7aiuVD", - "84dkxgorYNNc0GTmrb1AXxD1i2lxiwdtZ/Bs/JwIR9UuhTDsOtuW6YqCKQkuzYbgQXq57ntimjRTfc14", - "35PmC+BZR2KxIP8hojRQdnNYLVNwT2xduNvTb2GGtdTbr/fOaxHMA2RwNhk5i7UpuYblnAUb39VT753c", - "ZgbYD/IyO0ujyL14zIhQKCu/XLwDNj+BW9Jq2d5R29Lr4N2bl13CAg5+aJkPlV+Isl++soRvDsxs5Qea", - "NNEJTcAtdfdZnYTzBedv3AVRVt77v7af2wLf/7X93JT4/q+dQ1Pke+PWkKV/V6z5riXuB4x8WuCmZaAB", - "a2JQ6nOVhJq1aiikuvbflZxqNr2WpJrB9Yew2kRYLYJrqbxqj+JWJVYzxz09yWTI5oM2fHL+id+ZpHq3", - "Vj6LkS79LpXlZw9bX4ULsPPCJ8pQKskDdKCkGcYVr42G5uqcIJdeHw51T447AMiOBh3kA7IBIndkvHbr", - "uHPh1s5795brw3hEJylPZTH2JMYqmBJpg5UiUmbAD03szq/nWsH7G8bS/l1eHXcuV//A+1uS+KsHapi3", - "eYFaJfO7Vk1lfttey/y29r2JXXvjaurbNEcbNU6FLoi6KRqXYs0XnR196/LpIuidVlRydQGBBnEwYP+t", - "9Y/fFcHxh59dkEza72/vwe+EzT787OJk2KlDFcKUoDb35uGrY3j2m0D0OaTnzEPyqusw+fwB9VwCm/84", - "BSl/+WyuITks/KEhNdKQCuBariHZs7hdFamcw+rOdSSHbz6A2yQmP7Sku9CSZDoe04ASpvJyVwtOYrZa", - "3gOMLWP2fajg3FG6aBtrSRlRrhBA82Trd+7Yk01+98qRy+v+MH3kuYmKCZ06kl+G9frIt4YP/btlznev", - "hzxkFDMC/yLoEi1T+qooQqbHOFXglJhnCAGvTySM1J6N2EN58UKZJgkXSppskSAAQzUrNdUCsC+zZDlZ", - "pC87JOS1pUR2Bgzyv+vPJpZ/85LMTS5IyvOi/NlObf5HX+xVOZXmvZLR15ex/HlCG8lYd0zGNh3y/clY", - "98Y67kTSOillmW9nhAEK5YhklMyz4D76kbLJxoPyQDXMKttbIZ+RR9TaHNt6gn4V6DkXl02Zgqe+zQPg", - "DcUdfoPal14e5E+6fyUM1BNDPxpp7pxvLBQtuk+ndVrlJEGUhpp1OBbiLt+x4PHQ/mgyfGqqsPkTQakL", - "7Kj3zWT07HegYr/iCtE4iYiWe0iIugab9GlaYcmlyaayUOJrPSaoyaYYQmDSd0lXJsQWQoPnCHdgbXiZ", - "XDwuL9eM+GR12oBschcj78kbMGAmjTdxOb8vUMZkkeJIkogECl1NaTCFHAL6N1NZEML7cZJcZEmDNg7Q", - "C6DUYu4kmLwtidCiY8CZ5BExqQFmcXxxsJjj8v3pKXQy6QNMNsuLA+TyWmYXhNStijkBsvojr2ymg7bG", - "JMGjyJzohZazC/vbsNkC8qROA+bLHMDIlR2QjtFFIYnARU0WAcdQX/KJvC9RtlOfis/sRXEkAHAGNwkL", - "W3Wmaxr58wds9b21GRrmMjDLuOVUBguLecknWRrAEirjJGmKvnaZgMWzOF6Cw6hdKBIoVchT9TepQiIE", - "dLbYXYfcqI0D8w+FLzWi2iIfWZlFQD/vA43Jy+UFlWaqhYoW5l+zOG51WnY9hXxea1gYVuSEqA64+JCg", - "T6aQ+OGHLWGdlA5lZl/I6VC5OWwZ6nqR21bX/u4tWhZQ4fegl5ZfAPJVUOZEFThbnlfEeVCx4abwelUW", - "M+V7fDTidtmVhUJ+zR4EFkoAfgNK66p3gqyeW1Zs7q4fDBZX8JDDBuTCbqBQPFvrJeGbR6SvdyQLW22C", - "IT9wc/0nh0aImaRL6vpBWUKJsKt1B5lwgynnsoD2IzLFM8qFzVltayZlmAkmC6M9Wn+jC42qF7Ys2oUV", - "zw+srQnh4ic7Rw+6Wy8lfw/3Ke/xvKBtZxy/40RqyJsnEUYjQckYJTiVREtLaUyQqclgUx8THExd1dre", - "gL2dEmRr1RUMCFlpUyrRxVZ80UGjVKEIiwloO+aj8T0SJOBxTFho6k8O2JTgGdWqmkARVoQF864kUI90", - "RvKSD1p1t286CJ50soqHHeQKZYKB4aJQBvMCJYIAEhl1mZVqTg6YSNnfTa4/PeyFW+gFIlLhUUTlNMuu", - "H+CQsMCbSO/822ZjX9+Ie07UYqXIe3nluREvvc9nn6ItM6vV+028CD0w1xYuXEXABmx+idAr61XDsq/Y", - "eV4d8z+QpM1e3R7v6WUmA/EyKv42nmRK5bF/PMsoS5JhaqYj5RLS3+1bS8ZQUMpKzy3WJnvTB5csd3wG", - "5rV43uYn9+fJDWxk3wgn7Cwp1O6fIN/0t8ByLVRvxHPvyThobUkFq9g9smC7qPsTn7gocLlvgg0bgsu4", - "cZHnKIFBpzJF7X8w49Lbt3EPuCkzdhbXhQfwAnumrJtEuI4vW+NsLQOulFD/D/MXrCkQ/9lywvtkfPmL", - "wJ0xu5OMvRmGl+B5xPH3/i4TcCFMCJwt4PpwUjAVbIGFB6Y2WNw6GYfoOP/796enG3VcQqilPEKoB8wh", - "yoUgg9hT3+71jAhBQ1ds7+j02JbmoxKJlPXQ65hCBbxLQhIorUF5Kk1Z/l6xQv1i2bBKCXrClJgnnDK1", - "chV509tZzOcbFRu7Yz5pk9B994/HYIV/eEwKeIcWV+wGlmuRCqtaZzznnEaZqRCopS084qkefaGCPhrT", - "iMi5VCQ2nnnjNAIigjSltoqN7Wdi8DqIKok0PXQgZikhIqZSUs7kgNmC2QkRem7dHcql5k5GXuO9whnX", - "PDOs79twYIOi+uCzhVUd1Mr19HGSuHr6Picpu7wvWNJz8EhDch6PeEQDFFF2KVE7opdG6UAziSL9x8ZS", - "l7Yh9PvaNXpuTlka0idszL1lDAzOZsj8fYRtlNmae0R8cGztBSkSi+M/cNB+tiZX8jVBcNRVNCZZuDBK", - "FY3oR8Pq9CBUKhqY2s95sBqUrbXxagN2SpTQbbAgKOBRRALljCubieDB5iDt93eChEJehx0CiwOGV/85", - "hhmPzt5BO1NatzNg+h8w8NvDM/MSO8bWRlBYKCPqiotLdLL5eoWT7zmA6T/YS85scGnUmPfAfzzfrR8L", - "WktDsoZEebJMAeLJd+/GaSW4H9aCh2ktgGD8bDfticABCMVymqqQXzG/ZWDGozTW/zB/nKxK6aBwMH0P", - "Tb8ZadcsZ+U0boMPgijtnkJiyqzcywOFAdhD9S/VgHNbACGm5LnnvQUO1feI3V/fKF+E4zf4NGkh6koY", - "fTO0ddc3n12Dy1RUhMdDIXODaW4nii+3Pl1hWm99ehrx4FKilCkagcGkIGliyJyof8wz3dmHPxATIDrS", - "FV9G5DqhAnJ+gE23MBLRO5YII0VETBmONmHPZhDI2eesWHjGKQQpBxGFMDEaEpTwKIK8JFdTwpDeDRiq", - "3ACFd1ppc+YX2xSfGBVHIxLwmLg8hhs+1e03TNVzLspJCb8Vvvi2AH+9H71Vvc8VeRjrZ/yivIyn+Brc", - "msPUPhO7FbVf8PxHYwrqIDibQWunLwetDhq0tuNBS5/AEQYTKlboEYopSxWRPXRs7FsQhrrXR5IEnIXS", - "pVN0FrydvqwLSjVoWRPhuAf97lLssVgFoHxjJ/GxB90O6f4QYIPaRYKzNBl2gOhCxFMFDtyOrmyrkCgw", - "j2zc+QtsgUZ+6PZNOPlvlnxLPApOWbPLwtEbzp4l3FtpdXNBFVMu8zx9KMAJDqiadxCOIh7k1oNUZq8D", - "3WwpI0HwpdahegP2Jkv1ZwMh0NHZu44zmqGQykszgrWL9dDrGREyHWWLQ8ANjAUPDoOEA6Y4CnAUpJHG", - "WzIekwBiGCIaUyVr7GrZUm6zcFw+iefg3UcLuodmTPLjBJxejhaygnGb5qg3BQkiTOOiUakKHBB94UkX", - "zL4jPSjX1/A4ss9bgeBSIjtUl0R0QkeRfayRPfRWixw4JgOWRJgxIlAqjd+RXno3EUTK1ATG6AGgMqfB", - "qA7KE50kgitrJo44F9JYdjWGvz9FUpFkCZq9MSOfwp5vKbGqGdzOdE8KQ2UN9deSbYL0gRhMMQDXeKSv", - "6Xtw9jELuu8ErA+F8N8KOpkQoakCGyZrnkYNWTtwGqIvRXrUZhU/z1o1yyqejVrw5i54Oi9NVDF0DYcg", - "QK/zAuuZ/JLW5jKxn9aLvvhVd2o4d9nL378I++kLd/m9FGs6LzhXN81FnmP4Q0sLXlh5iVRLAQqr0xE0", - "jki4zQiBxnkH7i3dwEPOMoBLYQd16QS+PUTo32103F0nJn7YuFXKElAqRVITKrU6fec3gYG3k7fznqND", - "b5C385uKV4K8i/cXN/pNRSqV7ICu3MJ3n5nztgKUTHpOSGNRF6BkuJ51JFiqKL23bZqpSXbE70mCt2/P", - "a8jvDuw/tP4GKkMBWH6TnYmNdnlbSJyouXtc5OPKA6CkHyEYw5f4IfMhuL18Czd4Xv966OHwtPZx/UcF", - "ojt7v8/LtJ4cP/yyQ0WaK10sm/rW6WIRTOmM1BvdyxRsQZQI0k14Ao8roQGYhYe7yxQWvclHZIe3uars", - "vxB1KY5JiEIqSKCiOaJMceAIZo6fJBJcawLwnYu5z5hepNzngseHdjcr7kNLU9YYlr/5xvNuiBXuzhy3", - "WWJC+4KXdve2rRkeogy9eIra5FoJk3EXjbXmg+g4Aym5DggJJeDkRnHBW/0ayyb9SIaTUZNVLsmd/Nrm", - "pkZBKhWP3dmfHKM2ThXvTgjTZ6FF/TFIsongMxqa0o05UGc8MlDdqgHounZXLVTY8L5cuTCLuxcZpsmF", - "NPlIkzJbMK4LrYPWiDIMi1uZpbhMUyagSs+HKYQ15LTjMKf14wqzml/bKTsaE7WS44CoODep8TZ+XHMP", - "+ZorOqa6O6102zUrrtfMV7WhC+ltJMzN/Jjv1mz9/ttxr6TyQXpWWtP5LFNI68zm3xYK9u/ufrhrc/n7", - "B+yO/4I45btgKocB9Ig+hHnJAxyhkMxIxBOou2fatjqtVEStg9ZUqeRgczPS7aZcqoP9/n6/9fnD5/8b", - "AAD//4ZlynVGYgEA", + "H4sIAAAAAAAC/+y9e3Mbu7E4+FVQ3NwKlZAU9bAs69ap38qS7aN7LFtr2c7eHHopcAYkEc0AcwAMJdrl", + "f/MB8hHzSbbQAOZFDDmU9bBi35tKZA4ejUaj0d3ox5dWwOOEM8KUbB18aclgSmIMfx4qhYPpRx6lMXlH", + "/kiJVPrnRPCECEUJNIp5ytQwwWqq/xUSGQiaKMpZ66B1htUUXU2JIGgGoyA55WkUohFB0I+ErU6LXOM4", + "iUjroLUZM7UZYoVbnZaaJ/onqQRlk9bXTksQHHIWzc00Y5xGqnUwxpEkncq0p3pohCXSXbrQJxtvxHlE", + "MGt9hRH/SKkgYevg9+IyPmWN+egfJFB68sNU8XOFWTian/GIBvPFxb6mLL2G2RBOFY+xogGSpg9KoBMa", + "YUlCxBnCgaIzgigb8ZSF6P3RGQo4YyTQg8kB4yNJxIyEaCx4jNSUoCmXCtoogYNLpPAoIr0Ba3Uq+0GY", + "/hKuxtLfpkRNifAASyWyo6AxF0hNqUSU6a8B6RU3TImULGK206JhRIaKxoSnahFRv/IrFHE2gWW5cVGc", + "SoWmeEbQZyI4+iPFER3PKZvUI2lExlwQ9Os8ITFmKIlwQCSiClGmuFuNwVFOY09iH3HRCeOCDEMiFWVY", + "jz9MuDAnogz9W/gDR6jQFkCD9khNsXJUzrhCl4Qk5YXiK3xZRuPv29udZ/1+/1OnRRWJzbHC1zRO49bB", + "3pMnO086rZgy8++tDHrKFJkQocG3v2Ah8LywHMlTEZBhQEOxbCVBRAlT6Ojk+N0NF9Da6vfg/zf3W53W", + "1rPt3tbePvx7a69VXNYC4suQf/UdvRmmkabGYzKjAVnkQEEqBGFqGAo6I2JxnUfmezRHho5MO9RmaRQh", + "OkaMM7JRohE2oyHVTEg30VNXSD0HPwSYhjT0ML+jE2Q+o5Nj1J6S6/Ik209H+636IRmOiefspDFmXc3X", + "NFhufGhbHPv1rpfIeRynw4ngabI48snb09MPCD4ilsYjIooj7m/7yC4J6BCHoSBS+tfvPhZh6/f7/QO8", + "fdDv9/o+KGeEhVzUotR89qN0qx+SJUM2QqkdfwGlbz6eHJ8coiMuEi7g1C/OVLlTiugprqtINuVd8V09", + "z1MahYtUP9I/EzF0x9KLsBN3Zk+OER/DMbb90MdT1NbcPSSjdDKhbLLRhN61rBARRcIh9nB2ABXZNpov", + "6jtAKhwnrU5rzEWsO7VCrEhXf2k0oSB4xXS6RaPJFo9aanZyGMu60V0TRBmKaRRRSQLOQlmcgzK1t1u/", + "mMKBIUJwD4d6oX9GMZESTwhqgwxxNSVM32EqlfpCHmMakbDRHvkIwSzmH3yEaEiYomNaPt+GnLp4FGxt", + "73h5R4wnZBjSiRUCy8Mfw++axPQ4CkFr/0L0QZs3WwdMKch4cb6XwLphEkHGRBBN4984XSL4jDB9WvR8", + "f4J5W//XZi4db1rReBOQeZY3/9pp/ZGSlAwTLqmBcIFz2S+ajADVCHr4YYZPy/a6QFFSYbH8fECLWziJ", + "Br5GuDk3TfVtjicru7zXbaq8E1ijnbLEBWpZ5IsZYR79JOBM2Q8VeZ1PUEQZQbaF3QuQeOcJ+SXiwBJv", + "CQ8Z+hcPv4b7BszL/FAzmv7WaRGmJcbfWxGfFLE5JVioESkhs+YKswPl0NWi/6x0fCp3FZZkuJyDnFHG", + "SAgKkj3YpiVKJSiJC8uHU3RJ1XBGhPSeOQDrN6qQbVE7VMSDyzGNyHCK5dRAjMOQGun4rLQSj7RW0jxx", + "opmgGxCkCIkUR+e/Hm4/2UN2Ag8OraiuGyyupNBbD2/aIoXFCEeRlzbqyW39O3qRQvwUcJ4djLq7J6NA", + "R5iG07XsburhO60klVPzF/BuDRXcfZoNaPKK9N+fPIs+AiZhtIRac4VfBsxUoUnENU7nKGX0j7QkYPfQ", + "yRg0In1R0JCEHYThg2bZWonuTggjQvOpXGsvCMGoTXqTXgcNtFzY1VJwF293+/1uf9Aqi7HRbneSpBoV", + "WCkiNID/3++4+/mw+/d+99mn/M9hr/vpr3/yEUBTydxJhXadbXf2O8gBWxTXq4CuEuVvzP2L4Ps4jtnq", + "E80n1t3po5NFwcGsNeTBJRE9yjcjOhJYzDfZhLLrgwgrIlV55cvb3iouYB1LkMAmGk1roqGi9AAZtyN+", + "RUSgOXBENOHJjmbCVMkOwlpvBuaF9C353yjATJ8FI1xwgQgL0RVVU4ShXRlb8byLE9qlBtRWpxXj69eE", + "TdS0dbC3s0Dnmsjb9o/up7+4nzb+j5fURRoRD5G/46mibILgc9GO5WDIbBLLdsRhN41AzIspOzHdthaN", + "Lt+2w24hy3baKHO1W62Z0NCavVYBsmjQ1MpW7FEd3s6IEDR01/LR6TFqR/SS2POCRMrQIO33dwJoAH8S", + "+0vA4xiz0Py20UNvY6r0dZjmt7yxUVbMSSSYchBUooivYz8CSREUHBwtvceXocaL7aNs3MVb/1cuVTfG", + "DE8IqKO2IRoJfkk0oMb8S4lEl2SupZw5muhBuzMqqT5/hM3QDBurQ2/A3k+5JKaJ+6Q1mYDQGUExDy6N", + "rXPKQZOf4SglsoOuplrk0NxcEBzZn5EgMaZswKYaSBnwhIRaCTHNYGnogrDZBYpxAsccCwJnHMVYEUFx", + "RD8bm7XuEpOQ6htuwAgcDJRgfeaDgAt9feu9JTiYFrDwZ4kujMByAcNfUKbJ+sIczIp19kvr7Yf3z99+", + "eHM8fHv24s3hyfC3F/+rfzadWge/f2mZt4lMUnlOsCAC/ekLrPerEW9DIloHrcNUTbmgn4215munpXEg", + "NX3hhPZ4QhimvYDHrU7rL8V/fvr6yQlkxqI+08fAA9hXrzBk7lIPSzp21kCJrIUJZEMMzyzAol6dfdjU", + "t3OCpVRTwdPJtHwwrGiw1pEIqbwcUj4cJT6YqLxEJ5tvkRZcUET1Ac0Ela1+//T5phy09D+euH9s9NCx", + "ObUAvuZBXFj5SU41+WTPHEdnHxCOIh5YG8pYK1tjOkkFCXsV0x2M7mPwhCkxTzj1KXEV5pQ3XeRR3W7+", + "dQ1WtDmibFPqbegG6+Ed6ObGqsQLNqOCs1irczMsqL6nZfmsvHl7/GL44s3H1oG+CMI0sFbJs7fv3rcO", + "Wjv9fr/lI1BNQSt44KuzD0ewU+bYqCRKJ0NJP3tEicNsfSgmMRdGhbZ9UHtaljTMuUWwOYPWzqvnhri2", + "XgFduU0JqYTWbhQzcJlitl8991HLdJ4QMaPSZ2f7Nfvmdn7xfatE2/AIJzKiBSruFfSXIOJp2C1M2WmN", + "qSCBwJrsWp3WHyTWgvzssyadHHZPP7/5q5EAu0IyxVFCGVkimn4nIuIVF5cRx2F365YlREaUHntxiW/M", + "h/L+WprIH7wWnlhHmIVXNFTTYcivmAbZw1ftF5Q1zpjrtV4Jjv79z399PM31rK1Xo8Ry2q3tJ9/IaSu8", + "VQ/ttaFkC0kT/zI+JP5FfDz99z//5VbysIswgsiNhDq7/y/MCMCyFx/PjTnU/3ae397Z66jiVqGG7sjR", + "3spXcx+j5jMiIjwvMF4LU2urD9yvApWg4BaAbD/NRi+R7ryCDevR3CX/qqrkb/f9jNYDlAem55pX2Huh", + "CSQZIFvbp/bP7UWQaiC6pMkQpOYhnmQ232U+EOeXNLGiOPQw2xhFhhGEKQjvI85Vb8D+NiUMwd7BBpNr", + "EgDPkwordHh2ItEVjSKwEAFTWbxatGBfeEeH5lLp/xYp66BRqrS0zhVBVm+CSVKABRqPCEoZdu/hFdnZ", + "LnDRGwPQckkEI9HQyMayIWZMJ2Q71SIHljrGUhFhuH2alPF1/NvpOWofzxmOaYB+M6Oe8jCNCDpPE80P", + "NsrY6wxYIshMqxBsAtZKauflY8RT1eXjrhKEOBBjGCyzsdnH2tmrsw/2uV9u9AbsHdGIJSy0ni3uxrFe", + "DyFnf9YnloTlYYvzV5Be5wEjGU7klKthknkLLeNO57Z5roo3NyZ0WrMgSctbul3dzjfwoq+RN6NCpTjS", + "vLYkTnof+I3XlkdtME5hRfXF8r3cS0SVX2abWlzMyODC5fUP8RhOjKTU2HBSUOUXTChOz/zSDNgV458w", + "B8hSw1Guan7DXOdmkCqK7Ngdt7IbYOkkw0nF3HQ76DmUBdW8kbcVuMJZiVCi9oXW5i0da/39ooMu/lL6", + "QZ99p1po+eIKGWwAP2H6p+L4VaPESnPBWv5Nxc3B8ub7cShrPZ3QbAspgZnUd6uWsRLSQ78CE0eKxInm", + "ZGyCqETSMF8SIsav/htxI9S4rgOmQZPGT8SiIzMaSTphlE02tJivLyYchsayNE5VKnS7GZU5Nsuk46w3", + "1QW8N9ARw4/BJZCyIEpDgi6cheeiLBcu2n8WVUJrEFrQcAxKQLMBZU9txqnS0+sFx1gFU40nnirjOGaX", + "LssAlK1Mqx5ULSzZU9sN9v88YxdVz8+ZR8XRi7OPPGAWLNgn68yAVlDxmygvyRy23Jkj8YJBsmiJ9NsL", + "BZE8mhF77RZtmSPwbeVGcMrNmMYgaW2Q+vhXvTp91rlVW6Hx1Rj9ZVXB49MqVdctNqcYK/1b99+MC+nF", + "mfk6WjGWBJAPqscBAnHsomN0JQIWCMQ0sUQopIIEamF4yiYDBj4kF/aXnh3tQh9yLaPciqew4tYvuLS1", + "qLCzTuyDYfTSeEyVImGnLBtcEpLI1YvS4rU1XHus64JcCeoYmTUYhQ3FM8LGXAQktkrCtymOLwqDedW4", + "9YZYdOkw+C3A7NzJcZJElITGf8jsB5hZM//t3sKmxzysaG3GhaA85QWOogvUto02kCB6LdLtFeMsJ/b3", + "R2eOBLJn74+nHU2RmgtcTJVKhvq/5FCf4ovqYLavO+G5K/V+H/Sr3d0du6vW6GYArgxbtq953SLqt8aJ", + "37Uva5ouNJTWz6SJKH+Ud8ktqZeUhU0H+E23rbXOZYKR0zTu2kCXCNJNk4nA4GJ7m+a5G7+bAjbrOfiK", + "wBWfm2TuEp9KxeOCsyRqV1w8aNkZpIysGY+6IVYYTJkN7a0G3EXH43huhjK6WJ0lZjgZefyG6GfNddGE", + "TvBorsrvB1t9byTBNz5iO1h821LnwG80SBIOFV/uwkzHyLVt4rEI98lQ8eFsTD0jZ9da7v9CJQoq0QJW", + "r9VDdJOAWnMCyDjB1HiYGiSA0PjxtPh21xuwLly/B+g4myAbNhsSg2yJQ/Ny0uaiAAQFtzU0mm8gjD6e", + "9tD7DNo/S6QVlhlxEQ1TLNGIEIZSMD3Dbdg1d3ERgFTCpamq3a3txAQ/bMATJbffelmQDVhpspAhcJUa", + "0cp64DoyG2XfhDErWsEaWa2WOX6/IxMqlai4faP2u5dHOzs7z6r2y+0n3f5Wd+vJ+63+QV//5+/NPcRv", + "P77DN9ZhmbdY57Mi9zn6cHK8bY2l5XnU5138bP/6Gqtne/RKPvscj8TkHzv4XiJA/KzsOPeaQ+1UEtF1", + "bFJTlc9XruCSVuMLd2MXtzvyWMsdcJe1NZh4r1veRWiLz2nauuyuH3xSZZgr3a4Li1vU5OcJ6J35KSlI", + "cNa7MaBeP85jKi+fC4IvQ37FPPd2jCdEDs195vdnSKVxsiHX1rohOFdjad5Ny1bPrd2nu/s7e7v7/b4n", + "omOR4HlAh4G+gRoB8PboBEV4TgSCPqgND14hGkV8VCb0Jzt7+0/7z7a2m8Jhnnia4SFTvFwv1LYY+asL", + "zHVfSkBtbz/d29nZ6e/tbe82gsraixsB5WzLJZHk6c7T3a397d1GWPAJ9C9chE1VgA99rgtafzKPjV2Z", + "kICOaYAgRgfpDqgdwxVGsteq8pkc4XBojSf+u0NhGsmlHhNmMtvSGNriNFI0iYj5BhvSyBYNKz+Gkbwh", + "oYwRMcwCkNYYycYlrfQQcGvJmqBSfFkJdadUghSSC0+UROGBOaEr+RzsZg7Ypzo6sGtoSA2vterUjciM", + "REUiMFeXBjbmgqCMTsymlVZF2QxHNBxSlqRekqhF5ctUgCxqBkV4xFNlnhlhw4qTgNcz6B5jza6b6bkv", + "ubhc6T+qb+KhSBnTw6y0Ch2CIX1sTTVwi2Nke7sQhYLQlz0HmkdT+12id6aHsRDlPydpOYy7AzNZSxJD", + "gkjFgZNag6Edpql06ZdbwFjq3D/MfDnvvCffl+7YuAvcroYtJkQNpcJqpcSiKeU9tD+H5o3d0XXHlYaU", + "Bnhn5Oo+kA7++l1Ntl3JcHI3GF/mjJbZGvJGcAsLGpIegtMFXjEuPrBy0s4VTxISZvaf3oBZf+7sJ2le", + "UHRHgwc1JVQgLuiElicuG9ju0qttHVJ01HRjcix2XJRQ4SO4b9QfejxWRBgMutDnYvyS3YRWp2Vx3+q0", + "LCcqo8b96MFI7mq5AOKrsw/r+qYlgo9p5Fku+ELYr1Yzc15br3f7592t/8d4YGp6AxGNMuM/EfOwnO3B", + "tW9287w6+3BWB1OW2gEVoVtYU+bx4uEcmV+Dw4h9VLKvklaDceSvL5Zsklz2fuaTZccCx2SUjsdEDGOP", + "ce2l/o5MA+PaRBk6fV6WZ7Xc3FRrPittDqjNYxzYyPxm2PcY5CrL6BSw+cm/Xe+IuYbr4vn0Vgnbxob0", + "9dCbLJkGenX2QaLcS8ljqStvb62//Nl0LmmAIzOiCc+lrGhgA+JsLCGf5R2tKdIjJ8de2dAdBNSeTZIU", + "juH5u+7J24+bcUhmnRJM4Fk05RHRcG8UuMXMRfXlzv0lJjGrs3QYwpBND1ABV9kJboykwnn1YEdxhaOh", + "jLjPWeO9/ojgI2p/fGmirjQEHZSUtlL/XsBCib73vCdGc6S6ac9hwqrJtHTAvbpjOf2TMa8Ullea1HdU", + "fiU4MlmvyvScB4i7jeeX5Y3mlytPrx3EN++JcwyvKDW+4K2j02MjMAScKUwZESgmCtscWwUXFxCHWp1W", + "V99RISYxuNqN/3u5d0uNCb4YjVVrxD1ayNtxJwbcmnjzd8YFIUQxZnRMpLLx5qWZ5RRvP9k7MFkxQjLe", + "fbLX6/XWjVF5kQelNNqKTePCXwhX6cnpt+3DHYSiNFnLl9bZ4ftfWwetzVSKzYgHONqUI8oOCv/O/pl/", + "gD/MP0eUeUNYGiVSoeOFBCrlJ019Z5nfDwqZxZDLXNbgialGnwHPBoib88YbKzzR+omhuG8NLL5x6pE8", + "/5UqpBwpOoQ2SD9CPy+3hDrBCNrYOVOmaJRnZlm0gd4ot45cmn5gIfVAQliWcCCKzF8BZzN9KnzZB0oM", + "3H37pvcD6+UyDKmHkv9mtT3jJAFRVavPW2sTJ8lqsvULihn/a5p1xcZGe26iB+f6N3ljK8/+dvI/f/y/", + "8uzpP7b+eP3x4//OXv3P8Rv6vx+js7ffFEG1PCz+QWPbby2cHR6WSjHtTUnpFKvAI1BNuVQ1GLZfkOLG", + "X7OHjkDxOxiwLnpNFRE4OkCDVsVFeNBCbXKNA2V6Ic6QHspGOmzozmfG/KM7f3G65dfqGKENaRB2Q7JI", + "JpmOQh5jyjYGbMDsWMgtRMKbvv4rRAFOVCqI3j0tw0ZzNBKQx9Kq5/nkHfQFJ8nXjQEDDZdcK6FXkGCh", + "sjwebgYgCguV8RmwzUnoAsONhjxg2b2UxYUbG00vM4KAbb7qcelHild94aIcirPf90XQg9eX3siISkXA", + "MTujbE1GmTsa2u+XWMV+f7+/UsDPaGgJ+cFJWExw64iywVkyBAxTG8YNHmoNbOmaN5kzgn59//5Mo0H/", + "7zlyA+W4yLbYKHnGB1AaG6GKZMH7b6PlM32b3W24IGMkg25Rg6ihF8Y99P3rc6SIiJ3DfjvQ6BzTQK8P", + "nv+plKkmRYrR4dHpi41egwy9gNsM/iX7+D5bYTW4wxrN6myBGcVr/HbQyTG459oTmgtw4FbzkgsUGQaT", + "n+sD9EGSsq8rbJV51Tc7Gc1zy5u5AQatDTdiUuUUB+hdJjfiDJRSVuCyMS8/lzCsfXgxPj8Lo1f8csGb", + "yepFlrWBhw9WmZO4vnHrWcHy4+/BOJx569ddsGmud7aLxlA9mZ808r2/7awpty/u7Kyr5K6b4aEchFkI", + "4M2SPDTPznAXWQ4WFb5rqoa1r/hIf7Zv9k6t+XiKpliyPyv4WFFutnaeNsrXqWdt+v5dfPnmYwNSdixd", + "RGf2bmtiWy9pFBl3CEknDEfoGWqfn7z67eT16w3URW/fnla3YlkP3/40SPbgzsarsw8QLoPl0D0h1XtN", + "4tzzmFxTqeRiwGujl9jlySV+LSWA8EYQb9xiVgj3fL2wjPvI9/CQfoHfX66JpdkhvjXFg5WW7yjDQy1z", + "9WVHKPNZ8/Pt5mq4E3BKwUM+/lAUKpzT9o2TI3Ra1OOweig1CyQhOjnLkyzmVi03fGVNNjn9Vr/f2+o3", + "sfHFOFgy9+nhUfPJ+9vGknGARwdBeEDG32BjtIRtpD8cXeG5RAMnnw9aRiEoaAKFY2tl+Ebvt4s5KG6W", + "cqIqUKxKKrFOEolm2SG+NSR/Warl83KS5cZC3pO/f1M+ZtL0arfOE7bXcB3zOUEBT6NQC1IjfXSNYkdC", + "q39KovL81XDaP7BLxq9YeenGiqoZwB8pEXP08fS0ZHMXZGzT8zZYODhd1OwDT9bahu0VsvZKaG6YqOE+", + "kjNU2W7hurv1VAxFo59z4jQU2sD4l4uf3od3yszWaDpZsqaK2SYks2Ga+qQq/cmFbnz4cHJcIg6M97b2", + "+/vPuvujrb3ubtjf6uKtnb3u9hPcH+8ET3dqEuQ3d7y5uS9N+TTXh0oB4sEEaiLhwgN93jJnmFGqUOYo", + "pw/ykRZPUUEONoFBYJU4YVRBEkjKJnoYMBJYMdlEeJo8lZRRBSkFIKENZXrJYI3Rg1j3pwP0CtrCJxxD", + "wJIDQitHZUMEDufGEKsZg5s6gX8tB/l8miott0EfOU0V0v+CZWs0WHVl+RCGxxygNxz6COelynhV7zHN", + "wSaw2LyqI7WtX5LzX4XJLMM8QC8zJpmxWctW25LYPw3vtq7V4Da+UXLeszve0tSS71zBL63TMhhtdVoO", + "UeC/tujJZuHyBmkUSdH3QkFwBCw09xRKFY1slgRYCZWKBkZrxLC5dSfZZgQj4dCIAHXvjcb9xIoJWSfH", + "KD6eojbEQ/4VWaVS/2sje5ssnsrd7We7z/aebj/baxT1kAO4msEfgXPUInAruX2QpENXe6Rm6UdnH+Du", + "0/eqTGNjJbBrLziZJoIHWlqlDOXFTPLJn/WeFYM9Qp6OooLVyUaGQURBk8ozNQ9sf9BoRsdj9sfn4HL7", + "H4LGW9d7cnvkVe6yifyS8EnR1LqgNpJR12Rx9PvjA0EJWRuy8o5IWAE6JwoB/XQRDuCSznyaLMm5wBaL", + "cS9h7e7s7Ow/fbLdiK4sdIWDMwT9dRHKUwtB4YhBS9R+d36ONgsEZ8Z0jp6QYIJZAc5/zpBN6Nwv+YBq", + "3WnHRyU18lJONXbsWVyL8o9WCLKLskgH16xMQFo45V5s7+z0n+4+2X/S7BhbjW0orpdzGJfUw6DH5kEp", + "7nwbzPPvD8+QHl2McVDWULa2d3af7D3dXwsqtRZUkMPH5N5YA7D9p3tPdne2t5rFXvlM8DaqsHRgy7zL", + "c+g8ROHZDQ8qFllvp+628AmehsDekSDCND4MnPtM5fYxOTaGwjTLN6HJxWCNBAsXV4O+jVS0SuEgIxpw", + "gVKWZXbqrTaH3sy6Wc+mzX2wmo0vytARZhpdNkjApHK8Ae4SQWaUp/IWBuKKBJqYxhHnYq2+df5I74hM", + "I2VMkFSij6d/BiaiiQtJRZKyr70lvyWhFDdc3FoHuEQTfqquQ1aj3Wiy9csW3Kk5pp1lfrSl418bsRRq", + "VpWy1W/fRzgKUkhehrP91KuC2AOeKnipnxsvkSjinKFgitmEQDJ4kyqRTRBGUx6FvZb/qSQKh2PvE0ZW", + "UpXnBTsdELqbKzjbfsXzknaGlCr5eZ/EhqvYzE29BsVTBcHS53mYRThpfGLFC2kATJeSNh/xiQQtUIH/", + "S6+afSbBwri1YGby1M1iozyWQ7e29W3vAbHCvX1XqLk6+dhqtFbGUDzDJA4ElxKRiE4gJ9rH0zKYyxwY", + "swKuq9+zy8A2IF2ZcCZ9T9umUm7jdJa+C9HjGfYtVyLQMDiAep8OjCeA9eVHMWYpZPoqEDK5Tqgw5NHs", + "cXzKpRpm4ShrAivVELI4pYLkMWvuvpxCAMDcsDho470XHWu7Cbqs28WNei9QlX+oOgDreaoXo35sdTIa", + "9JHxYkDO0higPKioGkGyTshYnvaHShiVFqKVUJtxVWJLhdQ1G00eqvw6qp6nrqTs693+edNoruXBW2dY", + "TU/YmHtyQ65h8Lcu8c53ISEippDHDIWEURI65TGz/FvbFjjZR5KgMCUWc0YgFdgiHJvjDTkgmTOKUTap", + "8PrqhE3M8AaG5UmeYF7bsMmTo/S7Zr8XKeDKOAlIhHMn7UYeD1QO/ZbixYEFmaQRFqgasbgEZDmPI8ou", + "m4wu5/GIRzRAukP1OWfMo4hfDfUn+QusZaPR6nSHYe5jWHmeMcBZD1OzIZV58yX8ole5UfFvB9PLpum/", + "CeX6m7zgev2GXtKI2KC+D4xeFwi9nAVld7tfF/pQM2gp6GExIHRdzm1J1nfiXazmYVY1weOObzyAKq8S", + "ZUNkab2+1YKL2bJAj0VTDGq7R2GXZaaM10K2l0aWkGZeblX3BwfNpiRBefbd/SdP9xqm2/kmW+eSqsrf", + "YNmcxUssmjU7ddrEbLb/ZP/Zs53dJ8+21zJQOU+Zmv2p85Yp7k+lOErFaPakD/+3FlDGV8YPUo2/TBmg", + "UqGTGwP0dcnRzcOsa549avN8R8WddO8sZQtoMxvjEmnpsCRyFWp5tcl4TECpHBq8dXNgKu75jWAIcIID", + "quYegwm+MinfsyaVcOEm1rQysB6U2rFtxgfNuWQ6yh06225y9BdjWq/Qwn7jrF0yHdWZ8d9WZzVG/NwG", + "VHwiavBCkxcWWDQXZOu5wrLk1aH/DiBnc16rreo/ZFo0r0rtaD0rTJ17RvpC3v1FqIvbX9nOgtm3JCRX", + "Mb7sCq0/gmvp0J4b2VemcrVXboU/2AvwZr2Go2I+vaUJC0vJ9/Jbd/15m1WZW+xnbrD15yu4gK7TsZpa", + "DOjRwmBRno/dKZFEDTUpLlZnlL6DBEHGp+BGKYKsO8K9ZAmyP99JZqCF7TgnyrU91xp9Gi1JCM0UETPs", + "MUy5IZBrUrajGk7cQdbEh7bijUqpwt2pX1azIbwN3ce0FDhMBBnT6yXUYhqY67rsPi4tBsJy0nCJ2jG+", + "RrtPUTDFQlZgZ3QyVdG8bGTd9URPfFsRZ6K06Nw8vXq+m67j4ouG3c7i6L4je16IdfCnfSfhcFmg+1HW", + "zNmMEzwH2bJWEXy6s9vv72z3bxTpflvZ6Avj1HmEFvpZY07p6bE4Qub/uZiy8EpQU9TMoUkqQXB8AN5U", + "CQ4IisgY4sCyVLErdfqFqZcDbx9JreN/Rv9uo1zlUmtnsSyOcQaChxvHJglwy2i5V9pyrEfx+yLYS4LF", + "MjYTLESNVf1X97r9nW5/7/3WzsGTvYOtrbsIjc+QVOfC8/Tz1tXTaBuPd6P9+dM/tqZPJ9vxjjdc4A4K", + "H1TqCFbqINg1JERUc1FWc7hKElFGujJze1vtgLyEF5iXpJXnfz3rg1nBUmHhvLzIosyAVY6cakm2+whs", + "stAvNaFUwT85Xg72jfzIqoD4CawKCtBTM2AgVcvWt+YFSVnDe+dDoWHjm2epb+Oqu8fn9A1H27vLNRj3", + "0XOJMZZO2LIbe/FW86hwEy6omsbLr4esWZZlAB7DP0sVlgNpeuhkwiDzbPHn7O2jWBxad251WtHn3fKZ", + "sb83D6myUfUZAdqtLooBDd4GILHxcixAk1y1EMY9AQsCiPhlq7v1DF7oo8+7v/S7z3robwVPgY7BVhF9", + "W6516dd+ExxmjBLkTvNyvvVsrWd0h89lFPSbvZfqLmIbb29pPE/76e4K5zZd2uD888IeV6KK7qjQ0Ncl", + "K3aC83ppe1wvj2jS88smz973t9ZO27PeFdH7Bo/ib1L1mql3EZaqTq5+jaXK9DEkUitdd6DEDatUurdp", + "xW0oALh2Qqz9AZIVDd/lU5e50OOsAB004QrlQQArpRwAX6TMSw9l+PMyxZAeopYgtlcQRDOYskA+uuzo", + "nhyjRPAwDXIP2AiAToOASDlOoepyr6lEu/qN8S6VeXAt1xr9amW+TntfHWZKruv3+w25VoUpNcHWb/VW", + "f/VW34kFoNNKk3A1DzONmnGwtTJxrPCp9NgjymivSEGFxXxqwNHfFTG4qOBBwSYUaHEgTVwFQU1Ti5Qk", + "PXUD8fXQmyPgmEREEd8gyJT0tG4fVOZcdDVL3drb95vM8PUwgIDEBUB+IyTRcnrMpS2xGWM29wJWzaKN", + "2n1XQVIiGL5rMnlZbJWBe7pSCqndquYJySsWXRPxVcz/nmXouN1s5LbnyloRdyeovLeqUl1GmaqvZ9Hu", + "eNj9u7EzomHvYPOXv/7f3U9/+ZO/NktJkZJEdEMyhgewSzLvmqKzWmnrlfOZQrabllTYVjRRBMdgRQgu", + "ibFaxPi6CO+TfnaS5m9wvLAEeDmMKcv+vXJBf/1T/btbAY0fgHms3Mdvzn50F6llFXc8uh0TMXH52527", + "GJTzZtFcb5VEhfx19p53x/rPMutSLBx7YWSjHpQ7HlFIAyoHTKs5OAhIokjYs3m8KMAiOBzJavlkm0fP", + "uXfrw4oheagNt6mkyfriLVJ80GLkqmtmCLua9naf7NmK8UVMbi1ssW/TTcR2XYVFjWWPGeE1lRCO4Lxu", + "C41Rm8SJmrsssc4vcmO9CPLDbEDvU+gtp8/qP7uNbKEflqYH/QHrexYD/B1AK0P7F/a/Nief37HquJqp", + "x5xJW7OsnFmmojJJ1a33u4r1HT8E98FFHyn9zbgm2nyYk7SaGHwzZmrTZt/1hUOEUJJ4qTNqfspctHsX", + "Oq32sVwqZRZWVoCkfm9OnTBVLRNdj6AzjZqrKRGksBHQIU8huibKrKNggyAbkyMzIaJbLWpn6i4ICp6H", + "mR7sUJA5ky4axpZnuDnF19kMYFTFcuHpAdaR53rbevUcaqm8c8XN6NgNAWBURF1/upoyFTUpz764GUWq", + "Wly3ae89eJZXLeF+dWerQpz5HCXS9NHj3zBVL7kA4bg+pOXOs96A4B0SATG91Zw2jRLC0JiEQ56q5eff", + "Jpm38SwhGpExF6SQf9cpAhiI2NY8XsELXNBFDsMnn9wgSZAKquZac7Qi6YhgQcRhag48IBImgp/ziSGd", + "7devYEIbezzYXhFGBA3Q4dkJnEeo268Px8dTFNExCeaBVsAhG+lC/g0Q8t4enVjly2V8gwdFqoD0XBni", + "w7MTqGoqjALS6ve2e304zAlhOKGtg9ZObwtqvGqCgyVuQvZ7+NM6p5u4NMrZSWjloOemie4lcEwUEbJ1", + "8LvHyVsRYbLpS5A68aSgNySYCqs4JBG4nhtSobovpD9yV+mBuY87BuGNbUdSza0jHkne2m39pCnBnBpY", + "4na/b/Q0puzFi/Nql5v/sAF7+byN5DlAjycX0IJc72RKi/KvndZuf2steFYWqPRN+4HhVE25oJ8JgPlk", + "TSTcaNITZryDkckzYf0fiucMSKh4wn7/pPdLpnGMxdyhK8dVwmWdMEwkwlAiz5Ry+Acf9ZC1jEP6Uznl", + "aaS5CTKuzyTUFxbWPKU3+YywCKZ0RgbM3tOm2CgWkF86Rvp+NmpL+WiYqc3uZ0Fpz3k4r2A3G25TD9d1", + "9tAcwdU0i5IMIVvUsK5OS24JpYxBvUdJbE7KrGDBopMFFOiVAfdWJiYMM5XXezWVeS/J3BpbvQM2Suui", + "GR5sC4FC8Fm+8u0NfzwDZM/0hwIdZ9+QRW9ZnGDwQhFEaZjLXM7FFosRjiJv3P8k4iMc2QLGl8Qjor6C", + "FhYpxUSjTrhhPCQmaWQyV1POzN/pKGUqNX+PBL+SRGgRyGaftri21Tst6UIleRpDBmhT20LPuWlA3Pxy", + "SeZfewN2GMaubom0xecjyW1lZ5M+h0rkfDIN7frTm9Y89x+lUvHYkhQrFqI0YPJUJamyT52SKJsyG5pD", + "nVI5JeGAKY6+CFOWfv5180s+41fQXQgONZ0UmpglbX6h4dc6qOUQ69UPoalH+yOAgEFL3y6Dlv57IrDW", + "XVI5BVOGBPPFpLil7SwWW8uFG1UMB5ihhCcmjh2IyhSsLo0B5QdwFCEFR8n11dIm7GTNemxoiq+Wno1L", + "MYEElWMEVfUKh6m/u+8/T5IEgvgMHP9z/vYNgqtK74FplpuNAEeU6VsUhSlI8jB7b8Be4GCKjNwEucoG", + "LRoOWpl2EW4ArKm0jrPdLoi4v2jQfjHTdGj4S6+nhzLS8wH6/YsZ5UCfpSQeKn5J2KD1tYMKHyZUTdNR", + "9u2TH6F17v3nJUaA2ob3b7jiMZBmIL8Gzb2BWYi45bXRHGGUc6CiHWVEGRZLK994UG8xqFV5PJFFZHwZ", + "gAV10DoYOBvqoNUZtAibwW/W0DpoffVjwArR9YmxTPEfJ2tnRLTX72+sjruz+PWI0KWG+vh9XZC+tm9N", + "8LBC16LgYRbnsvrpHTRlnIy4dQ+Sz3McusIAP0W8FSKetVwUhDfoX7wHDPlGxCi4FQlM67ORk8CWaieG", + "LCCtJWgcLkrWKBzUSXA58RbVj6o6v6hW7NadsgBAjBz97d4D/cG8eSl0mPfZfc2LI8hRmRUGflzkCJvl", + "CLHj14hfEfU9UFz/vlipTaj5kPT7WOjnFbFyX460CjfbJDP33uTPBQAxANKOYhprXfUcYOqeE6bQC/i1", + "Z//XaTyQ2fYi4pOLA2RQGPEJiiizr3GF1yJ9KVpcQicTBpD1s1EBLhFT29yf//7nvwAoyib//ue/tDRt", + "/oLjvmmyY0Di1ospwUKNCFYXB+g3QpIujuiMuMVAakUyI2KOdvogZiYCPnmKTcoBG7B3RKWCFV4tTU4k", + "aQcE1YPBeihLibRhFLohHduEDcbA7FHh3Vk2qLzXE91Z9Dk1KygsQN+KjgYgApea7LVW/2r5rWdmzSX7", + "WdVWvmAxXc1fFLlWhnq7BsA1GQyg2Hfu4INdNGqfn7/Y6CHQMQxVQFIOkJjzYazw3PvJk1bzJMNRygwF", + "sGx4U6HOeK3999i2aWYAtiP+SBbgusLp9SZgY/IggoQOXz91hSbmYD/enGnYZ589dsFz9Qbam6+3OIXz", + "JmqkCN/ePjvaW8S5+VJA2UOowKjtHLVdzb+zoxNXG2bjwYj+Xm4NvVJbUSG7OhA3lQbvTS074mwc0UCh", + "roMFkvrHJFPVygTyWNjBOws1wm5d1fR3xftts5TNpfamyxK75Ffe3d8elUnXuUbyFH05rf28SVaRzjGV", + "Add9C9TSDXBiCxYa8SU7p0UqWmWQMn7f2ZWzVFyy7Pnk2B3I+zNN2alTVr0b7oEpHlcY4gMywkoRtkJS", + "y8dEzR+yXXSJApZYrr4v0uzfnxR031YsH5k/JjNWWEGb5oImtW/tBfqKqF9NizvcaDuDZ+HnRLhT7XIQ", + "w6qzZZmuKJiS4NIsCB6kl+u+J6ZJM9XXjPcjab6AnnUkFovynyJKA2U3x9UyBffEFpa7O/0WZlhLvb29", + "d15LYB4kg7PJyFmsTc02LOcs2Pihnnrv5TYzyH6Ul9lZGkXuxWNGhEJZ/ebiHbD5BdySVsv27rQtvQ4+", + "vHvdJSzg4IeW+VD5hSj75ZYlfLNhZik/yaSJTmgidqm7z+oknG/Yf+MuiLL64P+1/dJWCP+v7ZemRvh/", + "7RyaKuEbd0Ys/ftizfctcT9i4tMCNy0jDVgTg1qhqyTUrFVDIdW1/6HkVLPotSTVDK8/hdUmwmoRXUvl", + "VbsVdyqxmjke6EkmIzYftuGT80/8wSTV+7XyWYp0+XupLD972AItXICdFz5RhlJJHqEDJc0ornhtNDRX", + "5wdy6fXhSPfkuAOI7GjUQUIhGyByT8ZrB8e9C7d23vu3XB/GIzpJeSqLsScxVsGUSBusFJEyA35sYnd+", + "PdcK3t8xlfbv8+q4d7n6J93fkcRf3VDDvM0L1CqZ37VqKvPb9lrmt8XzTezaO1eU3+ZJ2qhxKnRB1E3J", + "uBRrvujs6IPLp4ugD1pRydUFBBrEwYD9H61//K4Ijj/94oJk0n5/ew9+J2z26RcXJ8NOHakQpgS1yTsP", + "3xzDs98Eos8hv2cekleFwxQEANJzCWz+4xSk/OWzuYbkqPCnhtRIQyqga7mGZPfiblWkchKse9eRHL35", + "EG6TmPzUku5DS5LpeEwDSpjK62UtOInZcnuPMLaM2fehgnNH6aJtrCVlh3KFAJpna793x55s8vtXjlxi", + "+MfpI89NVEzo1JH8MqzXR743eujfL3O+fz3kMZOYEfgXUZdomdJXhhEyPcapAqfEPEMIeH0iYaT2bMQe", + "yqsfyjRJuFDSZIsEARjKYampFoB9mSXLySJ92SEhMS4lsjNgkEBefzax/JuXZG5yQVKeV/XPVmrzP/pi", + "r8q5OB/0GN2+jOVPNNpIxrrnY2zzKT+cjPVgrONeJK2TUpr6dnYwQKEckewk8yy4j36mbLLxqDxQDbPK", + "1lbIZ+QRtTbHtiChXwV6ycVlU6bgKZDzCHhDcYXfofalwYP8SQ+vhIF6Ys6PJpp75xsLVY8e0mmdVjlJ", + "EKWhZh2OhbjLdyx4PLQ/mgyf+lTY/Img1AV21IdmMnr2e1Cx33CFaJxERMs9JERdQ016N62w5NJkU1mo", + "EbYeE9THphhCYNJ3SVdnxGYTh+cIt2FteJlc3C4v14z4ZHXagGxyFyPvyRswYCaNN3E5vy9QxmSR4kiS", + "iAQKXU1pMIUcAvo3U5oQwvtxklxkSYM2DtArOKnF3EkweVsSoUXHgDPJI2JSA8zi+OJgMcflx9NT6GTS", + "B5hslhcHyOW1zC4IqVsVcwJkBUze2EwHbU1JgkeR2dELLWcX1rdhswXkSZ0GzJc5gJErOyAdo4tCEoGL", + "miwCjqG+5hP5UKJspz4Vn1mL4kgA4gxtEha26kzXNPLnD9jqe4s7NMxlYMC441QGC8C85pMsDWCJlHGS", + "NCVfCyZQ8SyOl9AwaheqDEoV8lT9VaqQCAGdLXXXETdq48D8Q+FLTai2SkhWpxHIz/tAY/JyeVGlmWqh", + "JIb51yyOW52WhaeQz2sNC8OKnBDVARcfEvTOFBI//LQlrJPSoczsCzkdKjeHrWNdL3Lb8tw/vEXLIir8", + "EfTS8gtADgVlTlSBveV5SZ1HFRtuKrdXZTFT0MV3Rtwqu7JQCbDZg8BCDcHvQGld9U6QFYTLqtXd94PB", + "IgSPOWxALqwGKs2ztV4SvntCur0tWVhqEwr5SZvrPzk0IswkXVIYEOoaSoRdsTzIhBtMOZcFsh+RKZ5R", + "LmzOalszKaNMMFkY7dH6G11oUr2wdbMurHh+YG1NCBc/2Tl60N16Kfl7uE95j5cFbTvj+B0nUkPePIkw", + "GglKxijBqSRaWkpjgkxNBpv6mOBg6sre9gbs/ZQgW+yuYEDIaqNSiS624osOGqUKRVhMQNsxH43vkSAB", + "j2PCQlPAcsCmBM+oVtUEirAiLJh3JYGCpjOSl3zQqrt900HwpJOVTOwgV2kTDAwXhTqaFygRBIjIqMus", + "VLRywETK/tvk+tPDXjhALxCRCo8iKqdZdv0Ah4QF3kR65983G7t9I+45UYulJh/kledGvPQhn32Ktsys", + "2O938SL0yFxbuHAVARuw+SVCr6xXDcu+Yud5ec3/wCNt1urW+EAvMxmKl53i7+NJplRf++ezjLJHMkzN", + "dKRcg/qHfWvJGApKWem5xdpkb/rgkuWOz9C8Fs/b/OL+PLmBjew74YSdJZXe/RPki/4eWK7F6o147gMZ", + "B60tqWAVe0AWbIF6OPGJiwKX+y7YsDlwGTcu8hwlMOhUpir+T2Zcevs27gE3ZcbO4rrwAF5gz5R1kwjX", + "8eW82rafAVdqsP+H+QvWVJj/ajnhQzK+/EXg3pjdScbeDMNL8Dzi+Ed/lwm4ECYEzhZwfTwpmAq2wMID", + "Uxssbp2MQ3Sc//3H09ONOi4h1FIeIdQj5hDlQpBB7Klv93ZGhKChK7Z3dHpsS/NRiUTKeuhtTKEC3iUh", + "CZTWoDyVpix/r1ihfrFsWKUEPWFKzBNOmVoJRd70boD5eqNiY/fMJ20Suh/+8Ris8I+PSQHv0OKKXcBy", + "LVJhVeuM55zTKDMVArW0hUc81aMvVNBHYxoROZeKxMYzb5xGcIggTamtYmP7mRi8DqJKIn0eOhCzlBAR", + "UykpZ3LAbMHshAg9t+4O5VJzJyOv8V7hjGueGdb3fTiwQVF98NnCqg5r5Xr6OElcPX2fk5QF7xtAegke", + "aUjO4xGPaIAiyi4lakf00igdaCZRpP/YWOrSNoR+t12j5+YnS2P6hI25t4yBodmMmH+MsI0yW3OPiI+O", + "rb0ixcPi+A9stJ+tyZV8TRAcdRWNSRYujFJFI/rZsDo9CJWKBqb2cx6sBmVrbbzagJ0SJXQbLAgKeBSR", + "QDnjymYieLA5SPv9nSChkNdhhwBwwPDqP8cw49HZB2hnSut2Bkz/AwZ+f3hmXmLH2NoICoAyoq64uEQn", + "m29XOPmeA5r+g73kzAKXRo15N/zn8936saC1Z0jWHFGeLFOAePLDu3FaCe6nteBxWgsgGD9bTXsicABC", + "sZymKuRXzG8ZmPEojfU/zB8nq1I6KBxMP0LT70baNeCsnMYt8FEcSrumkJgyKw/yQGEQ9lj9SzXi3BJA", + "iCl57nlvgUP1I1L37Rvli3j8Dp8mLUZdCaPv5mzd981nYXCZior4eCzH3FCaW4niy61PV5jWW5+eRzy4", + "lChlikZgMClImhgyJ+of80x39uEPxASIjnTFlxG5TqiAnB9g0y2MRPSKJcJIERFThqNNWLMZBHL2OSsW", + "nnEKQcpBRCFMjIYEJTyKIC/J1ZQwpFcDhio3QOGdVtqc+cU2xSdGxdGIBDwmLo/hhk91+xum6iUX5aSE", + "3wtffF/Av16PXqpe54o8jPUzflNexlN8DW7NYWqfiR1E7Vc8/9GYgjoI9mbQ2unLQauDBq3teNDSO3CE", + "wYSKFXqCYspSRWQPHRv7FoSh7vWRJAFnoXTpFJ0Fb6cv64JSDVnWRDjuQb/7FHssVQEq39lJfOxBt0O6", + "PwTYoHbxwNkzGXbg0IWIpwocuN25sq1CosA8snHvL7CFM/JTt2/Cyf9mj2+JR8Eua3ZZ2HrD2bOEeyut", + "bi6oYsplnqcPBTjBAVXzDsJRxIPcepDK7HWgm4EyEgRfah2qN2DvslR/NhACHZ196DijGQqpvDQjWLtY", + "D72dESHTUQYcAm5gLHiwGSQcMMVRgKMgjTTdkvGYBBDDENGYKlljV8tAucvCcfkkno13Hy3qHpsxyU8T", + "sHs5WcgKxW2ard4UJIgwjYtGpSpyQPSFJ10w+470oFxfw+PIPm8FgkuJ7FBdEtEJHUX2sUb20HstcuCY", + "DFgSYcaIQKk0fkca9G4iiJSpCYzRA0BlTkNRHZQnOkkEV9ZMHHEupLHsagr/eIqkIskSMntnRj6FNd9R", + "YlUzuJ3pgRSGCgz115JtgvSGGEoxCNd0pK/pB3D2MQA9dALWx3Lw3ws6mRChTwU2TNY8jZpj7dBpDn0p", + "0qM2q/h51qpZVvFs1II3d8HTeWmiiqFrOAQBep0XWM/kl7Q2l4n9tF70xW+6U8O5y17+fiDsp29c5Y9S", + "rOm84FzdNBd5TuGPLS14AfLSUS0FKKxOR9A4IuEuIwQa5x14sHQDjznLAC6FHdSlE/j+CKF/v9Fx952Y", + "+HHTVilLQKkUSU2o1Or0nd8FBd5N3s4Hjg69Qd7O7ypeCfIuPlzc6HcVqVSyA7pyCz98Zs67ClAy6Tkh", + "jUVdgJLhetaRYKmi9NG2aaYm2RF/JAnevj2vIb87tP/U+huoDAVk+U12Jjba5W0hcaLm7nGRjysPgJJ+", + "hmAMX+KHzIfg7vIt3OB5/fbIw9Fp7eP6zwpE9/Z+n5dpPTl+/GWHimeudLFs6luni0UwpTNSb3Qvn2CL", + "okSQbsITeFwJDcIsPtxdprDoTT4jO7zNVWX/hahLcUxCFFJBAhXNEWWKA0cwc/xZIsG1JgDfuZj7jOnF", + "k/tS8PjQrmbFfWjPlDWG5W++8bwbYoW7M8dtlpjQvuGl3b1ta4aHKEOvnqM2uVbCZNxFY635IDrOUEqu", + "A0JCCTS5UQR4q19j2aSfyXAyagLlktzJb21uahSkUvHY7f3JMWrjVPHuhDC9F1rUH4Mkmwg+o6Ep3Zgj", + "dcYjg9WtGoSua3fVQoUN78uVCwPcg8gwTS6kyWealNmCcV1oHbRGlGEAbmWW4vKZMgFVej5MIawhPzuO", + "clo/rzCr+bWdsqMpUSs5DomKc5Mab+PnNfeYr7miY6q700q3XbPies18VRu6kN5FwtzMj/l+zdYfvx/3", + "SiofpWelNZ3PMoW0zmz+fZFg//7uh/s2l398xO74r4hTvgumchhAj+gjmNc8wBEKyYxEPIG6e6Ztq9NK", + "RdQ6aE2VSg42NyPdbsqlOtjv7/dbXz99/f8DAAD//zlim4+EZgEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index dae2115b..fd55aedb 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -203,6 +203,38 @@ components: $ref: "#/components/schemas/CreateInstanceRequestCredentialInject" minItems: 1 + AutoStandbyPolicy: + type: object + description: | + Linux-only automatic standby policy based on active inbound TCP connections + observed from the host conntrack table. + properties: + enabled: + type: boolean + description: Whether automatic standby is enabled for this instance. + default: false + example: true + idle_timeout: + type: string + description: | + How long the instance must have zero qualifying inbound TCP connections + before Hypeman places it into standby. + example: "5m" + ignore_source_cidrs: + type: array + description: Optional client CIDRs that should not keep the instance awake. + items: + type: string + example: ["10.0.0.0/8", "192.168.0.0/16"] + ignore_destination_ports: + type: array + description: Optional destination TCP ports that should not keep the instance awake. + items: + type: integer + minimum: 1 + maximum: 65535 + example: [22, 9000] + UpdateInstanceRequest: type: object properties: @@ -217,6 +249,8 @@ components: are accepted. Use this to rotate real credential values without restarting the VM. example: OUTBOUND_OPENAI_KEY: new-rotated-key-456 + auto_standby: + $ref: "#/components/schemas/AutoStandbyPolicy" CreateInstanceRequest: type: object @@ -323,6 +357,8 @@ components: snapshot_policy: description: Snapshot compression policy for this instance. Controls compression settings applied when creating snapshots or entering standby. $ref: "#/components/schemas/SnapshotPolicy" + auto_standby: + $ref: "#/components/schemas/AutoStandbyPolicy" skip_kernel_headers: type: boolean description: | @@ -766,6 +802,8 @@ components: example: cloud-hypervisor snapshot_policy: $ref: "#/components/schemas/SnapshotPolicy" + auto_standby: + $ref: "#/components/schemas/AutoStandbyPolicy" PathInfo: type: object From d2892cb50b301cdf498cdc7ed80fdeda9905e83a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Sat, 4 Apr 2026 16:16:57 -0400 Subject: [PATCH 02/13] Expose auto-standby policy in Stainless config --- stainless.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stainless.yaml b/stainless.yaml index c88f36cc..fa68eb7c 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -72,6 +72,7 @@ resources: instances: models: + auto_standby_policy: "#/components/schemas/AutoStandbyPolicy" snapshot_policy: "#/components/schemas/SnapshotPolicy" snapshot_schedule: "#/components/schemas/SnapshotSchedule" snapshot_schedule_retention: "#/components/schemas/SnapshotScheduleRetention" From d3016516f09addd082b69ab20a6beb6eb106dccd Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Sun, 5 Apr 2026 11:38:36 -0400 Subject: [PATCH 03/13] Address auto-standby review feedback --- cmd/api/api/auto_standby.go | 2 +- cmd/api/api/instances_test.go | 42 ++++++++++++++++++++++++ lib/instances/metadata_clone.go | 11 +++++++ lib/instances/update.go | 14 ++++---- lib/instances/update_test.go | 58 +++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 lib/instances/metadata_clone.go diff --git a/cmd/api/api/auto_standby.go b/cmd/api/api/auto_standby.go index 96e20fed..c483727b 100644 --- a/cmd/api/api/auto_standby.go +++ b/cmd/api/api/auto_standby.go @@ -26,7 +26,7 @@ func toDomainAutoStandbyPolicy(policy *oapi.AutoStandbyPolicy) (*autostandby.Pol if policy.IgnoreDestinationPorts != nil { out.IgnoreDestinationPorts = make([]uint16, 0, len(*policy.IgnoreDestinationPorts)) for _, port := range *policy.IgnoreDestinationPorts { - if port < 0 || port > 65535 { + if port < 1 || port > 65535 { return nil, fmt.Errorf("auto_standby.ignore_destination_ports must be between 1 and 65535") } out.IgnoreDestinationPorts = append(out.IgnoreDestinationPorts, uint16(port)) diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index c31dd400..3244e9c2 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -636,6 +636,48 @@ func TestUpdateInstance_MapsAutoStandbyPatch(t *testing.T) { assert.True(t, *instance.AutoStandby.Enabled) } +func TestUpdateInstance_RejectsZeroAutoStandbyIgnoreDestinationPort(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-auto-standby", + Name: "inst-update-auto-standby", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + enabled := true + idleTimeout := "10m" + ignoreDestinationPorts := []int{0} + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + AutoStandby: &oapi.AutoStandbyPolicy{ + Enabled: &enabled, + IdleTimeout: &idleTimeout, + IgnoreDestinationPorts: &ignoreDestinationPorts, + }, + }, + }) + require.NoError(t, err) + + badReq, ok := resp.(oapi.UpdateInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_auto_standby", badReq.Code) + assert.Contains(t, badReq.Message, "between 1 and 65535") + assert.Nil(t, mockMgr.lastReq) +} + func TestUpdateInstance_RequiresBody(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go new file mode 100644 index 00000000..1112cff4 --- /dev/null +++ b/lib/instances/metadata_clone.go @@ -0,0 +1,11 @@ +package instances + +func cloneMetadata(src *metadata) *metadata { + if src == nil { + return nil + } + + return &metadata{ + StoredMetadata: cloneStoredMetadataForFork(src.StoredMetadata), + } +} diff --git a/lib/instances/update.go b/lib/instances/update.go index b0690349..c9975624 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -43,11 +43,13 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta if len(req.Env) > 0 && inst.State != StateRunning && inst.State != StateInitializing { return nil, fmt.Errorf("%w: instance must be running or initializing to update env (current state: %s)", ErrInvalidState, inst.State) } + nextMeta := meta if req.AutoStandby != nil { - meta.AutoStandby = cloneAutoStandbyPolicy(req.AutoStandby) + nextMeta = cloneMetadata(meta) + nextMeta.AutoStandby = cloneAutoStandbyPolicy(req.AutoStandby) } if len(req.Env) == 0 { - if err := m.saveMetadata(meta); err != nil { + if err := m.saveMetadata(nextMeta); err != nil { return nil, fmt.Errorf("save metadata: %w", err) } @@ -60,8 +62,8 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta return updated, nil } - prevEnv := cloneEnvMap(meta.Env) - nextEnv := cloneEnvMap(meta.Env) + prevEnv := cloneEnvMap(nextMeta.Env) + nextEnv := cloneEnvMap(nextMeta.Env) if nextEnv == nil { nextEnv = make(map[string]string) } @@ -69,7 +71,7 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta nextEnv[k] = v } - if err := validateCredentialEnvBindings(meta.Credentials, nextEnv); err != nil { + if err := validateCredentialEnvBindings(nextMeta.Credentials, nextEnv); err != nil { return nil, err } @@ -79,7 +81,7 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta return nil, fmt.Errorf("egress proxy service unavailable") } - if err := applyUpdatedInstanceEnv(ctx, log, id, meta, prevEnv, nextEnv, m.saveMetadata, svc); err != nil { + if err := applyUpdatedInstanceEnv(ctx, log, id, nextMeta, prevEnv, nextEnv, m.saveMetadata, svc); err != nil { return nil, err } diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 135dd04e..3389820a 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -181,3 +181,61 @@ func TestApplyUpdatedInstanceEnvReturnsRollbackFailure(t *testing.T) { assert.Equal(t, prevEnv, meta.Env) require.Len(t, svc.calls, 2) } + +func TestApplyUpdatedInstanceEnvSavesAutoStandbyAlongsideEnvWithoutMutatingOriginal(t *testing.T) { + t.Parallel() + + original := &metadata{ + StoredMetadata: StoredMetadata{ + Id: "inst-autostandby-copy", + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + Credentials: map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{{ + As: CredentialInjectAs{ + Header: "Authorization", + Format: "Bearer ${value}", + }, + }}, + }, + }, + Env: map[string]string{"OUTBOUND_OPENAI_KEY": "old"}, + AutoStandby: &autostandby.Policy{ + Enabled: false, + IdleTimeout: "5m0s", + }, + }, + } + updated := cloneMetadata(original) + updated.AutoStandby = &autostandby.Policy{ + Enabled: true, + IdleTimeout: "10m0s", + IgnoreSourceCIDRs: []string{"10.0.0.0/8"}, + IgnoreDestinationPorts: []uint16{22}, + } + + prevEnv := cloneEnvMap(updated.Env) + nextEnv := map[string]string{"OUTBOUND_OPENAI_KEY": "new"} + svc := &fakeUpdateInstanceRulesService{} + + var saved *metadata + err := applyUpdatedInstanceEnv(context.Background(), nil, updated.Id, updated, prevEnv, nextEnv, func(meta *metadata) error { + saved = cloneMetadata(meta) + return nil + }, svc) + require.NoError(t, err) + + require.NotNil(t, saved) + require.NotNil(t, saved.AutoStandby) + assert.True(t, saved.AutoStandby.Enabled) + assert.Equal(t, "10m0s", saved.AutoStandby.IdleTimeout) + assert.Equal(t, []string{"10.0.0.0/8"}, saved.AutoStandby.IgnoreSourceCIDRs) + assert.Equal(t, []uint16{22}, saved.AutoStandby.IgnoreDestinationPorts) + assert.Equal(t, nextEnv, saved.Env) + + require.NotNil(t, original.AutoStandby) + assert.False(t, original.AutoStandby.Enabled) + assert.Equal(t, "5m0s", original.AutoStandby.IdleTimeout) + assert.Equal(t, map[string]string{"OUTBOUND_OPENAI_KEY": "old"}, original.Env) +} From 875dd25ffbbff09fd5211c49b5c6245d49c315cb Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Sun, 5 Apr 2026 11:46:41 -0400 Subject: [PATCH 04/13] Default-skip compression standby integration test --- .../compression_integration_linux_test.go | 1 + ...compression_integration_test_helpers_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 lib/instances/compression_integration_test_helpers_test.go diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index f58b5b1f..75ceef96 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -35,6 +35,7 @@ const compressionGuestExecTimeout = 20 * time.Second func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { t.Parallel() + requireStandbyRestoreCompressionManualRun(t) runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "cloud-hypervisor", diff --git a/lib/instances/compression_integration_test_helpers_test.go b/lib/instances/compression_integration_test_helpers_test.go new file mode 100644 index 00000000..deff89f8 --- /dev/null +++ b/lib/instances/compression_integration_test_helpers_test.go @@ -0,0 +1,17 @@ +//go:build linux + +package instances + +import ( + "os" + "testing" +) + +const standbyRestoreCompressionManualEnv = "HYPEMAN_RUN_STANDBY_RESTORE_COMPRESSION_TESTS" + +func requireStandbyRestoreCompressionManualRun(t *testing.T) { + t.Helper() + if os.Getenv(standbyRestoreCompressionManualEnv) != "1" { + t.Skipf("set %s=1 to run standby/restore compression integration tests", standbyRestoreCompressionManualEnv) + } +} From 4043a131e48f346727f13aaf3e3807ab9715f754 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 11:03:41 -0400 Subject: [PATCH 05/13] Clarify auto-standby update metadata handling --- lib/autostandby/README.md | 1 + lib/instances/metadata_clone.go | 4 +++- lib/instances/update.go | 3 +-- lib/instances/update_test.go | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/autostandby/README.md b/lib/autostandby/README.md index c5b90718..8c745d62 100644 --- a/lib/autostandby/README.md +++ b/lib/autostandby/README.md @@ -40,6 +40,7 @@ This is intended for probes, internal callers, or ports that should not keep a V - Linux only - TCP only +- IPv4 conntrack only - Wake-on-traffic is not part of this feature Wake-on-traffic would require a separate host-owned listener or forwarding layer that can accept a connection while the VM is asleep, trigger restore, and then hand traffic through once the VM is running. diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go index 1112cff4..69bdb053 100644 --- a/lib/instances/metadata_clone.go +++ b/lib/instances/metadata_clone.go @@ -1,6 +1,8 @@ package instances -func cloneMetadata(src *metadata) *metadata { +// deepCopyMetadata returns a metadata copy that can be mutated without +// affecting the originally loaded instance metadata. +func deepCopyMetadata(src *metadata) *metadata { if src == nil { return nil } diff --git a/lib/instances/update.go b/lib/instances/update.go index c9975624..f8556219 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -43,9 +43,8 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta if len(req.Env) > 0 && inst.State != StateRunning && inst.State != StateInitializing { return nil, fmt.Errorf("%w: instance must be running or initializing to update env (current state: %s)", ErrInvalidState, inst.State) } - nextMeta := meta + nextMeta := deepCopyMetadata(meta) if req.AutoStandby != nil { - nextMeta = cloneMetadata(meta) nextMeta.AutoStandby = cloneAutoStandbyPolicy(req.AutoStandby) } if len(req.Env) == 0 { diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 3389820a..b55b7d67 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -207,7 +207,7 @@ func TestApplyUpdatedInstanceEnvSavesAutoStandbyAlongsideEnvWithoutMutatingOrigi }, }, } - updated := cloneMetadata(original) + updated := deepCopyMetadata(original) updated.AutoStandby = &autostandby.Policy{ Enabled: true, IdleTimeout: "10m0s", @@ -221,7 +221,7 @@ func TestApplyUpdatedInstanceEnvSavesAutoStandbyAlongsideEnvWithoutMutatingOrigi var saved *metadata err := applyUpdatedInstanceEnv(context.Background(), nil, updated.Id, updated, prevEnv, nextEnv, func(meta *metadata) error { - saved = cloneMetadata(meta) + saved = deepCopyMetadata(meta) return nil }, svc) require.NoError(t, err) From 5c1df95a413e7948333b37af6a0e9c6acc1b3174 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 11:42:08 -0400 Subject: [PATCH 06/13] Wire auto-standby controller into API app --- cmd/api/auto_standby_unsupported.go | 15 -------------- cmd/api/main.go | 7 +++++-- cmd/api/wire.go | 3 +++ cmd/api/wire_gen.go | 4 ++++ .../providers}/auto_standby_linux.go | 20 ++++++++----------- lib/providers/auto_standby_unsupported.go | 15 ++++++++++++++ 6 files changed, 35 insertions(+), 29 deletions(-) delete mode 100644 cmd/api/auto_standby_unsupported.go rename {cmd/api => lib/providers}/auto_standby_linux.go (70%) create mode 100644 lib/providers/auto_standby_unsupported.go diff --git a/cmd/api/auto_standby_unsupported.go b/cmd/api/auto_standby_unsupported.go deleted file mode 100644 index b568b7ab..00000000 --- a/cmd/api/auto_standby_unsupported.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build !linux - -package main - -import ( - "context" - "log/slog" - - "github.com/kernel/hypeman/lib/instances" - "golang.org/x/sync/errgroup" -) - -func startAutoStandbyController(*errgroup.Group, context.Context, *slog.Logger, instances.Manager) bool { - return false -} diff --git a/cmd/api/main.go b/cmd/api/main.go index 8696d39e..42de6b88 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -495,8 +495,11 @@ func run() error { logger.Info("starting guest memory controller") return app.GuestMemoryController.Start(gctx) }) - if startAutoStandbyController(grp, gctx, logger, app.InstanceManager) { - logger.Info("auto-standby controller enabled") + if app.AutoStandbyController != nil { + grp.Go(func() error { + logger.Info("starting auto-standby controller") + return app.AutoStandbyController.Run(gctx) + }) } // Run the server diff --git a/cmd/api/wire.go b/cmd/api/wire.go index b50c27e2..0888ec4a 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -9,6 +9,7 @@ import ( "github.com/google/wire" "github.com/kernel/hypeman/cmd/api/api" "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" @@ -39,6 +40,7 @@ type application struct { BuildManager builds.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller VMMetricsManager *vm_metrics.Manager Registry *registry.Registry ApiService *api.ApiService @@ -61,6 +63,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideBuildManager, providers.ProvideResourceManager, providers.ProvideGuestMemoryController, + providers.ProvideAutoStandbyController, providers.ProvideVMMetricsManager, providers.ProvideRegistry, api.New, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index a5007a7e..05e95b3c 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -10,6 +10,7 @@ import ( "context" "github.com/kernel/hypeman/cmd/api/api" "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" @@ -69,6 +70,7 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } + autostandbyController := providers.ProvideAutoStandbyController(instancesManager, logger) vm_metricsManager, err := providers.ProvideVMMetricsManager(instancesManager, config, logger) if err != nil { return nil, nil, err @@ -92,6 +94,7 @@ func initializeApp() (*application, func(), error) { BuildManager: buildsManager, ResourceManager: resourcesManager, GuestMemoryController: controller, + AutoStandbyController: autostandbyController, VMMetricsManager: vm_metricsManager, Registry: registry, ApiService: apiService, @@ -117,6 +120,7 @@ type application struct { BuildManager builds.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller VMMetricsManager *vm_metrics.Manager Registry *registry.Registry ApiService *api.ApiService diff --git a/cmd/api/auto_standby_linux.go b/lib/providers/auto_standby_linux.go similarity index 70% rename from cmd/api/auto_standby_linux.go rename to lib/providers/auto_standby_linux.go index 6a901e03..f73bb47c 100644 --- a/cmd/api/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -1,6 +1,6 @@ //go:build linux -package main +package providers import ( "context" @@ -9,7 +9,6 @@ import ( "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/instances" - "golang.org/x/sync/errgroup" ) type autoStandbyInstanceStore struct { @@ -42,19 +41,16 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string return err } -func startAutoStandbyController(grp *errgroup.Group, ctx context.Context, logger *slog.Logger, manager instances.Manager) bool { - if grp == nil || ctx == nil || logger == nil || manager == nil { - return false +// ProvideAutoStandbyController provides the Linux auto-standby controller. +func ProvideAutoStandbyController(instanceManager instances.Manager, log *slog.Logger) *autostandby.Controller { + if instanceManager == nil || log == nil { + return nil } - controller := autostandby.NewController( - autoStandbyInstanceStore{manager: manager}, + return autostandby.NewController( + autoStandbyInstanceStore{manager: instanceManager}, autostandby.NewConntrackSource(), - logger.With("controller", "auto_standby"), + log.With("controller", "auto_standby"), 5*time.Second, ) - grp.Go(func() error { - return controller.Run(ctx) - }) - return true } diff --git a/lib/providers/auto_standby_unsupported.go b/lib/providers/auto_standby_unsupported.go new file mode 100644 index 00000000..f1180785 --- /dev/null +++ b/lib/providers/auto_standby_unsupported.go @@ -0,0 +1,15 @@ +//go:build !linux + +package providers + +import ( + "log/slog" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" +) + +// ProvideAutoStandbyController is unavailable on non-Linux platforms. +func ProvideAutoStandbyController(instances.Manager, *slog.Logger) *autostandby.Controller { + return nil +} From 789a08c5d839e25ae43fa3f9b96d47912fd9f4b8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 13:32:15 -0400 Subject: [PATCH 07/13] Make auto-standby event-driven and observable --- cmd/api/api/api.go | 4 + cmd/api/api/auto_standby_status.go | 80 ++ cmd/api/api/auto_standby_status_test.go | 170 ++++ cmd/api/wire_gen.go | 2 +- lib/autostandby/README.md | 24 +- lib/autostandby/conntrack_events_linux.go | 338 +++++++ .../conntrack_events_linux_test.go | 47 + lib/autostandby/conntrack_linux.go | 1 + lib/autostandby/conntrack_linux_test.go | 2 + lib/autostandby/conntrack_unsupported.go | 5 + lib/autostandby/controller.go | 932 ++++++++++++++++-- lib/autostandby/controller_test.go | 338 +++++-- lib/autostandby/metrics.go | 164 +++ lib/autostandby/status.go | 50 + lib/autostandby/types.go | 11 +- lib/autostandby/types_test.go | 15 + lib/instances/README.md | 2 + lib/instances/auto_standby.go | 17 + .../auto_standby_integration_linux_test.go | 55 +- lib/instances/auto_standby_runtime.go | 34 + lib/instances/fork.go | 4 +- lib/instances/fork_test.go | 2 +- lib/instances/lifecycle_events.go | 74 ++ lib/instances/manager.go | 50 +- lib/instances/metadata_clone.go | 3 +- lib/instances/snapshot.go | 8 +- lib/instances/storage.go | 2 + lib/oapi/oapi.go | 825 +++++++++++----- lib/providers/auto_standby_linux.go | 67 +- openapi.yaml | 98 ++ stainless.yaml | 4 + 31 files changed, 3014 insertions(+), 414 deletions(-) create mode 100644 cmd/api/api/auto_standby_status.go create mode 100644 cmd/api/api/auto_standby_status_test.go create mode 100644 lib/autostandby/conntrack_events_linux.go create mode 100644 lib/autostandby/conntrack_events_linux_test.go create mode 100644 lib/autostandby/metrics.go create mode 100644 lib/autostandby/status.go create mode 100644 lib/autostandby/types_test.go create mode 100644 lib/instances/auto_standby_runtime.go create mode 100644 lib/instances/lifecycle_events.go diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index 47f828ed..61e415d6 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -2,6 +2,7 @@ package api import ( "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" @@ -27,6 +28,7 @@ type ApiService struct { BuildManager builds.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller + AutoStandbyController *autostandby.Controller VMMetricsManager *vm_metrics.Manager } @@ -44,6 +46,7 @@ func New( buildManager builds.Manager, resourceManager *resources.Manager, guestMemoryController guestmemory.Controller, + autoStandbyController *autostandby.Controller, vmMetricsManager *vm_metrics.Manager, ) *ApiService { return &ApiService{ @@ -57,6 +60,7 @@ func New( BuildManager: buildManager, ResourceManager: resourceManager, GuestMemoryController: guestMemoryController, + AutoStandbyController: autoStandbyController, VMMetricsManager: vmMetricsManager, } } diff --git a/cmd/api/api/auto_standby_status.go b/cmd/api/api/auto_standby_status.go new file mode 100644 index 00000000..a9fda3c3 --- /dev/null +++ b/cmd/api/api/auto_standby_status.go @@ -0,0 +1,80 @@ +package api + +import ( + "context" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/oapi" + "github.com/samber/lo" +) + +func (s *ApiService) GetAutoStandbyStatus(ctx context.Context, request oapi.GetAutoStandbyStatusRequestObject) (oapi.GetAutoStandbyStatusResponseObject, error) { + log := logger.FromContext(ctx) + + inst, err := s.InstanceManager.GetInstance(ctx, request.Id) + if err != nil { + if err == instances.ErrNotFound || err == instances.ErrAmbiguousName { + return oapi.GetAutoStandbyStatus404JSONResponse{ + Code: "not_found", + Message: "instance not found", + }, nil + } + log.ErrorContext(ctx, "failed to resolve instance for auto-standby status", "instance_id", request.Id, "error", err) + return oapi.GetAutoStandbyStatus500JSONResponse{ + Code: "internal_error", + Message: "failed to load instance", + }, nil + } + + snapshot := autostandby.StatusSnapshot{ + Supported: false, + Configured: inst.AutoStandby != nil, + Enabled: inst.AutoStandby != nil && inst.AutoStandby.Enabled, + TrackingMode: "conntrack_events_v4_tcp", + Status: autostandby.StatusUnsupported, + Reason: autostandby.ReasonUnsupportedPlatform, + } + if s.AutoStandbyController != nil { + snapshot = s.AutoStandbyController.Describe(instanceToAutoStandby(*inst)) + } + + return oapi.GetAutoStandbyStatus200JSONResponse(toOAPIAutoStandbyStatus(snapshot)), nil +} + +func instanceToAutoStandby(inst instances.Instance) autostandby.Instance { + return autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + } +} + +func toOAPIAutoStandbyStatus(status autostandby.StatusSnapshot) oapi.AutoStandbyStatus { + out := oapi.AutoStandbyStatus{ + ActiveInboundConnections: status.ActiveInboundCount, + Configured: status.Configured, + Eligible: status.Eligible, + Enabled: status.Enabled, + Reason: oapi.AutoStandbyStatusReason(status.Reason), + Status: oapi.AutoStandbyStatusStatus(status.Status), + Supported: status.Supported, + TrackingMode: status.TrackingMode, + } + if status.IdleTimeout != "" { + out.IdleTimeout = lo.ToPtr(status.IdleTimeout) + } + out.IdleSince = status.IdleSince + out.LastInboundActivityAt = status.LastInboundActivityAt + out.NextStandbyAt = status.NextStandbyAt + if status.CountdownRemaining != nil { + out.CountdownRemaining = lo.ToPtr(status.CountdownRemaining.String()) + } + return out +} + diff --git a/cmd/api/api/auto_standby_status_test.go b/cmd/api/api/auto_standby_status_test.go new file mode 100644 index 00000000..b4aecfd1 --- /dev/null +++ b/cmd/api/api/auto_standby_status_test.go @@ -0,0 +1,170 @@ +package api + +import ( + "context" + "net/netip" + "testing" + "time" + + "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type captureStatusManager struct { + instances.Manager + instance *instances.Instance + err error +} + +func (m *captureStatusManager) GetInstance(context.Context, string) (*instances.Instance, error) { + if m.err != nil { + return nil, m.err + } + return m.instance, nil +} + +type statusStore struct { + instances []autostandby.Instance + runtime map[string]*autostandby.Runtime + events chan autostandby.InstanceEvent +} + +func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, error) { + return append([]autostandby.Instance(nil), s.instances...), nil +} + +func (s *statusStore) StandbyInstance(context.Context, string) error { return nil } + +func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { + if s.runtime == nil { + s.runtime = make(map[string]*autostandby.Runtime) + } + s.runtime[id] = runtime + return nil +} + +func (s *statusStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { + if s.events == nil { + s.events = make(chan autostandby.InstanceEvent) + } + return s.events, func() {}, nil +} + +type statusConnectionSource struct { + connections []autostandby.Connection +} + +func (s *statusConnectionSource) ListConnections(context.Context) ([]autostandby.Connection, error) { + return append([]autostandby.Connection(nil), s.connections...), nil +} + +func (s *statusConnectionSource) OpenStream(context.Context) (autostandby.ConnectionStream, error) { + return &statusConnectionStream{ + events: make(chan autostandby.ConnectionEvent), + errs: make(chan error), + }, nil +} + +type statusConnectionStream struct { + events chan autostandby.ConnectionEvent + errs chan error +} + +func (s *statusConnectionStream) Events() <-chan autostandby.ConnectionEvent { return s.events } + +func (s *statusConnectionStream) Errors() <-chan error { return s.errs } + +func (s *statusConnectionStream) Close() error { return nil } + +func TestGetAutoStandbyStatusUnsupportedWithoutController(t *testing.T) { + t.Parallel() + + base := newTestService(t) + base.InstanceManager = &captureStatusManager{ + Manager: base.InstanceManager, + instance: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-1", + Name: "inst-1", + NetworkEnabled: true, + IP: "192.168.100.10", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + }, + State: instances.StateRunning, + }, + } + + resp, err := base.GetAutoStandbyStatus(ctx(), oapi.GetAutoStandbyStatusRequestObject{Id: "inst-1"}) + require.NoError(t, err) + + statusResp, ok := resp.(oapi.GetAutoStandbyStatus200JSONResponse) + require.True(t, ok) + assert.False(t, statusResp.Supported) + assert.Equal(t, oapi.AutoStandbyStatusStatusUnsupported, statusResp.Status) + assert.Equal(t, oapi.AutoStandbyStatusReasonUnsupportedPlatform, statusResp.Reason) +} + +func TestGetAutoStandbyStatusActive(t *testing.T) { + t.Parallel() + + inst := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-2", + Name: "inst-2", + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + }, + State: instances.StateRunning, + } + + now := time.Date(2026, 4, 6, 12, 0, 0, 0, time.UTC) + store := &statusStore{ + instances: []autostandby.Instance{{ + ID: "inst-2", + Name: "inst-2", + State: autostandby.StateRunning, + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &autostandby.Policy{Enabled: true, IdleTimeout: "5m"}, + }}, + } + source := &statusConnectionSource{connections: []autostandby.Connection{{ + OriginalSourceIP: mustStatusAddr("1.2.3.4"), + OriginalSourcePort: 51234, + OriginalDestinationIP: mustStatusAddr("192.168.100.20"), + OriginalDestinationPort: 8080, + TCPState: autostandby.TCPStateEstablished, + }}} + controller := autostandby.NewController(store, source, autostandby.ControllerOptions{ + Now: func() time.Time { return now }, + }) + require.NoError(t, controller.Run(withCanceledContext(t))) + + base := newTestService(t) + base.InstanceManager = &captureStatusManager{Manager: base.InstanceManager, instance: inst} + base.AutoStandbyController = controller + + resp, err := base.GetAutoStandbyStatus(ctx(), oapi.GetAutoStandbyStatusRequestObject{Id: "inst-2"}) + require.NoError(t, err) + + statusResp, ok := resp.(oapi.GetAutoStandbyStatus200JSONResponse) + require.True(t, ok) + assert.True(t, statusResp.Supported) + assert.Equal(t, oapi.AutoStandbyStatusStatusActive, statusResp.Status) + assert.Equal(t, 1, statusResp.ActiveInboundConnections) +} + +func withCanceledContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +func mustStatusAddr(raw string) netip.Addr { + return netip.MustParseAddr(raw) +} diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 05e95b3c..57f55c6d 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -79,7 +79,7 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } - apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, controller, vm_metricsManager) + apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, controller, autostandbyController, vm_metricsManager) mainApplication := &application{ Ctx: context, Logger: logger, diff --git a/lib/autostandby/README.md b/lib/autostandby/README.md index 8c745d62..feccf8a6 100644 --- a/lib/autostandby/README.md +++ b/lib/autostandby/README.md @@ -10,7 +10,7 @@ A VM is considered active when there is at least one tracked TCP flow where: - the original destination is the VM's private IP - the VM is the server/responding side of the connection -- the flow is still in an active TCP state +- the flow is currently tracked as live by conntrack That means: @@ -20,12 +20,23 @@ That means: ## Idle behavior +Hypeman seeds its controller from a conntrack snapshot on startup, then keeps state current with conntrack netlink events. + +- new inbound TCP flows are tracked from conntrack `NEW` events +- TCP teardown is treated as inactivity once conntrack reports a terminal state or the flow disappears +- connections that were already open when Hypeman started are reconciled against fresh conntrack snapshots until they drain, so restart-seeded traffic can still age out correctly + When the active inbound TCP connection count reaches zero, Hypeman starts an idle timer for that instance. - if a new inbound TCP connection appears before the timer expires, the timer is cleared - if the count stays at zero for the full `idle_timeout`, Hypeman places the VM into `Standby` -The timer is in-memory. After a Hypeman restart, idle timers begin from controller startup time instead of being reconstructed from the past. This avoids immediate standby caused only by restarting the control plane. +The idle timestamps are also persisted in instance metadata. + +- if Hypeman restarts and a startup conntrack snapshot shows current inbound connections, the instance is treated as active immediately and any old idle countdown is cleared +- if Hypeman restarts and the snapshot shows zero current inbound connections, Hypeman resumes the persisted idle countdown + +This keeps the restart behavior conservative about current traffic while still allowing long idle windows to carry across control-plane restarts. ## Exclusions @@ -43,4 +54,13 @@ This is intended for probes, internal callers, or ports that should not keep a V - IPv4 conntrack only - Wake-on-traffic is not part of this feature +## Status endpoint + +Hypeman exposes a diagnostic status endpoint for each instance that reports: + +- whether auto-standby is supported, configured, enabled, and currently eligible +- how many qualifying inbound TCP connections are currently keeping the VM awake +- the current idle timer timestamps and next planned standby time +- the current controller reason, such as active inbound traffic, countdown still running, or observer failure + Wake-on-traffic would require a separate host-owned listener or forwarding layer that can accept a connection while the VM is asleep, trigger restore, and then hand traffic through once the VM is running. diff --git a/lib/autostandby/conntrack_events_linux.go b/lib/autostandby/conntrack_events_linux.go new file mode 100644 index 00000000..6510cdf1 --- /dev/null +++ b/lib/autostandby/conntrack_events_linux.go @@ -0,0 +1,338 @@ +//go:build linux + +package autostandby + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "sync" + "syscall" + "time" + + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +type conntrackStream struct { + fd int + closeOnce sync.Once + events chan ConnectionEvent + errs chan error +} + +var errUnsupportedConntrackProtocol = errors.New("unsupported conntrack protocol") + +// OpenStream subscribes to IPv4 conntrack NEW, UPDATE, and DESTROY events. +func (s *ConntrackSource) OpenStream(ctx context.Context) (ConnectionStream, error) { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_NETFILTER) + if err != nil { + return nil, fmt.Errorf("open netfilter netlink socket: %w", err) + } + + closeFD := func() { _ = unix.Close(fd) } + if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + closeFD() + return nil, fmt.Errorf("bind netfilter netlink socket: %w", err) + } + if err := unix.SetsockoptInt(fd, unix.SOL_NETLINK, unix.NETLINK_ADD_MEMBERSHIP, unix.NFNLGRP_CONNTRACK_NEW); err != nil { + closeFD() + return nil, fmt.Errorf("subscribe conntrack new events: %w", err) + } + if err := unix.SetsockoptInt(fd, unix.SOL_NETLINK, unix.NETLINK_ADD_MEMBERSHIP, unix.NFNLGRP_CONNTRACK_UPDATE); err != nil { + closeFD() + return nil, fmt.Errorf("subscribe conntrack update events: %w", err) + } + if err := unix.SetsockoptInt(fd, unix.SOL_NETLINK, unix.NETLINK_ADD_MEMBERSHIP, unix.NFNLGRP_CONNTRACK_DESTROY); err != nil { + closeFD() + return nil, fmt.Errorf("subscribe conntrack destroy events: %w", err) + } + + stream := &conntrackStream{ + fd: fd, + events: make(chan ConnectionEvent, 256), + errs: make(chan error, 16), + } + go stream.run(ctx) + return stream, nil +} + +func (s *conntrackStream) Events() <-chan ConnectionEvent { return s.events } + +func (s *conntrackStream) Errors() <-chan error { return s.errs } + +func (s *conntrackStream) Close() error { + var err error + s.closeOnce.Do(func() { + err = unix.Close(s.fd) + }) + return err +} + +func (s *conntrackStream) run(ctx context.Context) { + defer close(s.events) + defer close(s.errs) + + go func() { + <-ctx.Done() + _ = s.Close() + }() + + buf := make([]byte, 1<<20) + for { + n, _, err := unix.Recvfrom(s.fd, buf, 0) + if err != nil { + if ctx.Err() != nil || err == unix.EBADF || err == syscall.ENOTCONN { + return + } + select { + case s.errs <- fmt.Errorf("recv conntrack events: %w", err): + default: + } + return + } + + msgs, err := syscall.ParseNetlinkMessage(buf[:n]) + if err != nil { + select { + case s.errs <- fmt.Errorf("parse conntrack netlink messages: %w", err): + default: + } + continue + } + + for _, msg := range msgs { + event, ok, err := connectionEventFromNetlinkMessage(msg) + if err != nil { + if errors.Is(err, errUnsupportedConntrackProtocol) { + continue + } + select { + case s.errs <- err: + default: + } + continue + } + if !ok { + continue + } + event.ObservedAt = time.Now().UTC() + select { + case s.events <- event: + case <-ctx.Done(): + return + } + } + } +} + +func connectionEventFromNetlinkMessage(msg syscall.NetlinkMessage) (ConnectionEvent, bool, error) { + switch msg.Header.Type { + case unix.NLMSG_NOOP, unix.NLMSG_DONE: + return ConnectionEvent{}, false, nil + case unix.NLMSG_ERROR: + if len(msg.Data) >= 4 { + errno := -int32(binary.LittleEndian.Uint32(msg.Data[:4])) + if errno != 0 { + return ConnectionEvent{}, false, unix.Errno(errno) + } + } + return ConnectionEvent{}, false, nil + } + + if len(msg.Data) < nl.SizeofNfgenmsg { + return ConnectionEvent{}, false, nil + } + if msg.Data[0] != unix.AF_INET { + return ConnectionEvent{}, false, nil + } + + var eventType ConnectionEventType + switch int(msg.Header.Type & 0x00ff) { + case nl.IPCTNL_MSG_CT_NEW: + eventType = ConnectionEventNew + case nl.IPCTNL_MSG_CT_DELETE: + eventType = ConnectionEventDestroy + default: + return ConnectionEvent{}, false, nil + } + + conn, ok, err := connectionFromRawData(msg.Data) + if err != nil || !ok { + return ConnectionEvent{}, false, err + } + return ConnectionEvent{Type: eventType, Connection: conn}, true, nil +} + +func connectionFromRawData(data []byte) (Connection, bool, error) { + if len(data) < nl.SizeofNfgenmsg { + return Connection{}, false, nil + } + + reader := bytes.NewReader(data[nl.SizeofNfgenmsg:]) + conn := Connection{} + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return Connection{}, false, err + } + switch attr.typ { + case nl.CTA_TUPLE_ORIG: + if err := parseTuple(attr.payload, &conn); err != nil { + return Connection{}, false, err + } + case nl.CTA_PROTOINFO: + if err := parseProtoInfo(attr.payload, &conn); err != nil { + return Connection{}, false, err + } + } + } + + if !conn.OriginalSourceIP.IsValid() || !conn.OriginalDestinationIP.IsValid() { + return Connection{}, false, nil + } + return conn, true, nil +} + +type nfAttr struct { + typ uint16 + nested bool + payload []byte +} + +func readNfAttr(reader *bytes.Reader) (nfAttr, error) { + var rawLen uint16 + var rawType uint16 + if err := binary.Read(reader, nl.NativeEndian(), &rawLen); err != nil { + return nfAttr{}, err + } + if err := binary.Read(reader, nl.NativeEndian(), &rawType); err != nil { + return nfAttr{}, err + } + if rawLen < nl.SizeofNfattr { + return nfAttr{}, fmt.Errorf("invalid netfilter attribute length %d", rawLen) + } + + payloadLen := int(rawLen) - nl.SizeofNfattr + payload := make([]byte, payloadLen) + if _, err := reader.Read(payload); err != nil { + return nfAttr{}, err + } + + padding := nlaAlignedLen(rawLen) - rawLen + if padding > 0 { + if _, err := reader.Seek(int64(padding), 1); err != nil { + return nfAttr{}, err + } + } + + return nfAttr{ + typ: rawType & nl.NLA_TYPE_MASK, + nested: rawType&nl.NLA_F_NESTED == nl.NLA_F_NESTED, + payload: payload, + }, nil +} + +func parseTuple(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + switch attr.typ { + case nl.CTA_TUPLE_IP: + if err := parseTupleIP(attr.payload, conn); err != nil { + return err + } + case nl.CTA_TUPLE_PROTO: + if err := parseTupleProto(attr.payload, conn); err != nil { + return err + } + } + } + return nil +} + +func parseTupleIP(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + switch attr.typ { + case nl.CTA_IP_V4_SRC: + addr, ok := addrFromIP(attr.payload) + if ok { + conn.OriginalSourceIP = addr + } + case nl.CTA_IP_V4_DST: + addr, ok := addrFromIP(attr.payload) + if ok { + conn.OriginalDestinationIP = addr + } + } + } + return nil +} + +func parseTupleProto(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + var protocol uint8 + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + switch attr.typ { + case nl.CTA_PROTO_NUM: + if len(attr.payload) > 0 { + protocol = attr.payload[0] + } + case nl.CTA_PROTO_SRC_PORT: + if len(attr.payload) >= 2 { + conn.OriginalSourcePort = binary.BigEndian.Uint16(attr.payload[:2]) + } + case nl.CTA_PROTO_DST_PORT: + if len(attr.payload) >= 2 { + conn.OriginalDestinationPort = binary.BigEndian.Uint16(attr.payload[:2]) + } + } + } + if protocol != unix.IPPROTO_TCP { + return fmt.Errorf("%w %d", errUnsupportedConntrackProtocol, protocol) + } + return nil +} + +func parseProtoInfo(payload []byte, conn *Connection) error { + reader := bytes.NewReader(payload) + for reader.Len() > 0 { + attr, err := readNfAttr(reader) + if err != nil { + return err + } + if attr.typ != nl.CTA_PROTOINFO_TCP { + continue + } + + tcpReader := bytes.NewReader(attr.payload) + for tcpReader.Len() > 0 { + tcpAttr, err := readNfAttr(tcpReader) + if err != nil { + return err + } + if tcpAttr.typ == nl.CTA_PROTOINFO_TCP_STATE && len(tcpAttr.payload) > 0 { + conn.TCPState = TCPState(tcpAttr.payload[0]) + } + } + } + return nil +} + +func nlaAlignedLen(length uint16) uint16 { + return (length + nl.NLA_ALIGNTO - 1) & ^(nl.NLA_ALIGNTO - 1) +} diff --git a/lib/autostandby/conntrack_events_linux_test.go b/lib/autostandby/conntrack_events_linux_test.go new file mode 100644 index 00000000..65604e50 --- /dev/null +++ b/lib/autostandby/conntrack_events_linux_test.go @@ -0,0 +1,47 @@ +//go:build linux + +package autostandby + +import ( + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConnectionEventFromNetlinkMessageParsesIPv4TCPEvent(t *testing.T) { + t.Parallel() + + msg := syscall.NetlinkMessage{ + Header: syscall.NlMsghdr{Type: 0x100 | uint16(0)}, + Data: []byte{ + 2, 0, 0, 0, + 52, 0, 1, 128, + 20, 0, 1, 128, + 8, 0, 1, 0, 192, 168, 0, 10, + 8, 0, 2, 0, 192, 168, 77, 73, + 28, 0, 2, 128, + 5, 0, 1, 0, 6, 0, 0, 0, + 6, 0, 2, 0, 166, 129, 0, 0, + 6, 0, 3, 0, 13, 5, 0, 0, + 48, 0, 4, 128, + 44, 0, 1, 128, + 5, 0, 1, 0, 8, 0, 0, 0, + 5, 0, 2, 0, 0, 0, 0, 0, + 5, 0, 3, 0, 0, 0, 0, 0, + 6, 0, 4, 0, 39, 0, 0, 0, + 6, 0, 5, 0, 32, 0, 0, 0, + }, + } + + event, ok, err := connectionEventFromNetlinkMessage(msg) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, ConnectionEventNew, event.Type) + assert.Equal(t, mustAddr("192.168.0.10"), event.Connection.OriginalSourceIP) + assert.Equal(t, uint16(42625), event.Connection.OriginalSourcePort) + assert.Equal(t, mustAddr("192.168.77.73"), event.Connection.OriginalDestinationIP) + assert.Equal(t, uint16(3333), event.Connection.OriginalDestinationPort) + assert.Equal(t, TCPStateClose, event.Connection.TCPState) +} diff --git a/lib/autostandby/conntrack_linux.go b/lib/autostandby/conntrack_linux.go index 28cc9999..d308d7b3 100644 --- a/lib/autostandby/conntrack_linux.go +++ b/lib/autostandby/conntrack_linux.go @@ -64,6 +64,7 @@ func connectionFromFlow(flow *netlink.ConntrackFlow) (Connection, bool) { return Connection{ OriginalSourceIP: srcIP, + OriginalSourcePort: flow.Forward.SrcPort, OriginalDestinationIP: dstIP, OriginalDestinationPort: flow.Forward.DstPort, TCPState: TCPState(tcpInfo.State), diff --git a/lib/autostandby/conntrack_linux_test.go b/lib/autostandby/conntrack_linux_test.go index 44ba938b..aecc2936 100644 --- a/lib/autostandby/conntrack_linux_test.go +++ b/lib/autostandby/conntrack_linux_test.go @@ -19,6 +19,7 @@ func TestConnectionFromFlowNormalizesTCPConntrackEntry(t *testing.T) { Forward: netlink.IPTuple{ Protocol: unix.IPPROTO_TCP, SrcIP: net.ParseIP("1.2.3.4").To4(), + SrcPort: 12345, DstIP: net.ParseIP("192.168.100.10").To4(), DstPort: 8080, }, @@ -27,6 +28,7 @@ func TestConnectionFromFlowNormalizesTCPConntrackEntry(t *testing.T) { require.True(t, ok) assert.Equal(t, mustAddr("1.2.3.4"), conn.OriginalSourceIP) + assert.Equal(t, uint16(12345), conn.OriginalSourcePort) assert.Equal(t, mustAddr("192.168.100.10"), conn.OriginalDestinationIP) assert.Equal(t, uint16(8080), conn.OriginalDestinationPort) assert.Equal(t, TCPStateEstablished, conn.TCPState) diff --git a/lib/autostandby/conntrack_unsupported.go b/lib/autostandby/conntrack_unsupported.go index 8f929d7c..2cdbd38d 100644 --- a/lib/autostandby/conntrack_unsupported.go +++ b/lib/autostandby/conntrack_unsupported.go @@ -19,3 +19,8 @@ func NewConntrackSource() *ConntrackSource { func (*ConntrackSource) ListConnections(context.Context) ([]Connection, error) { return nil, fmt.Errorf("conntrack-backed auto-standby is only supported on Linux") } + +// OpenStream reports that conntrack-backed auto-standby is unsupported. +func (*ConntrackSource) OpenStream(context.Context) (ConnectionStream, error) { + return nil, fmt.Errorf("conntrack-backed auto-standby is only supported on Linux") +} diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 14c9499c..3294fb59 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -3,135 +3,928 @@ package autostandby import ( "context" "errors" + "fmt" "log/slog" + "net/netip" + "sync" "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +const ( + trackingModeConntrackEventsV4TCP = "conntrack_events_v4_tcp" + defaultReconnectDelay = 2 * time.Second + defaultReconcileDelay = 2 * time.Second +) + +// InstanceEventAction identifies an instance lifecycle change relevant to auto-standby. +type InstanceEventAction string + +const ( + InstanceEventCreate InstanceEventAction = "create" + InstanceEventUpdate InstanceEventAction = "update" + InstanceEventStart InstanceEventAction = "start" + InstanceEventStop InstanceEventAction = "stop" + InstanceEventStandby InstanceEventAction = "standby" + InstanceEventRestore InstanceEventAction = "restore" + InstanceEventDelete InstanceEventAction = "delete" + InstanceEventFork InstanceEventAction = "fork" ) -const defaultPollInterval = 5 * time.Second +// InstanceEvent carries an instance lifecycle update into the controller. +type InstanceEvent struct { + Action InstanceEventAction + InstanceID string + Instance *Instance +} -// InstanceStore supplies the controller with instance state and standby actions. +// InstanceStore supplies the controller with instance state, lifecycle events, +// runtime persistence, and standby actions. type InstanceStore interface { ListInstances(ctx context.Context) ([]Instance, error) StandbyInstance(ctx context.Context, id string) error + SetRuntime(ctx context.Context, id string, runtime *Runtime) error + SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) +} + +type ConnectionEventType string + +const ( + ConnectionEventNew ConnectionEventType = "new" + ConnectionEventDestroy ConnectionEventType = "destroy" +) + +// ConnectionEvent is a single conntrack event delivered from the host observer. +type ConnectionEvent struct { + Type ConnectionEventType + Connection Connection + ObservedAt time.Time } -// ConnectionSource lists current TCP flows that may keep an instance awake. +// ConnectionStream is a live conntrack event stream. +type ConnectionStream interface { + Events() <-chan ConnectionEvent + Errors() <-chan error + Close() error +} + +// ConnectionSource provides startup snapshots and live conntrack events. type ConnectionSource interface { ListConnections(ctx context.Context) ([]Connection, error) + OpenStream(ctx context.Context) (ConnectionStream, error) +} + +// ControllerOptions configures logging, timing, and observability. +type ControllerOptions struct { + Log *slog.Logger + Meter metric.Meter + Tracer trace.Tracer + Now func() time.Time + ReconnectDelay time.Duration + ReconcileDelay time.Duration +} + +type controllerState struct { + instance Instance + compiledPolicy *compiledPolicy + activeInbound map[ConnectionKey]struct{} + idleTimeout time.Duration + idleSince *time.Time + lastInboundAt *time.Time + nextStandbyAt *time.Time + timer *time.Timer + reconcileTimer *time.Timer + standbyRequested bool } // Controller decides when eligible instances should transition to standby. type Controller struct { - store InstanceStore - source ConnectionSource - log *slog.Logger - now func() time.Time - pollInterval time.Duration - idleSince map[string]time.Time + store InstanceStore + source ConnectionSource + log *slog.Logger + now func() time.Time + tracer trace.Tracer + metrics *Metrics + + reconnectDelay time.Duration + reconcileDelay time.Duration + timerFired chan string + reconcileFired chan string + streamReady chan ConnectionStream + + mu sync.RWMutex + states map[string]*controllerState + observerConnected bool + lastObserverErr error } -// NewController creates a new auto-standby controller. -func NewController(store InstanceStore, source ConnectionSource, log *slog.Logger, pollInterval time.Duration) *Controller { +// NewController creates a new event-driven auto-standby controller. +func NewController(store InstanceStore, source ConnectionSource, opts ControllerOptions) *Controller { + log := opts.Log if log == nil { log = slog.Default() } - if pollInterval <= 0 { - pollInterval = defaultPollInterval + now := opts.Now + if now == nil { + now = time.Now } - return &Controller{ - store: store, - source: source, - log: log, - now: time.Now, - pollInterval: pollInterval, - idleSince: make(map[string]time.Time), + reconnectDelay := opts.ReconnectDelay + if reconnectDelay <= 0 { + reconnectDelay = defaultReconnectDelay } + reconcileDelay := opts.ReconcileDelay + if reconcileDelay <= 0 { + reconcileDelay = defaultReconcileDelay + } + + c := &Controller{ + store: store, + source: source, + log: log, + now: now, + tracer: opts.Tracer, + reconnectDelay: reconnectDelay, + reconcileDelay: reconcileDelay, + timerFired: make(chan string, 128), + reconcileFired: make(chan string, 128), + streamReady: make(chan ConnectionStream, 4), + states: make(map[string]*controllerState), + } + c.metrics = newMetrics(opts.Meter, opts.Tracer, c) + return c } -// Run starts the controller loop and blocks until the context is canceled. +// Run starts the controller and blocks until ctx is cancelled. func (c *Controller) Run(ctx context.Context) error { - c.log.Info("auto-standby controller started", "poll_interval", c.pollInterval) - if err := c.Poll(ctx); err != nil { - c.log.Warn("auto-standby poll failed", "error", err) + log := c.log.With("tracking_mode", trackingModeConntrackEventsV4TCP) + log.Info("auto-standby controller started") + + var stream ConnectionStream + if c.source != nil { + initialStream, err := c.source.OpenStream(ctx) + if err != nil { + c.setObserverError(err) + c.recordObserverError("connect") + log.Warn("auto-standby conntrack subscription failed", "error", err) + go c.reconnectStream(ctx) + } else { + stream = initialStream + c.setObserverConnected(true) + } } - ticker := time.NewTicker(c.pollInterval) - defer ticker.Stop() + if err := c.startupResync(ctx); err != nil { + return err + } + + instanceEvents, unsubscribe, err := c.store.SubscribeInstanceEvents() + if err != nil { + return err + } + defer unsubscribe() + defer c.stopAllTimers() + if stream != nil { + defer stream.Close() + } for { + var streamEvents <-chan ConnectionEvent + var streamErrors <-chan error + if stream != nil { + streamEvents = stream.Events() + streamErrors = stream.Errors() + } + select { case <-ctx.Done(): return nil - case <-ticker.C: - if err := c.Poll(ctx); err != nil { - c.log.Warn("auto-standby poll failed", "error", err) + case replacement := <-c.streamReady: + if stream != nil { + _ = stream.Close() + } + stream = replacement + c.setObserverConnected(true) + log.Info("auto-standby conntrack subscription restored") + case err := <-streamErrors: + if err == nil { + continue + } + c.setObserverError(err) + c.recordObserverError("stream") + log.Warn("auto-standby conntrack subscription failed", "error", err) + if stream != nil { + _ = stream.Close() + stream = nil + } + go c.reconnectStream(ctx) + case event, ok := <-streamEvents: + if !ok { + if stream != nil { + _ = stream.Close() + stream = nil + } + c.setObserverError(errors.New("conntrack stream closed")) + c.recordObserverError("stream_closed") + go c.reconnectStream(ctx) + continue + } + c.handleConnectionEvent(ctx, event) + case event, ok := <-instanceEvents: + if !ok { + return nil } + if err := c.handleInstanceEvent(ctx, event); err != nil { + c.recordObserverError("instance_event") + log.Warn("auto-standby instance event handling failed", "action", event.Action, "instance_id", event.InstanceID, "error", err) + } + case id := <-c.timerFired: + c.handleStandbyTimer(ctx, id) + case id := <-c.reconcileFired: + c.handleActiveReconcile(ctx, id) } } } -// Poll runs a single reconciliation pass. -func (c *Controller) Poll(ctx context.Context) error { +// Describe returns the current diagnostic view for an instance. +func (c *Controller) Describe(inst Instance) StatusSnapshot { + snapshot := StatusSnapshot{ + Supported: c != nil, + Configured: inst.AutoStandby != nil, + Enabled: inst.AutoStandby != nil && inst.AutoStandby.Enabled, + TrackingMode: trackingModeConntrackEventsV4TCP, + } + if c == nil { + snapshot.Status = StatusUnsupported + snapshot.Reason = ReasonUnsupportedPlatform + return snapshot + } + if inst.AutoStandby != nil { + snapshot.IdleTimeout = inst.AutoStandby.IdleTimeout + } + + if inst.AutoStandby == nil { + snapshot.Status = StatusDisabled + snapshot.Reason = ReasonPolicyMissing + return snapshot + } + if !inst.AutoStandby.Enabled { + snapshot.Status = StatusDisabled + snapshot.Reason = ReasonPolicyDisabled + return snapshot + } + if inst.State != StateRunning { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonInstanceNotRunning + return snapshot + } + if !inst.NetworkEnabled { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonNetworkDisabled + return snapshot + } + if inst.IP == "" { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonMissingIP + return snapshot + } + if inst.HasVGPU { + snapshot.Status = StatusIneligible + snapshot.Reason = ReasonHasVGPU + return snapshot + } + + snapshot.Eligible = true + + c.mu.RLock() + state := c.states[inst.ID] + observerConnected := c.observerConnected + lastObserverErr := c.lastObserverErr + c.mu.RUnlock() + + if state != nil { + snapshot.ActiveInboundCount = len(state.activeInbound) + snapshot.IdleSince = cloneTimePtr(state.idleSince) + snapshot.LastInboundActivityAt = cloneTimePtr(state.lastInboundAt) + snapshot.NextStandbyAt = cloneTimePtr(state.nextStandbyAt) + if state.nextStandbyAt != nil { + remaining := state.nextStandbyAt.Sub(c.now().UTC()) + if remaining < 0 { + remaining = 0 + } + snapshot.CountdownRemaining = &remaining + } + if state.standbyRequested { + snapshot.Status = StatusStandbyRequested + snapshot.Reason = ReasonReadyForStandby + return snapshot + } + } + + if !observerConnected && lastObserverErr != nil { + snapshot.Status = StatusError + snapshot.Reason = ReasonObserverError + return snapshot + } + if state != nil && len(state.activeInbound) > 0 { + snapshot.Status = StatusActive + snapshot.Reason = ReasonActiveInbound + return snapshot + } + if state != nil && state.nextStandbyAt != nil { + if state.nextStandbyAt.After(c.now().UTC()) { + snapshot.Status = StatusIdleCountdown + snapshot.Reason = ReasonIdleTimeoutNotElapsed + return snapshot + } + snapshot.Status = StatusReadyForStandby + snapshot.Reason = ReasonReadyForStandby + return snapshot + } + + snapshot.Status = StatusIdleCountdown + snapshot.Reason = ReasonIdleTimeoutNotElapsed + return snapshot +} + +func (c *Controller) startupResync(ctx context.Context) error { + start := c.now() + ctx, span := c.startSpan(ctx, "AutoStandbyStartupResync") + defer func() { + if span != nil { + span.End() + } + }() + instances, err := c.store.ListInstances(ctx) if err != nil { + c.recordStartupResync(start, "error") + recordSpanError(span, err) return err } conns, err := c.source.ListConnections(ctx) if err != nil { + c.recordStartupResync(start, "error") + recordSpanError(span, err) return err } now := c.now().UTC() - seen := make(map[string]struct{}, len(instances)) - var reconcileErrs []error - + c.log.Info("auto-standby startup resync seeded state", "instance_count", len(instances), "current_connection_count", len(conns)) for _, inst := range instances { - seen[inst.ID] = struct{}{} + if err := c.seedInstanceState(ctx, inst, conns, now); err != nil { + c.log.Warn("auto-standby startup resync failed for instance", "instance_id", inst.ID, "instance_name", inst.Name, "error", err) + } + } - if !eligible(inst) { - delete(c.idleSince, inst.ID) - continue + c.recordStartupResync(start, "success") + return nil +} + +func (c *Controller) seedInstanceState(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.refreshInstanceLocked(ctx, inst, conns, now) +} + +func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEvent) error { + if event.Action == InstanceEventDelete { + c.mu.Lock() + defer c.mu.Unlock() + c.removeStateLocked(event.InstanceID) + return nil + } + if event.Instance == nil { + return nil + } + + conns, err := c.source.ListConnections(ctx) + if err != nil { + return err + } + + c.mu.Lock() + defer c.mu.Unlock() + return c.refreshInstanceLocked(ctx, *event.Instance, conns, c.now().UTC()) +} + +func (c *Controller) refreshInstanceLocked(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { + state := c.ensureStateLocked(inst.ID) + state.instance = cloneInstance(inst) + state.standbyRequested = false + + if !eligible(inst) { + c.clearStateLocked(state) + if inst.Runtime != nil || state.idleSince != nil || state.lastInboundAt != nil { + return c.persistRuntime(ctx, inst.ID, nil) } + return nil + } - count, idleTimeout, err := ActiveInboundCount(inst, conns) - if err != nil { - delete(c.idleSince, inst.ID) - reconcileErrs = append(reconcileErrs, err) - continue + compiled, err := compilePolicy(inst.AutoStandby) + if err != nil { + return err + } + state.compiledPolicy = compiled + state.idleTimeout = compiled.idleTimeout + + activeSet, err := matchingConnections(inst, compiled, conns) + if err != nil { + return err + } + state.activeInbound = activeSet + + runtime := cloneRuntime(inst.Runtime) + if len(activeSet) > 0 { + state.idleSince = nil + if runtime != nil && runtime.LastInboundActivityAt != nil { + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + } else { + state.lastInboundAt = &now } + c.cancelTimerLocked(state) + c.armReconcileLocked(inst.ID, state) + return c.persistRuntime(ctx, inst.ID, &Runtime{ + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }) + } - if count > 0 { - delete(c.idleSince, inst.ID) - continue + if runtime != nil && runtime.IdleSince != nil { + state.idleSince = cloneTimePtr(runtime.IdleSince) + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + } else { + state.idleSince = &now + if runtime != nil { + state.lastInboundAt = cloneTimePtr(runtime.LastInboundActivityAt) + } else { + state.lastInboundAt = nil + } + runtime = &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + } + if err := c.persistRuntime(ctx, inst.ID, runtime); err != nil { + return err } + } + c.armTimerLocked(inst.ID, state, now) + return nil +} - start, ok := c.idleSince[inst.ID] - if !ok { - c.idleSince[inst.ID] = now - continue +func (c *Controller) handleConnectionEvent(ctx context.Context, event ConnectionEvent) { + ctx, span := c.startSpan(ctx, "AutoStandbyHandleConntrackEvent", + attribute.String("event", string(event.Type)), + ) + defer func() { + if span != nil { + span.End() } - if now.Sub(start) < idleTimeout { + }() + + if event.ObservedAt.IsZero() { + event.ObservedAt = c.now().UTC() + } + c.recordConntrackEvent(string(event.Type), "received") + + c.mu.Lock() + defer c.mu.Unlock() + + for id, state := range c.states { + if state.compiledPolicy == nil { continue } + key := connectionKey(event.Connection) + matches := matchesInboundConnectionForEvent(state.instance, state.compiledPolicy, event.Connection) + switch event.Type { + case ConnectionEventNew: + if !matches { + continue + } + if !event.Connection.TCPState.Active() { + if _, ok := state.activeInbound[key]; !ok { + continue + } + delete(state.activeInbound, key) + if len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + continue + } + idleSince := event.ObservedAt.UTC() + state.idleSince = &idleSince + state.standbyRequested = false + c.cancelReconcileLocked(state) + c.armTimerLocked(id, state, idleSince) + if err := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime when idle countdown started", "instance_id", id, "error", err) + } + c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) + continue + } + if state.activeInbound == nil { + state.activeInbound = make(map[ConnectionKey]struct{}) + } + state.activeInbound[key] = struct{}{} + state.idleSince = nil + state.lastInboundAt = cloneTimePtr(&event.ObservedAt) + state.standbyRequested = false + c.cancelTimerLocked(state) + c.armReconcileLocked(id, state) + if err := c.persistRuntime(ctx, id, &Runtime{ + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after inbound activity", "instance_id", id, "error", err) + } + c.log.Info("auto-standby inbound activity observed", "instance_id", id, "active_inbound_connections", len(state.activeInbound)) + case ConnectionEventDestroy: + if _, ok := state.activeInbound[key]; !ok { + continue + } + delete(state.activeInbound, key) + if len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + continue + } + idleSince := event.ObservedAt.UTC() + state.idleSince = &idleSince + state.standbyRequested = false + c.cancelReconcileLocked(state) + c.armTimerLocked(id, state, idleSince) + if err := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime when idle countdown started", "instance_id", id, "error", err) + } + c.log.Info("auto-standby idle countdown started", "instance_id", id, "idle_timeout", state.idleTimeout) + } + } +} - if err := c.store.StandbyInstance(ctx, inst.ID); err != nil { - c.log.Warn("auto-standby standby attempt failed", "instance_id", inst.ID, "instance_name", inst.Name, "error", err) - } else { - c.log.Info("instance entered standby due to inbound inactivity", "instance_id", inst.ID, "instance_name", inst.Name, "idle_timeout", idleTimeout) +func (c *Controller) handleStandbyTimer(ctx context.Context, id string) { + ctx, span := c.startSpan(ctx, "AutoStandbyStandbyAttempt", + attribute.String("instance_id", id), + ) + defer func() { + if span != nil { + span.End() } + }() - // Reset the timer after every attempt to avoid retrying every poll interval - // when the standby transition fails for a transient reason. - c.idleSince[inst.ID] = now + c.mu.Lock() + state := c.states[id] + if state == nil || state.compiledPolicy == nil || len(state.activeInbound) > 0 { + c.mu.Unlock() + return } + state.timer = nil + state.nextStandbyAt = nil + state.standbyRequested = true + instanceName := state.instance.Name + idleTimeout := state.idleTimeout + c.mu.Unlock() + + c.log.Info("auto-standby standby timer fired", "instance_id", id, "instance_name", instanceName) - for id := range c.idleSince { - if _, ok := seen[id]; !ok { - delete(c.idleSince, id) + if err := c.store.StandbyInstance(ctx, id); err != nil { + recordSpanError(span, err) + c.recordStandbyAttempt("error") + c.recordControllerError("standby") + c.log.Warn("auto-standby standby attempt failed", "instance_id", id, "instance_name", instanceName, "error", err) + + c.mu.Lock() + defer c.mu.Unlock() + if state := c.states[id]; state != nil { + state.standbyRequested = false + idleSince := c.now().UTC() + state.idleSince = &idleSince + c.armTimerLocked(id, state, idleSince) + if persistErr := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); persistErr != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after standby failure", "instance_id", id, "error", persistErr) + } + } + return + } + + c.recordStandbyAttempt("success") + c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) + + c.mu.Lock() + defer c.mu.Unlock() + if state := c.states[id]; state != nil { + c.clearStateLocked(state) + if err := c.persistRuntime(ctx, id, nil); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) + } + } +} + +func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { + conns, err := c.source.ListConnections(ctx) + if err != nil { + c.recordControllerError("reconcile") + c.log.Warn("auto-standby active connection reconcile failed", "instance_id", id, "error", err) + + c.mu.Lock() + defer c.mu.Unlock() + if state := c.states[id]; state != nil && len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + } + return + } + + now := c.now().UTC() + + c.mu.Lock() + defer c.mu.Unlock() + + state := c.states[id] + if state == nil || state.compiledPolicy == nil { + return + } + + activeSet, err := matchingConnections(state.instance, state.compiledPolicy, conns) + if err != nil { + c.recordControllerError("reconcile") + c.log.Warn("auto-standby active connection reconcile failed to classify connections", "instance_id", id, "error", err) + if len(state.activeInbound) > 0 { + c.armReconcileLocked(id, state) + } + return + } + + state.activeInbound = activeSet + if len(activeSet) > 0 { + c.armReconcileLocked(id, state) + return + } + + state.idleSince = &now + state.standbyRequested = false + c.cancelReconcileLocked(state) + c.armTimerLocked(id, state, now) + if err := c.persistRuntime(ctx, id, &Runtime{ + IdleSince: cloneTimePtr(state.idleSince), + LastInboundActivityAt: cloneTimePtr(state.lastInboundAt), + }); err != nil { + c.recordControllerError("persist_runtime") + c.log.Warn("auto-standby failed to persist runtime after active connection reconcile drained", "instance_id", id, "error", err) + } + c.log.Info("auto-standby idle countdown started after active connection reconcile", "instance_id", id, "idle_timeout", state.idleTimeout) +} + +func (c *Controller) reconnectStream(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + default: + } + + stream, err := c.source.OpenStream(ctx) + if err == nil { + select { + case c.streamReady <- stream: + case <-ctx.Done(): + _ = stream.Close() + } + return + } + + c.setObserverError(err) + c.recordObserverError("reconnect") + c.log.Warn("auto-standby conntrack subscription reconnect failed", "error", err) + + timer := time.NewTimer(c.reconnectDelay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } +} + +func (c *Controller) ensureStateLocked(id string) *controllerState { + state, ok := c.states[id] + if !ok { + state = &controllerState{} + c.states[id] = state + } + return state +} + +func (c *Controller) removeStateLocked(id string) { + state := c.states[id] + if state != nil { + c.cancelTimerLocked(state) + c.cancelReconcileLocked(state) + } + delete(c.states, id) +} + +func (c *Controller) clearStateLocked(state *controllerState) { + state.compiledPolicy = nil + state.activeInbound = nil + state.idleTimeout = 0 + state.idleSince = nil + state.lastInboundAt = nil + state.nextStandbyAt = nil + state.standbyRequested = false + c.cancelTimerLocked(state) + c.cancelReconcileLocked(state) +} + +func (c *Controller) armTimerLocked(id string, state *controllerState, now time.Time) { + if state.idleSince == nil || state.idleTimeout <= 0 { + c.cancelTimerLocked(state) + return + } + + when := state.idleSince.Add(state.idleTimeout) + delay := when.Sub(now) + if delay < 0 { + delay = 0 + } + + c.cancelTimerLocked(state) + state.nextStandbyAt = &when + c.log.Debug("auto-standby standby timer armed", "instance_id", id, "next_standby_at", when, "idle_timeout", state.idleTimeout) + state.timer = time.AfterFunc(delay, func() { + select { + case c.timerFired <- id: + default: + } + }) +} + +func (c *Controller) cancelTimerLocked(state *controllerState) { + if state.timer != nil { + state.timer.Stop() + c.log.Debug("auto-standby standby timer cancelled", "instance_id", state.instance.ID) + state.timer = nil + } + state.nextStandbyAt = nil +} + +func (c *Controller) armReconcileLocked(id string, state *controllerState) { + if len(state.activeInbound) == 0 || c.reconcileDelay <= 0 { + c.cancelReconcileLocked(state) + return + } + + c.cancelReconcileLocked(state) + state.reconcileTimer = time.AfterFunc(c.reconcileDelay, func() { + select { + case c.reconcileFired <- id: + default: + } + }) +} + +func (c *Controller) cancelReconcileLocked(state *controllerState) { + if state.reconcileTimer != nil { + state.reconcileTimer.Stop() + state.reconcileTimer = nil + } +} + +func (c *Controller) stopAllTimers() { + c.mu.Lock() + defer c.mu.Unlock() + for _, state := range c.states { + c.cancelTimerLocked(state) + c.cancelReconcileLocked(state) + } +} + +func (c *Controller) persistRuntime(ctx context.Context, id string, runtime *Runtime) error { + return c.store.SetRuntime(ctx, id, runtime) +} + +func (c *Controller) setObserverConnected(connected bool) { + c.mu.Lock() + defer c.mu.Unlock() + c.observerConnected = connected + if connected { + c.lastObserverErr = nil + } +} + +func (c *Controller) setObserverError(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.observerConnected = false + c.lastObserverErr = err +} + +func matchingConnections(inst Instance, compiled *compiledPolicy, conns []Connection) (map[ConnectionKey]struct{}, error) { + instanceIP, err := netip.ParseAddr(inst.IP) + if err != nil { + return nil, fmt.Errorf("parse instance IP %q: %w", inst.IP, err) + } + + out := make(map[ConnectionKey]struct{}) + for _, conn := range conns { + if matchesInboundConnection(instanceIP, compiled, conn) { + out[connectionKey(conn)] = struct{}{} + } + } + return out, nil +} + +func matchesInboundConnectionForEvent(inst Instance, policy *compiledPolicy, conn Connection) bool { + instanceIP, err := netip.ParseAddr(inst.IP) + if err != nil { + return false + } + if !conn.OriginalDestinationIP.IsValid() || conn.OriginalDestinationIP != instanceIP { + return false + } + if _, ignored := policy.ignorePorts[conn.OriginalDestinationPort]; ignored { + return false + } + if !conn.OriginalSourceIP.IsValid() { + return false + } + for _, prefix := range policy.ignoreSourceCIDRs { + if prefix.Contains(conn.OriginalSourceIP) { + return false } } + return true +} + +type ConnectionKey struct { + OriginalSourceIP netip.Addr + OriginalSourcePort uint16 + OriginalDestinationIP netip.Addr + OriginalDestinationPort uint16 +} + +func connectionKey(conn Connection) ConnectionKey { + return ConnectionKey{ + OriginalSourceIP: conn.OriginalSourceIP, + OriginalSourcePort: conn.OriginalSourcePort, + OriginalDestinationIP: conn.OriginalDestinationIP, + OriginalDestinationPort: conn.OriginalDestinationPort, + } +} + +func cloneRuntime(runtime *Runtime) *Runtime { + if runtime == nil { + return nil + } + return &Runtime{ + IdleSince: cloneTimePtr(runtime.IdleSince), + LastInboundActivityAt: cloneTimePtr(runtime.LastInboundActivityAt), + } +} - return errors.Join(reconcileErrs...) +func cloneInstance(inst Instance) Instance { + cloned := inst + cloned.AutoStandby = clonePolicy(inst.AutoStandby) + cloned.Runtime = cloneRuntime(inst.Runtime) + return cloned +} + +func clonePolicy(policy *Policy) *Policy { + if policy == nil { + return nil + } + out := &Policy{ + Enabled: policy.Enabled, + IdleTimeout: policy.IdleTimeout, + } + if len(policy.IgnoreSourceCIDRs) > 0 { + out.IgnoreSourceCIDRs = append([]string(nil), policy.IgnoreSourceCIDRs...) + } + if len(policy.IgnoreDestinationPorts) > 0 { + out.IgnoreDestinationPorts = append([]uint16(nil), policy.IgnoreDestinationPorts...) + } + return out +} + +func cloneTimePtr(t *time.Time) *time.Time { + if t == nil { + return nil + } + copied := t.UTC() + return &copied } func eligible(inst Instance) bool { @@ -143,3 +936,18 @@ func eligible(inst Instance) bool { } return inst.AutoStandby != nil && inst.AutoStandby.Enabled } + +func (c *Controller) startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + if c.tracer == nil { + return ctx, nil + } + return c.tracer.Start(ctx, name, trace.WithAttributes(attrs...)) +} + +func recordSpanError(span trace.Span, err error) { + if span == nil || err == nil { + return + } + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index 07c70273..cf763cde 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -2,6 +2,7 @@ package autostandby import ( "context" + "errors" "net/netip" "testing" "time" @@ -11,13 +12,27 @@ import ( ) type fakeInstanceStore struct { - instances []Instance - standbyIDs []string - standbyErr error + instances []Instance + standbyIDs []string + persistedRuntime map[string]*Runtime + events chan InstanceEvent + standbyErr error +} + +func newFakeInstanceStore(instances []Instance) *fakeInstanceStore { + return &fakeInstanceStore{ + instances: append([]Instance(nil), instances...), + persistedRuntime: make(map[string]*Runtime), + events: make(chan InstanceEvent, 16), + } } func (f *fakeInstanceStore) ListInstances(context.Context) ([]Instance, error) { - return append([]Instance(nil), f.instances...), nil + out := make([]Instance, 0, len(f.instances)) + for _, inst := range f.instances { + out = append(out, cloneInstance(inst)) + } + return out, nil } func (f *fakeInstanceStore) StandbyInstance(_ context.Context, id string) error { @@ -25,103 +40,290 @@ func (f *fakeInstanceStore) StandbyInstance(_ context.Context, id string) error return f.standbyErr } +func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { + f.persistedRuntime[id] = cloneRuntime(runtime) + for i := range f.instances { + if f.instances[i].ID == id { + f.instances[i].Runtime = cloneRuntime(runtime) + } + } + return nil +} + +func (f *fakeInstanceStore) SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) { + return f.events, func() {}, nil +} + type fakeConnectionSource struct { connections []Connection - err error } func (f *fakeConnectionSource) ListConnections(context.Context) ([]Connection, error) { - if f.err != nil { - return nil, f.err - } return append([]Connection(nil), f.connections...), nil } -func TestControllerWaitsFullIdleTimeoutFromStartup(t *testing.T) { - t.Parallel() +func (f *fakeConnectionSource) OpenStream(context.Context) (ConnectionStream, error) { + return &fakeConnectionStream{ + events: make(chan ConnectionEvent), + errs: make(chan error), + }, nil +} + +type fakeConnectionStream struct { + events chan ConnectionEvent + errs chan error +} - store := &fakeInstanceStore{ - instances: []Instance{{ - ID: "inst-idle", - Name: "inst-idle", - State: StateRunning, - NetworkEnabled: true, - IP: "192.168.100.10", - AutoStandby: &Policy{ - Enabled: true, - IdleTimeout: "5m", - }, - }}, +func (f *fakeConnectionStream) Events() <-chan ConnectionEvent { return f.events } + +func (f *fakeConnectionStream) Errors() <-chan error { return f.errs } + +func (f *fakeConnectionStream) Close() error { + select { + case <-f.events: + default: } - source := &fakeConnectionSource{} - controller := NewController(store, source, nil, 0) + return nil +} - now := time.Date(2026, 4, 3, 12, 0, 0, 0, time.UTC) - controller.now = func() time.Time { return now } +func TestStartupResyncClearsPersistedIdleWhenCurrentConnectionsExist(t *testing.T) { + t.Parallel() - require.NoError(t, controller.Poll(context.Background())) - assert.Empty(t, store.standbyIDs) + idleSince := time.Date(2026, 4, 6, 10, 0, 0, 0, time.UTC) + lastInbound := idleSince.Add(-time.Minute) + store := newFakeInstanceStore([]Instance{{ + ID: "inst-active", + Name: "inst-active", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.10", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "5m"}, + Runtime: &Runtime{ + IdleSince: &idleSince, + LastInboundActivityAt: &lastInbound, + }, + }}) + source := &fakeConnectionSource{connections: []Connection{{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 51234, + OriginalDestinationIP: mustAddr("192.168.100.10"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }}} + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, source, ControllerOptions{ + Now: func() time.Time { return now }, + }) - now = now.Add(4 * time.Minute) - require.NoError(t, controller.Poll(context.Background())) - assert.Empty(t, store.standbyIDs) + require.NoError(t, controller.startupResync(context.Background())) - now = now.Add(1 * time.Minute) - require.NoError(t, controller.Poll(context.Background())) - assert.Equal(t, []string{"inst-idle"}, store.standbyIDs) + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + require.Nil(t, status.IdleSince) + require.NotNil(t, store.persistedRuntime["inst-active"]) + require.Nil(t, store.persistedRuntime["inst-active"].IdleSince) } -func TestControllerClearsIdleTimerWhenTrafficReturns(t *testing.T) { +func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { t.Parallel() - store := &fakeInstanceStore{ - instances: []Instance{{ - ID: "inst-busy", - Name: "inst-busy", - State: StateRunning, - NetworkEnabled: true, - IP: "192.168.100.20", - AutoStandby: &Policy{ - Enabled: true, - IdleTimeout: "1m", - }, - }}, + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{{ + ID: "inst-idle", + Name: "inst-idle", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.20", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, + Runtime: &Runtime{ + IdleSince: &idleSince, + }, + }}) + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.NextStandbyAt) + assert.Equal(t, idleSince.Add(10*time.Minute), *status.NextStandbyAt) +} + +func TestConnectionEventsClearIdleAndStartCountdown(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-1", + Name: "inst-1", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.30", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + require.NoError(t, controller.startupResync(context.Background())) + + newEvent := ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50000, + OriginalDestinationIP: mustAddr("192.168.100.30"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: now.Add(5 * time.Second), + } + controller.handleConnectionEvent(context.Background(), newEvent) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + require.Nil(t, status.IdleSince) + + destroyEvent := newEvent + destroyEvent.Type = ConnectionEventDestroy + destroyEvent.ObservedAt = now.Add(10 * time.Second) + controller.handleConnectionEvent(context.Background(), destroyEvent) + + status = controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.IdleSince) +} + +func TestConnectionUpdateWithInactiveTCPStateStartsCountdown(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-update", + Name: "inst-update", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.31", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + require.NoError(t, controller.startupResync(context.Background())) + + event := ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50001, + OriginalDestinationIP: mustAddr("192.168.100.31"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: now.Add(5 * time.Second), } - source := &fakeConnectionSource{} - controller := NewController(store, source, nil, 0) + controller.handleConnectionEvent(context.Background(), event) + + event.ObservedAt = now.Add(10 * time.Second) + event.Connection.TCPState = TCPStateTimeWait + controller.handleConnectionEvent(context.Background(), event) - now := time.Date(2026, 4, 3, 13, 0, 0, 0, time.UTC) - controller.now = func() time.Time { return now } + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.IdleSince) +} - require.NoError(t, controller.Poll(context.Background())) - now = now.Add(30 * time.Second) - source.connections = []Connection{{ +func TestActiveReconcileStartsCountdownForStartupSeededConnections(t *testing.T) { + t.Parallel() + + idleTimeout := 30 * time.Second + store := newFakeInstanceStore([]Instance{{ + ID: "inst-reconcile", + Name: "inst-reconcile", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.32", + AutoStandby: &Policy{Enabled: true, IdleTimeout: idleTimeout.String()}, + }}) + source := &fakeConnectionSource{connections: []Connection{{ OriginalSourceIP: mustAddr("1.2.3.4"), - OriginalDestinationIP: mustAddr("192.168.100.20"), + OriginalSourcePort: 50002, + OriginalDestinationIP: mustAddr("192.168.100.32"), OriginalDestinationPort: 8080, TCPState: TCPStateEstablished, - }} - require.NoError(t, controller.Poll(context.Background())) + }}} + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, source, ControllerOptions{ + Now: func() time.Time { return now }, + ReconcileDelay: time.Second, + }) + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) - now = now.Add(70 * time.Second) source.connections = nil - require.NoError(t, controller.Poll(context.Background())) - assert.Empty(t, store.standbyIDs) + now = now.Add(5 * time.Second) + controller.handleActiveReconcile(context.Background(), "inst-reconcile") + + status = controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + require.NotNil(t, status.IdleSince) + require.NotNil(t, status.NextStandbyAt) } -func TestControllerSkipsIneligibleInstances(t *testing.T) { +func TestDuplicateDestroyDoesNotGoNegative(t *testing.T) { t.Parallel() - store := &fakeInstanceStore{ - instances: []Instance{ - {ID: "stopped", State: "Stopped", NetworkEnabled: true, IP: "192.168.1.10", AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}}, - {ID: "vgpu", State: StateRunning, NetworkEnabled: true, IP: "192.168.1.11", HasVGPU: true, AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}}, + store := newFakeInstanceStore([]Instance{{ + ID: "inst-dup", + Name: "inst-dup", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.40", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: time.Now, + }) + require.NoError(t, controller.startupResync(context.Background())) + + event := ConnectionEvent{ + Type: ConnectionEventDestroy, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50000, + OriginalDestinationIP: mustAddr("192.168.100.40"), + OriginalDestinationPort: 8080, }, + ObservedAt: time.Now().UTC(), } - controller := NewController(store, &fakeConnectionSource{}, nil, 0) + controller.handleConnectionEvent(context.Background(), event) + controller.handleConnectionEvent(context.Background(), event) + + status := controller.Describe(store.instances[0]) + require.Equal(t, 0, status.ActiveInboundCount) +} + +func TestStatusReportsObserverError(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-err", + Name: "inst-err", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.50", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "1m"}, + }}) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{}) + controller.setObserverError(errors.New("boom")) - require.NoError(t, controller.Poll(context.Background())) - assert.Empty(t, store.standbyIDs) + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusError, status.Status) + require.Equal(t, ReasonObserverError, status.Reason) } func mustAddr(raw string) netip.Addr { diff --git a/lib/autostandby/metrics.go b/lib/autostandby/metrics.go new file mode 100644 index 00000000..91d34213 --- /dev/null +++ b/lib/autostandby/metrics.go @@ -0,0 +1,164 @@ +package autostandby + +import ( + "context" + "time" + + hypotel "github.com/kernel/hypeman/lib/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +type Metrics struct { + conntrackEventsTotal metric.Int64Counter + startupResyncDuration metric.Float64Histogram + standbyAttemptsTotal metric.Int64Counter + controllerErrorsTotal metric.Int64Counter + trackedInstancesGauge metric.Int64ObservableGauge + activeConnectionsGauge metric.Int64ObservableGauge + tracer trace.Tracer +} + +func newMetrics(meter metric.Meter, tracer trace.Tracer, controller *Controller) *Metrics { + if meter == nil { + return &Metrics{tracer: tracer} + } + + conntrackEventsTotal, err := meter.Int64Counter( + "hypeman_auto_standby_conntrack_events_total", + metric.WithDescription("Total conntrack events processed by the auto-standby controller"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + startupResyncDuration, err := meter.Float64Histogram( + "hypeman_auto_standby_startup_resync_duration_seconds", + metric.WithDescription("Time spent rebuilding auto-standby state during controller startup"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(hypotel.CommonDurationHistogramBuckets()...), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + standbyAttemptsTotal, err := meter.Int64Counter( + "hypeman_auto_standby_standby_attempts_total", + metric.WithDescription("Total standby attempts issued by the auto-standby controller"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + controllerErrorsTotal, err := meter.Int64Counter( + "hypeman_auto_standby_controller_errors_total", + metric.WithDescription("Total controller and observer errors encountered by auto-standby"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + trackedInstancesGauge, err := meter.Int64ObservableGauge( + "hypeman_auto_standby_tracked_instances_total", + metric.WithDescription("Tracked instances by auto-standby controller phase"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + activeConnectionsGauge, err := meter.Int64ObservableGauge( + "hypeman_auto_standby_active_connections_total", + metric.WithDescription("Current number of active inbound connections tracked by auto-standby"), + ) + if err != nil { + return &Metrics{tracer: tracer} + } + + m := &Metrics{ + conntrackEventsTotal: conntrackEventsTotal, + startupResyncDuration: startupResyncDuration, + standbyAttemptsTotal: standbyAttemptsTotal, + controllerErrorsTotal: controllerErrorsTotal, + trackedInstancesGauge: trackedInstancesGauge, + activeConnectionsGauge: activeConnectionsGauge, + tracer: tracer, + } + + _, _ = meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error { + if controller == nil { + return nil + } + active, countdown, ready, ineligible, totalConnections := controller.metricSnapshot() + observer.ObserveInt64(m.trackedInstancesGauge, int64(active), metric.WithAttributes(attribute.String("phase", "active"))) + observer.ObserveInt64(m.trackedInstancesGauge, int64(countdown), metric.WithAttributes(attribute.String("phase", "idle_countdown"))) + observer.ObserveInt64(m.trackedInstancesGauge, int64(ready), metric.WithAttributes(attribute.String("phase", "ready_for_standby"))) + observer.ObserveInt64(m.trackedInstancesGauge, int64(ineligible), metric.WithAttributes(attribute.String("phase", "ineligible"))) + observer.ObserveInt64(m.activeConnectionsGauge, int64(totalConnections)) + return nil + }, trackedInstancesGauge, activeConnectionsGauge) + + return m +} + +func (c *Controller) recordConntrackEvent(event, result string) { + if c.metrics == nil || c.metrics.conntrackEventsTotal == nil { + return + } + c.metrics.conntrackEventsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("event", event), + attribute.String("result", result), + )) +} + +func (c *Controller) recordStartupResync(start time.Time, status string) { + if c.metrics == nil || c.metrics.startupResyncDuration == nil { + return + } + c.metrics.startupResyncDuration.Record(context.Background(), time.Since(start).Seconds(), metric.WithAttributes( + attribute.String("status", status), + )) +} + +func (c *Controller) recordStandbyAttempt(status string) { + if c.metrics == nil || c.metrics.standbyAttemptsTotal == nil { + return + } + c.metrics.standbyAttemptsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("status", status), + )) +} + +func (c *Controller) recordControllerError(operation string) { + if c.metrics == nil || c.metrics.controllerErrorsTotal == nil { + return + } + c.metrics.controllerErrorsTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("operation", operation), + )) +} + +func (c *Controller) recordObserverError(operation string) { + c.recordControllerError(operation) +} + +func (c *Controller) metricSnapshot() (active, countdown, ready, ineligible, totalConnections int) { + c.mu.RLock() + defer c.mu.RUnlock() + + now := c.now().UTC() + for _, state := range c.states { + if state.compiledPolicy == nil { + ineligible++ + continue + } + totalConnections += len(state.activeInbound) + switch { + case state.standbyRequested: + ready++ + case len(state.activeInbound) > 0: + active++ + case state.nextStandbyAt != nil && state.nextStandbyAt.After(now): + countdown++ + default: + ready++ + } + } + return +} + diff --git a/lib/autostandby/status.go b/lib/autostandby/status.go new file mode 100644 index 00000000..689493c0 --- /dev/null +++ b/lib/autostandby/status.go @@ -0,0 +1,50 @@ +package autostandby + +import "time" + +type Status string + +const ( + StatusUnsupported Status = "unsupported" + StatusDisabled Status = "disabled" + StatusIneligible Status = "ineligible" + StatusActive Status = "active" + StatusIdleCountdown Status = "idle_countdown" + StatusReadyForStandby Status = "ready_for_standby" + StatusStandbyRequested Status = "standby_requested" + StatusError Status = "error" +) + +type Reason string + +const ( + ReasonUnsupportedPlatform Reason = "unsupported_platform" + ReasonPolicyMissing Reason = "policy_missing" + ReasonPolicyDisabled Reason = "policy_disabled" + ReasonInstanceNotRunning Reason = "instance_not_running" + ReasonNetworkDisabled Reason = "network_disabled" + ReasonMissingIP Reason = "missing_ip" + ReasonHasVGPU Reason = "has_vgpu" + ReasonActiveInbound Reason = "active_inbound_connections" + ReasonIdleTimeoutNotElapsed Reason = "idle_timeout_not_elapsed" + ReasonObserverError Reason = "observer_error" + ReasonReadyForStandby Reason = "ready_for_standby" +) + +// StatusSnapshot is a diagnostic view of the controller's current state for one VM. +type StatusSnapshot struct { + Supported bool + Configured bool + Enabled bool + Eligible bool + Status Status + Reason Reason + ActiveInboundCount int + IdleTimeout string + IdleSince *time.Time + LastInboundActivityAt *time.Time + NextStandbyAt *time.Time + CountdownRemaining *time.Duration + TrackingMode string +} + diff --git a/lib/autostandby/types.go b/lib/autostandby/types.go index 49ef8a68..38165056 100644 --- a/lib/autostandby/types.go +++ b/lib/autostandby/types.go @@ -26,16 +26,24 @@ type Instance struct { IP string HasVGPU bool AutoStandby *Policy + Runtime *Runtime } // Connection is the normalized network view used by activity classification. type Connection struct { OriginalSourceIP netip.Addr + OriginalSourcePort uint16 OriginalDestinationIP netip.Addr OriginalDestinationPort uint16 TCPState TCPState } +// Runtime stores persisted and in-memory idle-tracking timestamps. +type Runtime struct { + IdleSince *time.Time `json:"idle_since,omitempty"` + LastInboundActivityAt *time.Time `json:"last_inbound_activity_at,omitempty"` +} + // TCPState is the conntrack TCP state for a flow. type TCPState uint8 @@ -50,7 +58,8 @@ const ( TCPStateTimeWait TCPState = 7 TCPStateClose TCPState = 8 TCPStateListen TCPState = 9 - TCPStateIgnore TCPState = 11 + TCPStateIgnore TCPState = 10 + TCPStateRetrans TCPState = 11 ) // Active reports whether the TCP state should keep a VM awake. diff --git a/lib/autostandby/types_test.go b/lib/autostandby/types_test.go new file mode 100644 index 00000000..39b856b8 --- /dev/null +++ b/lib/autostandby/types_test.go @@ -0,0 +1,15 @@ +package autostandby + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTCPStateConstantsMatchExpectedKernelValues(t *testing.T) { + t.Parallel() + + assert.Equal(t, TCPState(10), TCPStateIgnore) + assert.Equal(t, TCPState(11), TCPStateRetrans) +} + diff --git a/lib/instances/README.md b/lib/instances/README.md index 77fdd3ae..b240a510 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -52,6 +52,8 @@ Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEM memory-ranges # Memory state ``` +`metadata.json` also carries controller-owned auto-standby runtime timestamps when that feature is enabled, so idle countdown state can survive Hypeman restarts. + **Benefits:** - Content-addressable IDs (ULID = time-ordered) - Self-contained: all instance data in one directory diff --git a/lib/instances/auto_standby.go b/lib/instances/auto_standby.go index 52f197e7..de061dbf 100644 --- a/lib/instances/auto_standby.go +++ b/lib/instances/auto_standby.go @@ -24,6 +24,23 @@ func cloneAutoStandbyPolicy(policy *autostandby.Policy) *autostandby.Policy { return cloned } +func cloneAutoStandbyRuntime(runtime *autostandby.Runtime) *autostandby.Runtime { + if runtime == nil { + return nil + } + + cloned := &autostandby.Runtime{} + if runtime.IdleSince != nil { + idleSince := runtime.IdleSince.UTC() + cloned.IdleSince = &idleSince + } + if runtime.LastInboundActivityAt != nil { + lastInboundActivityAt := runtime.LastInboundActivityAt.UTC() + cloned.LastInboundActivityAt = &lastInboundActivityAt + } + return cloned +} + func normalizeAutoStandbyPolicy(policy *autostandby.Policy) (*autostandby.Policy, error) { normalized, err := autostandby.NormalizePolicy(policy) if err != nil { diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go index 61212404..5f333b07 100644 --- a/lib/instances/auto_standby_integration_linux_test.go +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -57,6 +57,38 @@ func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id str return err } +func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { + return s.manager.SetAutoStandbyRuntime(ctx, id, runtime) +} + +func (s integrationAutoStandbyStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { + src, unsub := s.manager.SubscribeLifecycleEvents() + dst := make(chan autostandby.InstanceEvent, 16) + go func() { + defer close(dst) + for event := range src { + var inst *autostandby.Instance + if event.Instance != nil { + inst = &autostandby.Instance{ + ID: event.Instance.Id, + Name: event.Instance.Name, + State: string(event.Instance.State), + NetworkEnabled: event.Instance.NetworkEnabled, + IP: event.Instance.IP, + HasVGPU: event.Instance.GPUProfile != "" || event.Instance.GPUMdevUUID != "", + AutoStandby: event.Instance.AutoStandby, + } + } + dst <- autostandby.InstanceEvent{ + Action: autostandby.InstanceEventAction(event.Action), + InstanceID: event.InstanceID, + Instance: inst, + } + } + }() + return dst, unsub, nil +} + func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { requireAutoStandbyE2EManualRun(t) requireKVMAccess(t) @@ -92,13 +124,14 @@ func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { }, }) require.NoError(t, err) + instanceID := inst.Id t.Cleanup(func() { - logInstanceArtifactsOnFailure(t, mgr, inst.Id) - _ = mgr.DeleteInstance(context.Background(), inst.Id) + logInstanceArtifactsOnFailure(t, mgr, instanceID) + _ = mgr.DeleteInstance(context.Background(), instanceID) }) - inst, err = waitForInstanceState(ctx, mgr, inst.Id, StateRunning, 30*time.Second) + inst, err = waitForInstanceState(ctx, mgr, instanceID, StateRunning, 30*time.Second) require.NoError(t, err) require.NoError(t, waitForVMReady(ctx, inst.SocketPath, 10*time.Second)) require.NoError(t, waitForExecAgent(ctx, mgr, inst.Id, 30*time.Second)) @@ -106,7 +139,11 @@ func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { conn, err := dialGuestPortWithRetry(inst.IP, 80, 15*time.Second) require.NoError(t, err) - defer conn.Close() + defer func() { + if conn != nil { + _ = conn.Close() + } + }() require.Eventually(t, func() bool { conns, err := connSource.ListConnections(ctx) @@ -138,8 +175,10 @@ func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { controller := autostandby.NewController( integrationAutoStandbyStore{manager: mgr}, connSource, - slog.Default(), - 250*time.Millisecond, + autostandby.ControllerOptions{ + Log: slog.Default(), + ReconnectDelay: 250 * time.Millisecond, + }, ) go func() { controllerDone <- controller.Run(controllerCtx) @@ -158,14 +197,14 @@ func TestAutoStandbyCloudHypervisorActiveInboundTCP(t *testing.T) { time.Sleep(5 * time.Second) - current, err := mgr.GetInstance(ctx, inst.Id) + current, err := mgr.GetInstance(ctx, instanceID) require.NoError(t, err) require.Equal(t, StateRunning, current.State, "instance should remain running while inbound TCP connection is open") require.NoError(t, conn.Close()) conn = nil - inst, err = waitForInstanceState(ctx, mgr, inst.Id, StateStandby, 45*time.Second) + inst, err = waitForInstanceState(ctx, mgr, instanceID, StateStandby, 45*time.Second) require.NoError(t, err) require.Equal(t, StateStandby, inst.State) } diff --git a/lib/instances/auto_standby_runtime.go b/lib/instances/auto_standby_runtime.go new file mode 100644 index 00000000..a625bc63 --- /dev/null +++ b/lib/instances/auto_standby_runtime.go @@ -0,0 +1,34 @@ +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/autostandby" +) + +// GetAutoStandbyRuntime returns the persisted auto-standby runtime metadata for an instance. +func (m *manager) GetAutoStandbyRuntime(_ context.Context, id string) (*autostandby.Runtime, error) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + return cloneAutoStandbyRuntime(meta.AutoStandbyRuntime), nil +} + +// SetAutoStandbyRuntime persists auto-standby runtime metadata for an instance. +func (m *manager) SetAutoStandbyRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.AutoStandbyRuntime = cloneAutoStandbyRuntime(runtime) + return m.saveMetadata(meta) +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go index ec3e8142..31d2efb0 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -269,7 +269,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin } now := time.Now() - forkMeta := cloneStoredMetadataForFork(meta.StoredMetadata) + forkMeta := cloneStoredMetadata(meta.StoredMetadata) forkMeta.Id = forkID forkMeta.Name = req.Name forkMeta.CreatedAt = now @@ -470,7 +470,7 @@ func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) return err } -func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { +func cloneStoredMetadata(src StoredMetadata) StoredMetadata { dst := src if src.Env != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 2cc4fc02..1e73927c 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -304,7 +304,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { }, } - cloned := cloneStoredMetadataForFork(src) + cloned := cloneStoredMetadata(src) require.Equal(t, src, cloned) cloned.Env["A"] = "2" diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go new file mode 100644 index 00000000..149aa4f5 --- /dev/null +++ b/lib/instances/lifecycle_events.go @@ -0,0 +1,74 @@ +package instances + +import "sync" + +// LifecycleEventAction identifies which instance lifecycle action occurred. +type LifecycleEventAction string + +const ( + LifecycleEventCreate LifecycleEventAction = "create" + LifecycleEventUpdate LifecycleEventAction = "update" + LifecycleEventStart LifecycleEventAction = "start" + LifecycleEventStop LifecycleEventAction = "stop" + LifecycleEventStandby LifecycleEventAction = "standby" + LifecycleEventRestore LifecycleEventAction = "restore" + LifecycleEventDelete LifecycleEventAction = "delete" + LifecycleEventFork LifecycleEventAction = "fork" +) + +// LifecycleEvent is a global instance change event stream used by background +// controllers that need to react to instance eligibility or identity changes. +type LifecycleEvent struct { + Action LifecycleEventAction + InstanceID string + Instance *Instance +} + +type lifecycleSubscribers struct { + mu sync.Mutex + subs []chan LifecycleEvent +} + +func newLifecycleSubscribers() *lifecycleSubscribers { + return &lifecycleSubscribers{} +} + +func (s *lifecycleSubscribers) Subscribe() (<-chan LifecycleEvent, func()) { + ch := make(chan LifecycleEvent, 32) + + s.mu.Lock() + s.subs = append(s.subs, ch) + s.mu.Unlock() + + return ch, func() { + s.mu.Lock() + defer s.mu.Unlock() + for i, sub := range s.subs { + if sub == ch { + s.subs = append(s.subs[:i], s.subs[i+1:]...) + close(ch) + break + } + } + } +} + +func (s *lifecycleSubscribers) Notify(event LifecycleEvent) { + s.mu.Lock() + subs := append([]chan LifecycleEvent(nil), s.subs...) + s.mu.Unlock() + + for _, ch := range subs { + trySendLifecycleEvent(ch, event) + } +} + +func trySendLifecycleEvent(ch chan LifecycleEvent, event LifecycleEvent) { + defer func() { recover() }() + + select { + case ch <- event: + default: + } +} + diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 4ef8d9bf..38d27028 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -117,6 +117,7 @@ type manager struct { // State change subscriptions for waitForState stateSubscribers *subscribers + lifecycleEvents *lifecycleSubscribers // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -170,6 +171,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste compressionJobs: make(map[string]*compressionJob), nativeCodecPaths: make(map[string]string), stateSubscribers: newSubscribers(), + lifecycleEvents: newLifecycleSubscribers(), } m.deleteSnapshotFn = m.deleteSnapshot @@ -199,6 +201,11 @@ func (m *manager) Subscribe(instanceID string) (<-chan StateChange, func()) { return m.stateSubscribers.Subscribe(instanceID) } +// SubscribeLifecycleEvents returns a channel of global instance lifecycle events. +func (m *manager) SubscribeLifecycleEvents() (<-chan LifecycleEvent, func()) { + return m.lifecycleEvents.Subscribe() +} + // notifyStateChange broadcasts a state change to all subscribers for the instance. func (m *manager) notifyStateChange(instanceID string, inst *Instance) { m.stateSubscribers.Notify(instanceID, StateChange{ @@ -207,6 +214,24 @@ func (m *manager) notifyStateChange(instanceID string, inst *Instance) { }) } +func (m *manager) notifyLifecycleEvent(action LifecycleEventAction, inst *Instance) { + if inst == nil { + return + } + m.lifecycleEvents.Notify(LifecycleEvent{ + Action: action, + InstanceID: inst.Id, + Instance: inst, + }) +} + +func (m *manager) notifyLifecycleDelete(instanceID string) { + m.lifecycleEvents.Notify(LifecycleEvent{ + Action: LifecycleEventDelete, + InstanceID: instanceID, + }) +} + // getHypervisor creates a hypervisor client for the given socket and type. // Used for connecting to already-running VMs (e.g., for state queries). func (m *manager) getHypervisor(socketPath string, hvType hypervisor.Type) (hypervisor.Hypervisor, error) { @@ -270,7 +295,11 @@ func (m *manager) CreateInstance(ctx context.Context, req CreateInstanceRequest) // 1. ULID generation is unique // 2. Filesystem mkdir is atomic per instance directory // 3. Concurrent creates of different instances don't conflict - return m.createInstance(ctx, req) + inst, err := m.createInstance(ctx, req) + if err == nil { + m.notifyLifecycleEvent(LifecycleEventCreate, inst) + } + return inst, err } // DeleteInstance stops and deletes an instance @@ -281,6 +310,7 @@ func (m *manager) DeleteInstance(ctx context.Context, id string) error { err := m.deleteInstance(ctx, id) if err == nil { + m.notifyLifecycleDelete(id) m.stateSubscribers.CloseAll(id) // Clean up the lock after successful deletion m.instanceLocks.Delete(id) @@ -332,11 +362,16 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, fmt.Errorf("wait for fork guest agent readiness: %w", err) } } + m.notifyLifecycleEvent(LifecycleEventFork, inst) return inst, nil } func (m *manager) ForkSnapshot(ctx context.Context, snapshotID string, req ForkSnapshotRequest) (*Instance, error) { - return m.forkSnapshot(ctx, snapshotID, req) + inst, err := m.forkSnapshot(ctx, snapshotID, req) + if err == nil { + m.notifyLifecycleEvent(LifecycleEventFork, inst) + } + return inst, err } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) @@ -347,6 +382,7 @@ func (m *manager) StandbyInstance(ctx context.Context, id string, req StandbyIns inst, err := m.standbyInstance(ctx, id, req, false) if err == nil { m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(LifecycleEventStandby, inst) } return inst, err } @@ -359,6 +395,7 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er inst, err := m.restoreInstance(ctx, id) if err == nil { m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(LifecycleEventRestore, inst) } return inst, err } @@ -370,6 +407,7 @@ func (m *manager) RestoreSnapshot(ctx context.Context, id string, snapshotID str inst, err := m.restoreSnapshot(ctx, id, snapshotID, req) if err == nil { m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(LifecycleEventRestore, inst) } return inst, err } @@ -382,6 +420,7 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error inst, err := m.stopInstance(ctx, id) if err == nil { m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(LifecycleEventStop, inst) } return inst, err } @@ -394,6 +433,7 @@ func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanc inst, err := m.startInstance(ctx, id, req) if err == nil { m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(LifecycleEventStart, inst) } return inst, err } @@ -403,7 +443,11 @@ func (m *manager) UpdateInstance(ctx context.Context, id string, req UpdateInsta lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.updateInstance(ctx, id, req) + inst, err := m.updateInstance(ctx, id, req) + if err == nil { + m.notifyLifecycleEvent(LifecycleEventUpdate, inst) + } + return inst, err } // ListInstances returns instances, optionally filtered by the given criteria. diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go index 69bdb053..fdb53796 100644 --- a/lib/instances/metadata_clone.go +++ b/lib/instances/metadata_clone.go @@ -8,6 +8,7 @@ func deepCopyMetadata(src *metadata) *metadata { } return &metadata{ - StoredMetadata: cloneStoredMetadataForFork(src.StoredMetadata), + StoredMetadata: cloneStoredMetadata(src.StoredMetadata), + AutoStandbyRuntime: cloneAutoStandbyRuntime(src.AutoStandbyRuntime), } } diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index be982f74..9ea0296e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -147,7 +147,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps SourceHypervisor: stored.HypervisorType, CreatedAt: time.Now(), }, - StoredMetadata: cloneStoredMetadataForFork(meta.StoredMetadata), + StoredMetadata: cloneStoredMetadata(meta.StoredMetadata), } sizeBytes, err := snapshotstore.DirectoryFileSize(snapshotGuestDir) if err != nil { @@ -199,7 +199,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps SourceHypervisor: stored.HypervisorType, CreatedAt: time.Now(), }, - StoredMetadata: cloneStoredMetadataForFork(meta.StoredMetadata), + StoredMetadata: cloneStoredMetadata(meta.StoredMetadata), } sizeBytes, err := snapshotstore.DirectoryFileSize(snapshotGuestDir) if err != nil { @@ -289,7 +289,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } - restored := cloneStoredMetadataForFork(rec.StoredMetadata) + restored := cloneStoredMetadata(rec.StoredMetadata) restored.Id = sourceMeta.Id restored.Name = sourceMeta.Name restored.DataDir = m.paths.InstanceDir(id) @@ -427,7 +427,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } now := time.Now() - forkMeta := cloneStoredMetadataForFork(rec.StoredMetadata) + forkMeta := cloneStoredMetadata(rec.StoredMetadata) forkMeta.Id = forkID forkMeta.Name = req.Name forkMeta.CreatedAt = now diff --git a/lib/instances/storage.go b/lib/instances/storage.go index 4bf7f45b..3ba47b0e 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/images" ) @@ -27,6 +28,7 @@ import ( // metadata wraps StoredMetadata for JSON serialization type metadata struct { StoredMetadata + AutoStandbyRuntime *autostandby.Runtime `json:"auto_standby_runtime,omitempty"` } // ensureDirectories creates the instance directory structure diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 6e619df7..8ceb515c 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -29,6 +29,33 @@ const ( BearerAuthScopes = "bearerAuth.Scopes" ) +// Defines values for AutoStandbyStatusReason. +const ( + AutoStandbyStatusReasonActiveInboundConnections AutoStandbyStatusReason = "active_inbound_connections" + AutoStandbyStatusReasonHasVgpu AutoStandbyStatusReason = "has_vgpu" + AutoStandbyStatusReasonIdleTimeoutNotElapsed AutoStandbyStatusReason = "idle_timeout_not_elapsed" + AutoStandbyStatusReasonInstanceNotRunning AutoStandbyStatusReason = "instance_not_running" + AutoStandbyStatusReasonMissingIp AutoStandbyStatusReason = "missing_ip" + AutoStandbyStatusReasonNetworkDisabled AutoStandbyStatusReason = "network_disabled" + AutoStandbyStatusReasonObserverError AutoStandbyStatusReason = "observer_error" + AutoStandbyStatusReasonPolicyDisabled AutoStandbyStatusReason = "policy_disabled" + AutoStandbyStatusReasonPolicyMissing AutoStandbyStatusReason = "policy_missing" + AutoStandbyStatusReasonReadyForStandby AutoStandbyStatusReason = "ready_for_standby" + AutoStandbyStatusReasonUnsupportedPlatform AutoStandbyStatusReason = "unsupported_platform" +) + +// Defines values for AutoStandbyStatusStatus. +const ( + AutoStandbyStatusStatusActive AutoStandbyStatusStatus = "active" + AutoStandbyStatusStatusDisabled AutoStandbyStatusStatus = "disabled" + AutoStandbyStatusStatusError AutoStandbyStatusStatus = "error" + AutoStandbyStatusStatusIdleCountdown AutoStandbyStatusStatus = "idle_countdown" + AutoStandbyStatusStatusIneligible AutoStandbyStatusStatus = "ineligible" + AutoStandbyStatusStatusReadyForStandby AutoStandbyStatusStatus = "ready_for_standby" + AutoStandbyStatusStatusStandbyRequested AutoStandbyStatusStatus = "standby_requested" + AutoStandbyStatusStatusUnsupported AutoStandbyStatusStatus = "unsupported" +) + // Defines values for BuildEventType. const ( Heartbeat BuildEventType = "heartbeat" @@ -211,6 +238,50 @@ type AutoStandbyPolicy struct { IgnoreSourceCidrs *[]string `json:"ignore_source_cidrs,omitempty"` } +// AutoStandbyStatus defines model for AutoStandbyStatus. +type AutoStandbyStatus struct { + // ActiveInboundConnections Number of currently tracked qualifying inbound TCP connections. + ActiveInboundConnections int `json:"active_inbound_connections"` + + // Configured Whether the instance has any auto-standby policy configured. + Configured bool `json:"configured"` + + // CountdownRemaining Remaining time before the controller attempts standby, when applicable. + CountdownRemaining *string `json:"countdown_remaining"` + + // Eligible Whether the instance is currently eligible to enter standby. + Eligible bool `json:"eligible"` + + // Enabled Whether the configured auto-standby policy is enabled. + Enabled bool `json:"enabled"` + + // IdleSince When the controller most recently observed the instance become idle. + IdleSince *time.Time `json:"idle_since"` + + // IdleTimeout Configured idle timeout from the auto-standby policy. + IdleTimeout *string `json:"idle_timeout"` + + // LastInboundActivityAt Timestamp of the most recent qualifying inbound TCP activity the controller observed. + LastInboundActivityAt *time.Time `json:"last_inbound_activity_at"` + + // NextStandbyAt When the controller expects to attempt standby next, if a countdown is active. + NextStandbyAt *time.Time `json:"next_standby_at"` + Reason AutoStandbyStatusReason `json:"reason"` + Status AutoStandbyStatusStatus `json:"status"` + + // Supported Whether the current host platform supports auto-standby diagnostics. + Supported bool `json:"supported"` + + // TrackingMode Diagnostic identifier for the runtime tracking mode in use. + TrackingMode string `json:"tracking_mode"` +} + +// AutoStandbyStatusReason defines model for AutoStandbyStatus.Reason. +type AutoStandbyStatusReason string + +// AutoStandbyStatusStatus defines model for AutoStandbyStatus.Status. +type AutoStandbyStatusStatus string + // AvailableDevice defines model for AvailableDevice. type AvailableDevice struct { // CurrentDriver Currently bound driver (null if none) @@ -1705,6 +1776,9 @@ type ClientInterface interface { UpdateInstance(ctx context.Context, id string, body UpdateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetAutoStandbyStatus request + GetAutoStandbyStatus(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // ForkInstanceWithBody request with any body ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2155,6 +2229,18 @@ func (c *Client) UpdateInstance(ctx context.Context, id string, body UpdateInsta return c.Client.Do(req) } +func (c *Client) GetAutoStandbyStatus(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetAutoStandbyStatusRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) ForkInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewForkInstanceRequestWithBody(c.Server, id, contentType, body) if err != nil { @@ -3546,6 +3632,40 @@ func NewUpdateInstanceRequestWithBody(server string, id string, contentType stri return req, nil } +// NewGetAutoStandbyStatusRequest generates requests for GetAutoStandbyStatus +func NewGetAutoStandbyStatusRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/auto-standby/status", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewForkInstanceRequest calls the generic ForkInstance builder with application/json body func NewForkInstanceRequest(server string, id string, body ForkInstanceJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -4976,6 +5096,9 @@ type ClientWithResponsesInterface interface { UpdateInstanceWithResponse(ctx context.Context, id string, body UpdateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateInstanceResponse, error) + // GetAutoStandbyStatusWithResponse request + GetAutoStandbyStatusWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetAutoStandbyStatusResponse, error) + // ForkInstanceWithBodyWithResponse request with any body ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) @@ -5662,6 +5785,30 @@ func (r UpdateInstanceResponse) StatusCode() int { return 0 } +type GetAutoStandbyStatusResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *AutoStandbyStatus + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetAutoStandbyStatusResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetAutoStandbyStatusResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type ForkInstanceResponse struct { Body []byte HTTPResponse *http.Response @@ -6587,6 +6734,15 @@ func (c *ClientWithResponses) UpdateInstanceWithResponse(ctx context.Context, id return ParseUpdateInstanceResponse(rsp) } +// GetAutoStandbyStatusWithResponse request returning *GetAutoStandbyStatusResponse +func (c *ClientWithResponses) GetAutoStandbyStatusWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetAutoStandbyStatusResponse, error) { + rsp, err := c.GetAutoStandbyStatus(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetAutoStandbyStatusResponse(rsp) +} + // ForkInstanceWithBodyWithResponse request with arbitrary body returning *ForkInstanceResponse func (c *ClientWithResponses) ForkInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ForkInstanceResponse, error) { rsp, err := c.ForkInstanceWithBody(ctx, id, contentType, body, reqEditors...) @@ -7926,6 +8082,46 @@ func ParseUpdateInstanceResponse(rsp *http.Response) (*UpdateInstanceResponse, e return response, nil } +// ParseGetAutoStandbyStatusResponse parses an HTTP response from a GetAutoStandbyStatusWithResponse call +func ParseGetAutoStandbyStatusResponse(rsp *http.Response) (*GetAutoStandbyStatusResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetAutoStandbyStatusResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest AutoStandbyStatus + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseForkInstanceResponse parses an HTTP response from a ForkInstanceWithResponse call func ParseForkInstanceResponse(rsp *http.Response) (*ForkInstanceResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -9227,6 +9423,9 @@ type ServerInterface interface { // Update instance properties // (PATCH /instances/{id}) UpdateInstance(w http.ResponseWriter, r *http.Request, id string) + // Get auto-standby diagnostic status + // (GET /instances/{id}/auto-standby/status) + GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request, id string) // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) ForkInstance(w http.ResponseWriter, r *http.Request, id string) @@ -9458,6 +9657,12 @@ func (_ Unimplemented) UpdateInstance(w http.ResponseWriter, r *http.Request, id w.WriteHeader(http.StatusNotImplemented) } +// Get auto-standby diagnostic status +// (GET /instances/{id}/auto-standby/status) +func (_ Unimplemented) GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) func (_ Unimplemented) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { @@ -10319,6 +10524,37 @@ func (siw *ServerInterfaceWrapper) UpdateInstance(w http.ResponseWriter, r *http handler.ServeHTTP(w, r) } +// GetAutoStandbyStatus operation middleware +func (siw *ServerInterfaceWrapper) GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetAutoStandbyStatus(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // ForkInstance operation middleware func (siw *ServerInterfaceWrapper) ForkInstance(w http.ResponseWriter, r *http.Request) { @@ -11482,6 +11718,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Patch(options.BaseURL+"/instances/{id}", wrapper.UpdateInstance) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/instances/{id}/auto-standby/status", wrapper.GetAutoStandbyStatus) + }) r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/fork", wrapper.ForkInstance) }) @@ -12538,6 +12777,41 @@ func (response UpdateInstance500JSONResponse) VisitUpdateInstanceResponse(w http return json.NewEncoder(w).Encode(response) } +type GetAutoStandbyStatusRequestObject struct { + Id string `json:"id"` +} + +type GetAutoStandbyStatusResponseObject interface { + VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error +} + +type GetAutoStandbyStatus200JSONResponse AutoStandbyStatus + +func (response GetAutoStandbyStatus200JSONResponse) VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetAutoStandbyStatus404JSONResponse Error + +func (response GetAutoStandbyStatus404JSONResponse) VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetAutoStandbyStatus500JSONResponse Error + +func (response GetAutoStandbyStatus500JSONResponse) VisitGetAutoStandbyStatusResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type ForkInstanceRequestObject struct { Id string `json:"id"` Body *ForkInstanceJSONRequestBody @@ -13803,6 +14077,9 @@ type StrictServerInterface interface { // Update instance properties // (PATCH /instances/{id}) UpdateInstance(ctx context.Context, request UpdateInstanceRequestObject) (UpdateInstanceResponseObject, error) + // Get auto-standby diagnostic status + // (GET /instances/{id}/auto-standby/status) + GetAutoStandbyStatus(ctx context.Context, request GetAutoStandbyStatusRequestObject) (GetAutoStandbyStatusResponseObject, error) // Fork an instance from stopped, standby, or running (with from_running=true) // (POST /instances/{id}/fork) ForkInstance(ctx context.Context, request ForkInstanceRequestObject) (ForkInstanceResponseObject, error) @@ -14568,6 +14845,32 @@ func (sh *strictHandler) UpdateInstance(w http.ResponseWriter, r *http.Request, } } +// GetAutoStandbyStatus operation middleware +func (sh *strictHandler) GetAutoStandbyStatus(w http.ResponseWriter, r *http.Request, id string) { + var request GetAutoStandbyStatusRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetAutoStandbyStatus(ctx, request.(GetAutoStandbyStatusRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetAutoStandbyStatus") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetAutoStandbyStatusResponseObject); ok { + if err := validResponse.VisitGetAutoStandbyStatusResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // ForkInstance operation middleware func (sh *strictHandler) ForkInstance(w http.ResponseWriter, r *http.Request, id string) { var request ForkInstanceRequestObject @@ -15345,262 +15648,272 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9e3Mbu7E4+FVQ3NwKlZAU9bAs69ap38qS7aN7LFtr2c7eHHopcAYkEc0AcwAMJdrl", - "f/MB8hHzSbbQAOZFDDmU9bBi35tKZA4ejUaj0d3ox5dWwOOEM8KUbB18aclgSmIMfx4qhYPpRx6lMXlH", - "/kiJVPrnRPCECEUJNIp5ytQwwWqq/xUSGQiaKMpZ66B1htUUXU2JIGgGoyA55WkUohFB0I+ErU6LXOM4", - "iUjroLUZM7UZYoVbnZaaJ/onqQRlk9bXTksQHHIWzc00Y5xGqnUwxpEkncq0p3pohCXSXbrQJxtvxHlE", - "MGt9hRH/SKkgYevg9+IyPmWN+egfJFB68sNU8XOFWTian/GIBvPFxb6mLL2G2RBOFY+xogGSpg9KoBMa", - "YUlCxBnCgaIzgigb8ZSF6P3RGQo4YyTQg8kB4yNJxIyEaCx4jNSUoCmXCtoogYNLpPAoIr0Ba3Uq+0GY", - "/hKuxtLfpkRNifAASyWyo6AxF0hNqUSU6a8B6RU3TImULGK206JhRIaKxoSnahFRv/IrFHE2gWW5cVGc", - "SoWmeEbQZyI4+iPFER3PKZvUI2lExlwQ9Os8ITFmKIlwQCSiClGmuFuNwVFOY09iH3HRCeOCDEMiFWVY", - "jz9MuDAnogz9W/gDR6jQFkCD9khNsXJUzrhCl4Qk5YXiK3xZRuPv29udZ/1+/1OnRRWJzbHC1zRO49bB", - "3pMnO086rZgy8++tDHrKFJkQocG3v2Ah8LywHMlTEZBhQEOxbCVBRAlT6Ojk+N0NF9Da6vfg/zf3W53W", - "1rPt3tbePvx7a69VXNYC4suQf/UdvRmmkabGYzKjAVnkQEEqBGFqGAo6I2JxnUfmezRHho5MO9RmaRQh", - "OkaMM7JRohE2oyHVTEg30VNXSD0HPwSYhjT0ML+jE2Q+o5Nj1J6S6/Ik209H+636IRmOiefspDFmXc3X", - "NFhufGhbHPv1rpfIeRynw4ngabI48snb09MPCD4ilsYjIooj7m/7yC4J6BCHoSBS+tfvPhZh6/f7/QO8", - "fdDv9/o+KGeEhVzUotR89qN0qx+SJUM2QqkdfwGlbz6eHJ8coiMuEi7g1C/OVLlTiugprqtINuVd8V09", - "z1MahYtUP9I/EzF0x9KLsBN3Zk+OER/DMbb90MdT1NbcPSSjdDKhbLLRhN61rBARRcIh9nB2ABXZNpov", - "6jtAKhwnrU5rzEWsO7VCrEhXf2k0oSB4xXS6RaPJFo9aanZyGMu60V0TRBmKaRRRSQLOQlmcgzK1t1u/", - "mMKBIUJwD4d6oX9GMZESTwhqgwxxNSVM32EqlfpCHmMakbDRHvkIwSzmH3yEaEiYomNaPt+GnLp4FGxt", - "73h5R4wnZBjSiRUCy8Mfw++axPQ4CkFr/0L0QZs3WwdMKch4cb6XwLphEkHGRBBN4984XSL4jDB9WvR8", - "f4J5W//XZi4db1rReBOQeZY3/9pp/ZGSlAwTLqmBcIFz2S+ajADVCHr4YYZPy/a6QFFSYbH8fECLWziJ", - "Br5GuDk3TfVtjicru7zXbaq8E1ijnbLEBWpZ5IsZYR79JOBM2Q8VeZ1PUEQZQbaF3QuQeOcJ+SXiwBJv", - "CQ8Z+hcPv4b7BszL/FAzmv7WaRGmJcbfWxGfFLE5JVioESkhs+YKswPl0NWi/6x0fCp3FZZkuJyDnFHG", - "SAgKkj3YpiVKJSiJC8uHU3RJ1XBGhPSeOQDrN6qQbVE7VMSDyzGNyHCK5dRAjMOQGun4rLQSj7RW0jxx", - "opmgGxCkCIkUR+e/Hm4/2UN2Ag8OraiuGyyupNBbD2/aIoXFCEeRlzbqyW39O3qRQvwUcJ4djLq7J6NA", - "R5iG07XsburhO60klVPzF/BuDRXcfZoNaPKK9N+fPIs+AiZhtIRac4VfBsxUoUnENU7nKGX0j7QkYPfQ", - "yRg0In1R0JCEHYThg2bZWonuTggjQvOpXGsvCMGoTXqTXgcNtFzY1VJwF293+/1uf9Aqi7HRbneSpBoV", - "WCkiNID/3++4+/mw+/d+99mn/M9hr/vpr3/yEUBTydxJhXadbXf2O8gBWxTXq4CuEuVvzP2L4Ps4jtnq", - "E80n1t3po5NFwcGsNeTBJRE9yjcjOhJYzDfZhLLrgwgrIlV55cvb3iouYB1LkMAmGk1roqGi9AAZtyN+", - "RUSgOXBENOHJjmbCVMkOwlpvBuaF9C353yjATJ8FI1xwgQgL0RVVU4ShXRlb8byLE9qlBtRWpxXj69eE", - "TdS0dbC3s0Dnmsjb9o/up7+4nzb+j5fURRoRD5G/46mibILgc9GO5WDIbBLLdsRhN41AzIspOzHdthaN", - "Lt+2w24hy3baKHO1W62Z0NCavVYBsmjQ1MpW7FEd3s6IEDR01/LR6TFqR/SS2POCRMrQIO33dwJoAH8S", - "+0vA4xiz0Py20UNvY6r0dZjmt7yxUVbMSSSYchBUooivYz8CSREUHBwtvceXocaL7aNs3MVb/1cuVTfG", - "DE8IqKO2IRoJfkk0oMb8S4lEl2SupZw5muhBuzMqqT5/hM3QDBurQ2/A3k+5JKaJ+6Q1mYDQGUExDy6N", - "rXPKQZOf4SglsoOuplrk0NxcEBzZn5EgMaZswKYaSBnwhIRaCTHNYGnogrDZBYpxAsccCwJnHMVYEUFx", - "RD8bm7XuEpOQ6htuwAgcDJRgfeaDgAt9feu9JTiYFrDwZ4kujMByAcNfUKbJ+sIczIp19kvr7Yf3z99+", - "eHM8fHv24s3hyfC3F/+rfzadWge/f2mZt4lMUnlOsCAC/ekLrPerEW9DIloHrcNUTbmgn4215munpXEg", - "NX3hhPZ4QhimvYDHrU7rL8V/fvr6yQlkxqI+08fAA9hXrzBk7lIPSzp21kCJrIUJZEMMzyzAol6dfdjU", - "t3OCpVRTwdPJtHwwrGiw1pEIqbwcUj4cJT6YqLxEJ5tvkRZcUET1Ac0Ela1+//T5phy09D+euH9s9NCx", - "ObUAvuZBXFj5SU41+WTPHEdnHxCOIh5YG8pYK1tjOkkFCXsV0x2M7mPwhCkxTzj1KXEV5pQ3XeRR3W7+", - "dQ1WtDmibFPqbegG6+Ed6ObGqsQLNqOCs1irczMsqL6nZfmsvHl7/GL44s3H1oG+CMI0sFbJs7fv3rcO", - "Wjv9fr/lI1BNQSt44KuzD0ewU+bYqCRKJ0NJP3tEicNsfSgmMRdGhbZ9UHtaljTMuUWwOYPWzqvnhri2", - "XgFduU0JqYTWbhQzcJlitl8991HLdJ4QMaPSZ2f7Nfvmdn7xfatE2/AIJzKiBSruFfSXIOJp2C1M2WmN", - "qSCBwJrsWp3WHyTWgvzssyadHHZPP7/5q5EAu0IyxVFCGVkimn4nIuIVF5cRx2F365YlREaUHntxiW/M", - "h/L+WprIH7wWnlhHmIVXNFTTYcivmAbZw1ftF5Q1zpjrtV4Jjv79z399PM31rK1Xo8Ry2q3tJ9/IaSu8", - "VQ/ttaFkC0kT/zI+JP5FfDz99z//5VbysIswgsiNhDq7/y/MCMCyFx/PjTnU/3ae397Z66jiVqGG7sjR", - "3spXcx+j5jMiIjwvMF4LU2urD9yvApWg4BaAbD/NRi+R7ryCDevR3CX/qqrkb/f9jNYDlAem55pX2Huh", - "CSQZIFvbp/bP7UWQaiC6pMkQpOYhnmQ232U+EOeXNLGiOPQw2xhFhhGEKQjvI85Vb8D+NiUMwd7BBpNr", - "EgDPkwordHh2ItEVjSKwEAFTWbxatGBfeEeH5lLp/xYp66BRqrS0zhVBVm+CSVKABRqPCEoZdu/hFdnZ", - "LnDRGwPQckkEI9HQyMayIWZMJ2Q71SIHljrGUhFhuH2alPF1/NvpOWofzxmOaYB+M6Oe8jCNCDpPE80P", - "NsrY6wxYIshMqxBsAtZKauflY8RT1eXjrhKEOBBjGCyzsdnH2tmrsw/2uV9u9AbsHdGIJSy0ni3uxrFe", - "DyFnf9YnloTlYYvzV5Be5wEjGU7klKthknkLLeNO57Z5roo3NyZ0WrMgSctbul3dzjfwoq+RN6NCpTjS", - "vLYkTnof+I3XlkdtME5hRfXF8r3cS0SVX2abWlzMyODC5fUP8RhOjKTU2HBSUOUXTChOz/zSDNgV458w", - "B8hSw1Guan7DXOdmkCqK7Ngdt7IbYOkkw0nF3HQ76DmUBdW8kbcVuMJZiVCi9oXW5i0da/39ooMu/lL6", - "QZ99p1po+eIKGWwAP2H6p+L4VaPESnPBWv5Nxc3B8ub7cShrPZ3QbAspgZnUd6uWsRLSQ78CE0eKxInm", - "ZGyCqETSMF8SIsav/htxI9S4rgOmQZPGT8SiIzMaSTphlE02tJivLyYchsayNE5VKnS7GZU5Nsuk46w3", - "1QW8N9ARw4/BJZCyIEpDgi6cheeiLBcu2n8WVUJrEFrQcAxKQLMBZU9txqnS0+sFx1gFU40nnirjOGaX", - "LssAlK1Mqx5ULSzZU9sN9v88YxdVz8+ZR8XRi7OPPGAWLNgn68yAVlDxmygvyRy23Jkj8YJBsmiJ9NsL", - "BZE8mhF77RZtmSPwbeVGcMrNmMYgaW2Q+vhXvTp91rlVW6Hx1Rj9ZVXB49MqVdctNqcYK/1b99+MC+nF", - "mfk6WjGWBJAPqscBAnHsomN0JQIWCMQ0sUQopIIEamF4yiYDBj4kF/aXnh3tQh9yLaPciqew4tYvuLS1", - "qLCzTuyDYfTSeEyVImGnLBtcEpLI1YvS4rU1XHus64JcCeoYmTUYhQ3FM8LGXAQktkrCtymOLwqDedW4", - "9YZYdOkw+C3A7NzJcZJElITGf8jsB5hZM//t3sKmxzysaG3GhaA85QWOogvUto02kCB6LdLtFeMsJ/b3", - "R2eOBLJn74+nHU2RmgtcTJVKhvq/5FCf4ovqYLavO+G5K/V+H/Sr3d0du6vW6GYArgxbtq953SLqt8aJ", - "37Uva5ouNJTWz6SJKH+Ud8ktqZeUhU0H+E23rbXOZYKR0zTu2kCXCNJNk4nA4GJ7m+a5G7+bAjbrOfiK", - "wBWfm2TuEp9KxeOCsyRqV1w8aNkZpIysGY+6IVYYTJkN7a0G3EXH43huhjK6WJ0lZjgZefyG6GfNddGE", - "TvBorsrvB1t9byTBNz5iO1h821LnwG80SBIOFV/uwkzHyLVt4rEI98lQ8eFsTD0jZ9da7v9CJQoq0QJW", - "r9VDdJOAWnMCyDjB1HiYGiSA0PjxtPh21xuwLly/B+g4myAbNhsSg2yJQ/Ny0uaiAAQFtzU0mm8gjD6e", - "9tD7DNo/S6QVlhlxEQ1TLNGIEIZSMD3Dbdg1d3ERgFTCpamq3a3txAQ/bMATJbffelmQDVhpspAhcJUa", - "0cp64DoyG2XfhDErWsEaWa2WOX6/IxMqlai4faP2u5dHOzs7z6r2y+0n3f5Wd+vJ+63+QV//5+/NPcRv", - "P77DN9ZhmbdY57Mi9zn6cHK8bY2l5XnU5138bP/6Gqtne/RKPvscj8TkHzv4XiJA/KzsOPeaQ+1UEtF1", - "bFJTlc9XruCSVuMLd2MXtzvyWMsdcJe1NZh4r1veRWiLz2nauuyuH3xSZZgr3a4Li1vU5OcJ6J35KSlI", - "cNa7MaBeP85jKi+fC4IvQ37FPPd2jCdEDs195vdnSKVxsiHX1rohOFdjad5Ny1bPrd2nu/s7e7v7/b4n", - "omOR4HlAh4G+gRoB8PboBEV4TgSCPqgND14hGkV8VCb0Jzt7+0/7z7a2m8Jhnnia4SFTvFwv1LYY+asL", - "zHVfSkBtbz/d29nZ6e/tbe82gsraixsB5WzLJZHk6c7T3a397d1GWPAJ9C9chE1VgA99rgtafzKPjV2Z", - "kICOaYAgRgfpDqgdwxVGsteq8pkc4XBojSf+u0NhGsmlHhNmMtvSGNriNFI0iYj5BhvSyBYNKz+Gkbwh", - "oYwRMcwCkNYYycYlrfQQcGvJmqBSfFkJdadUghSSC0+UROGBOaEr+RzsZg7Ypzo6sGtoSA2vterUjciM", - "REUiMFeXBjbmgqCMTsymlVZF2QxHNBxSlqRekqhF5ctUgCxqBkV4xFNlnhlhw4qTgNcz6B5jza6b6bkv", - "ubhc6T+qb+KhSBnTw6y0Ch2CIX1sTTVwi2Nke7sQhYLQlz0HmkdT+12id6aHsRDlPydpOYy7AzNZSxJD", - "gkjFgZNag6Edpql06ZdbwFjq3D/MfDnvvCffl+7YuAvcroYtJkQNpcJqpcSiKeU9tD+H5o3d0XXHlYaU", - "Bnhn5Oo+kA7++l1Ntl3JcHI3GF/mjJbZGvJGcAsLGpIegtMFXjEuPrBy0s4VTxISZvaf3oBZf+7sJ2le", - "UHRHgwc1JVQgLuiElicuG9ju0qttHVJ01HRjcix2XJRQ4SO4b9QfejxWRBgMutDnYvyS3YRWp2Vx3+q0", - "LCcqo8b96MFI7mq5AOKrsw/r+qYlgo9p5Fku+ELYr1Yzc15br3f7592t/8d4YGp6AxGNMuM/EfOwnO3B", - "tW9287w6+3BWB1OW2gEVoVtYU+bx4uEcmV+Dw4h9VLKvklaDceSvL5Zsklz2fuaTZccCx2SUjsdEDGOP", - "ce2l/o5MA+PaRBk6fV6WZ7Xc3FRrPittDqjNYxzYyPxm2PcY5CrL6BSw+cm/Xe+IuYbr4vn0Vgnbxob0", - "9dCbLJkGenX2QaLcS8ljqStvb62//Nl0LmmAIzOiCc+lrGhgA+JsLCGf5R2tKdIjJ8de2dAdBNSeTZIU", - "juH5u+7J24+bcUhmnRJM4Fk05RHRcG8UuMXMRfXlzv0lJjGrs3QYwpBND1ABV9kJboykwnn1YEdxhaOh", - "jLjPWeO9/ojgI2p/fGmirjQEHZSUtlL/XsBCib73vCdGc6S6ac9hwqrJtHTAvbpjOf2TMa8Ullea1HdU", - "fiU4MlmvyvScB4i7jeeX5Y3mlytPrx3EN++JcwyvKDW+4K2j02MjMAScKUwZESgmCtscWwUXFxCHWp1W", - "V99RISYxuNqN/3u5d0uNCb4YjVVrxD1ayNtxJwbcmnjzd8YFIUQxZnRMpLLx5qWZ5RRvP9k7MFkxQjLe", - "fbLX6/XWjVF5kQelNNqKTePCXwhX6cnpt+3DHYSiNFnLl9bZ4ftfWwetzVSKzYgHONqUI8oOCv/O/pl/", - "gD/MP0eUeUNYGiVSoeOFBCrlJ019Z5nfDwqZxZDLXNbgialGnwHPBoib88YbKzzR+omhuG8NLL5x6pE8", - "/5UqpBwpOoQ2SD9CPy+3hDrBCNrYOVOmaJRnZlm0gd4ot45cmn5gIfVAQliWcCCKzF8BZzN9KnzZB0oM", - "3H37pvcD6+UyDKmHkv9mtT3jJAFRVavPW2sTJ8lqsvULihn/a5p1xcZGe26iB+f6N3ljK8/+dvI/f/y/", - "8uzpP7b+eP3x4//OXv3P8Rv6vx+js7ffFEG1PCz+QWPbby2cHR6WSjHtTUnpFKvAI1BNuVQ1GLZfkOLG", - "X7OHjkDxOxiwLnpNFRE4OkCDVsVFeNBCbXKNA2V6Ic6QHspGOmzozmfG/KM7f3G65dfqGKENaRB2Q7JI", - "JpmOQh5jyjYGbMDsWMgtRMKbvv4rRAFOVCqI3j0tw0ZzNBKQx9Kq5/nkHfQFJ8nXjQEDDZdcK6FXkGCh", - "sjwebgYgCguV8RmwzUnoAsONhjxg2b2UxYUbG00vM4KAbb7qcelHild94aIcirPf90XQg9eX3siISkXA", - "MTujbE1GmTsa2u+XWMV+f7+/UsDPaGgJ+cFJWExw64iywVkyBAxTG8YNHmoNbOmaN5kzgn59//5Mo0H/", - "7zlyA+W4yLbYKHnGB1AaG6GKZMH7b6PlM32b3W24IGMkg25Rg6ihF8Y99P3rc6SIiJ3DfjvQ6BzTQK8P", - "nv+plKkmRYrR4dHpi41egwy9gNsM/iX7+D5bYTW4wxrN6myBGcVr/HbQyTG459oTmgtw4FbzkgsUGQaT", - "n+sD9EGSsq8rbJV51Tc7Gc1zy5u5AQatDTdiUuUUB+hdJjfiDJRSVuCyMS8/lzCsfXgxPj8Lo1f8csGb", - "yepFlrWBhw9WmZO4vnHrWcHy4+/BOJx569ddsGmud7aLxlA9mZ808r2/7awpty/u7Kyr5K6b4aEchFkI", - "4M2SPDTPznAXWQ4WFb5rqoa1r/hIf7Zv9k6t+XiKpliyPyv4WFFutnaeNsrXqWdt+v5dfPnmYwNSdixd", - "RGf2bmtiWy9pFBl3CEknDEfoGWqfn7z67eT16w3URW/fnla3YlkP3/40SPbgzsarsw8QLoPl0D0h1XtN", - "4tzzmFxTqeRiwGujl9jlySV+LSWA8EYQb9xiVgj3fL2wjPvI9/CQfoHfX66JpdkhvjXFg5WW7yjDQy1z", - "9WVHKPNZ8/Pt5mq4E3BKwUM+/lAUKpzT9o2TI3Ra1OOweig1CyQhOjnLkyzmVi03fGVNNjn9Vr/f2+o3", - "sfHFOFgy9+nhUfPJ+9vGknGARwdBeEDG32BjtIRtpD8cXeG5RAMnnw9aRiEoaAKFY2tl+Ebvt4s5KG6W", - "cqIqUKxKKrFOEolm2SG+NSR/Warl83KS5cZC3pO/f1M+ZtL0arfOE7bXcB3zOUEBT6NQC1IjfXSNYkdC", - "q39KovL81XDaP7BLxq9YeenGiqoZwB8pEXP08fS0ZHMXZGzT8zZYODhd1OwDT9bahu0VsvZKaG6YqOE+", - "kjNU2W7hurv1VAxFo59z4jQU2sD4l4uf3od3yszWaDpZsqaK2SYks2Ga+qQq/cmFbnz4cHJcIg6M97b2", - "+/vPuvujrb3ubtjf6uKtnb3u9hPcH+8ET3dqEuQ3d7y5uS9N+TTXh0oB4sEEaiLhwgN93jJnmFGqUOYo", - "pw/ykRZPUUEONoFBYJU4YVRBEkjKJnoYMBJYMdlEeJo8lZRRBSkFIKENZXrJYI3Rg1j3pwP0CtrCJxxD", - "wJIDQitHZUMEDufGEKsZg5s6gX8tB/l8miott0EfOU0V0v+CZWs0WHVl+RCGxxygNxz6COelynhV7zHN", - "wSaw2LyqI7WtX5LzX4XJLMM8QC8zJpmxWctW25LYPw3vtq7V4Da+UXLeszve0tSS71zBL63TMhhtdVoO", - "UeC/tujJZuHyBmkUSdH3QkFwBCw09xRKFY1slgRYCZWKBkZrxLC5dSfZZgQj4dCIAHXvjcb9xIoJWSfH", - "KD6eojbEQ/4VWaVS/2sje5ssnsrd7We7z/aebj/baxT1kAO4msEfgXPUInAruX2QpENXe6Rm6UdnH+Du", - "0/eqTGNjJbBrLziZJoIHWlqlDOXFTPLJn/WeFYM9Qp6OooLVyUaGQURBk8ozNQ9sf9BoRsdj9sfn4HL7", - "H4LGW9d7cnvkVe6yifyS8EnR1LqgNpJR12Rx9PvjA0EJWRuy8o5IWAE6JwoB/XQRDuCSznyaLMm5wBaL", - "cS9h7e7s7Ow/fbLdiK4sdIWDMwT9dRHKUwtB4YhBS9R+d36ONgsEZ8Z0jp6QYIJZAc5/zpBN6Nwv+YBq", - "3WnHRyU18lJONXbsWVyL8o9WCLKLskgH16xMQFo45V5s7+z0n+4+2X/S7BhbjW0orpdzGJfUw6DH5kEp", - "7nwbzPPvD8+QHl2McVDWULa2d3af7D3dXwsqtRZUkMPH5N5YA7D9p3tPdne2t5rFXvlM8DaqsHRgy7zL", - "c+g8ROHZDQ8qFllvp+628AmehsDekSDCND4MnPtM5fYxOTaGwjTLN6HJxWCNBAsXV4O+jVS0SuEgIxpw", - "gVKWZXbqrTaH3sy6Wc+mzX2wmo0vytARZhpdNkjApHK8Ae4SQWaUp/IWBuKKBJqYxhHnYq2+df5I74hM", - "I2VMkFSij6d/BiaiiQtJRZKyr70lvyWhFDdc3FoHuEQTfqquQ1aj3Wiy9csW3Kk5pp1lfrSl418bsRRq", - "VpWy1W/fRzgKUkhehrP91KuC2AOeKnipnxsvkSjinKFgitmEQDJ4kyqRTRBGUx6FvZb/qSQKh2PvE0ZW", - "UpXnBTsdELqbKzjbfsXzknaGlCr5eZ/EhqvYzE29BsVTBcHS53mYRThpfGLFC2kATJeSNh/xiQQtUIH/", - "S6+afSbBwri1YGby1M1iozyWQ7e29W3vAbHCvX1XqLk6+dhqtFbGUDzDJA4ElxKRiE4gJ9rH0zKYyxwY", - "swKuq9+zy8A2IF2ZcCZ9T9umUm7jdJa+C9HjGfYtVyLQMDiAep8OjCeA9eVHMWYpZPoqEDK5Tqgw5NHs", - "cXzKpRpm4ShrAivVELI4pYLkMWvuvpxCAMDcsDho470XHWu7Cbqs28WNei9QlX+oOgDreaoXo35sdTIa", - "9JHxYkDO0higPKioGkGyTshYnvaHShiVFqKVUJtxVWJLhdQ1G00eqvw6qp6nrqTs693+edNoruXBW2dY", - "TU/YmHtyQ65h8Lcu8c53ISEippDHDIWEURI65TGz/FvbFjjZR5KgMCUWc0YgFdgiHJvjDTkgmTOKUTap", - "8PrqhE3M8AaG5UmeYF7bsMmTo/S7Zr8XKeDKOAlIhHMn7UYeD1QO/ZbixYEFmaQRFqgasbgEZDmPI8ou", - "m4wu5/GIRzRAukP1OWfMo4hfDfUn+QusZaPR6nSHYe5jWHmeMcBZD1OzIZV58yX8ole5UfFvB9PLpum/", - "CeX6m7zgev2GXtKI2KC+D4xeFwi9nAVld7tfF/pQM2gp6GExIHRdzm1J1nfiXazmYVY1weOObzyAKq8S", - "ZUNkab2+1YKL2bJAj0VTDGq7R2GXZaaM10K2l0aWkGZeblX3BwfNpiRBefbd/SdP9xqm2/kmW+eSqsrf", - "YNmcxUssmjU7ddrEbLb/ZP/Zs53dJ8+21zJQOU+Zmv2p85Yp7k+lOErFaPakD/+3FlDGV8YPUo2/TBmg", - "UqGTGwP0dcnRzcOsa549avN8R8WddO8sZQtoMxvjEmnpsCRyFWp5tcl4TECpHBq8dXNgKu75jWAIcIID", - "quYegwm+MinfsyaVcOEm1rQysB6U2rFtxgfNuWQ6yh06225y9BdjWq/Qwn7jrF0yHdWZ8d9WZzVG/NwG", - "VHwiavBCkxcWWDQXZOu5wrLk1aH/DiBnc16rreo/ZFo0r0rtaD0rTJ17RvpC3v1FqIvbX9nOgtm3JCRX", - "Mb7sCq0/gmvp0J4b2VemcrVXboU/2AvwZr2Go2I+vaUJC0vJ9/Jbd/15m1WZW+xnbrD15yu4gK7TsZpa", - "DOjRwmBRno/dKZFEDTUpLlZnlL6DBEHGp+BGKYKsO8K9ZAmyP99JZqCF7TgnyrU91xp9Gi1JCM0UETPs", - "MUy5IZBrUrajGk7cQdbEh7bijUqpwt2pX1azIbwN3ce0FDhMBBnT6yXUYhqY67rsPi4tBsJy0nCJ2jG+", - "RrtPUTDFQlZgZ3QyVdG8bGTd9URPfFsRZ6K06Nw8vXq+m67j4ouG3c7i6L4je16IdfCnfSfhcFmg+1HW", - "zNmMEzwH2bJWEXy6s9vv72z3bxTpflvZ6Avj1HmEFvpZY07p6bE4Qub/uZiy8EpQU9TMoUkqQXB8AN5U", - "CQ4IisgY4sCyVLErdfqFqZcDbx9JreN/Rv9uo1zlUmtnsSyOcQaChxvHJglwy2i5V9pyrEfx+yLYS4LF", - "MjYTLESNVf1X97r9nW5/7/3WzsGTvYOtrbsIjc+QVOfC8/Tz1tXTaBuPd6P9+dM/tqZPJ9vxjjdc4A4K", - "H1TqCFbqINg1JERUc1FWc7hKElFGujJze1vtgLyEF5iXpJXnfz3rg1nBUmHhvLzIosyAVY6cakm2+whs", - "stAvNaFUwT85Xg72jfzIqoD4CawKCtBTM2AgVcvWt+YFSVnDe+dDoWHjm2epb+Oqu8fn9A1H27vLNRj3", - "0XOJMZZO2LIbe/FW86hwEy6omsbLr4esWZZlAB7DP0sVlgNpeuhkwiDzbPHn7O2jWBxad251WtHn3fKZ", - "sb83D6myUfUZAdqtLooBDd4GILHxcixAk1y1EMY9AQsCiPhlq7v1DF7oo8+7v/S7z3robwVPgY7BVhF9", - "W6516dd+ExxmjBLkTvNyvvVsrWd0h89lFPSbvZfqLmIbb29pPE/76e4K5zZd2uD888IeV6KK7qjQ0Ncl", - "K3aC83ppe1wvj2jS88smz973t9ZO27PeFdH7Bo/ib1L1mql3EZaqTq5+jaXK9DEkUitdd6DEDatUurdp", - "xW0oALh2Qqz9AZIVDd/lU5e50OOsAB004QrlQQArpRwAX6TMSw9l+PMyxZAeopYgtlcQRDOYskA+uuzo", - "nhyjRPAwDXIP2AiAToOASDlOoepyr6lEu/qN8S6VeXAt1xr9amW+TntfHWZKruv3+w25VoUpNcHWb/VW", - "f/VW34kFoNNKk3A1DzONmnGwtTJxrPCp9NgjymivSEGFxXxqwNHfFTG4qOBBwSYUaHEgTVwFQU1Ti5Qk", - "PXUD8fXQmyPgmEREEd8gyJT0tG4fVOZcdDVL3drb95vM8PUwgIDEBUB+IyTRcnrMpS2xGWM29wJWzaKN", - "2n1XQVIiGL5rMnlZbJWBe7pSCqndquYJySsWXRPxVcz/nmXouN1s5LbnyloRdyeovLeqUl1GmaqvZ9Hu", - "eNj9u7EzomHvYPOXv/7f3U9/+ZO/NktJkZJEdEMyhgewSzLvmqKzWmnrlfOZQrabllTYVjRRBMdgRQgu", - "ibFaxPi6CO+TfnaS5m9wvLAEeDmMKcv+vXJBf/1T/btbAY0fgHms3Mdvzn50F6llFXc8uh0TMXH52527", - "GJTzZtFcb5VEhfx19p53x/rPMutSLBx7YWSjHpQ7HlFIAyoHTKs5OAhIokjYs3m8KMAiOBzJavlkm0fP", - "uXfrw4oheagNt6mkyfriLVJ80GLkqmtmCLua9naf7NmK8UVMbi1ssW/TTcR2XYVFjWWPGeE1lRCO4Lxu", - "C41Rm8SJmrsssc4vcmO9CPLDbEDvU+gtp8/qP7uNbKEflqYH/QHrexYD/B1AK0P7F/a/Nief37HquJqp", - "x5xJW7OsnFmmojJJ1a33u4r1HT8E98FFHyn9zbgm2nyYk7SaGHwzZmrTZt/1hUOEUJJ4qTNqfspctHsX", - "Oq32sVwqZRZWVoCkfm9OnTBVLRNdj6AzjZqrKRGksBHQIU8huibKrKNggyAbkyMzIaJbLWpn6i4ICp6H", - "mR7sUJA5ky4axpZnuDnF19kMYFTFcuHpAdaR53rbevUcaqm8c8XN6NgNAWBURF1/upoyFTUpz764GUWq", - "Wly3ae89eJZXLeF+dWerQpz5HCXS9NHj3zBVL7kA4bg+pOXOs96A4B0SATG91Zw2jRLC0JiEQ56q5eff", - "Jpm38SwhGpExF6SQf9cpAhiI2NY8XsELXNBFDsMnn9wgSZAKquZac7Qi6YhgQcRhag48IBImgp/ziSGd", - "7devYEIbezzYXhFGBA3Q4dkJnEeo268Px8dTFNExCeaBVsAhG+lC/g0Q8t4enVjly2V8gwdFqoD0XBni", - "w7MTqGoqjALS6ve2e304zAlhOKGtg9ZObwtqvGqCgyVuQvZ7+NM6p5u4NMrZSWjloOemie4lcEwUEbJ1", - "8LvHyVsRYbLpS5A68aSgNySYCqs4JBG4nhtSobovpD9yV+mBuY87BuGNbUdSza0jHkne2m39pCnBnBpY", - "4na/b/Q0puzFi/Nql5v/sAF7+byN5DlAjycX0IJc72RKi/KvndZuf2steFYWqPRN+4HhVE25oJ8JgPlk", - "TSTcaNITZryDkckzYf0fiucMSKh4wn7/pPdLpnGMxdyhK8dVwmWdMEwkwlAiz5Ry+Acf9ZC1jEP6Uznl", - "aaS5CTKuzyTUFxbWPKU3+YywCKZ0RgbM3tOm2CgWkF86Rvp+NmpL+WiYqc3uZ0Fpz3k4r2A3G25TD9d1", - "9tAcwdU0i5IMIVvUsK5OS24JpYxBvUdJbE7KrGDBopMFFOiVAfdWJiYMM5XXezWVeS/J3BpbvQM2Suui", - "GR5sC4FC8Fm+8u0NfzwDZM/0hwIdZ9+QRW9ZnGDwQhFEaZjLXM7FFosRjiJv3P8k4iMc2QLGl8Qjor6C", - "FhYpxUSjTrhhPCQmaWQyV1POzN/pKGUqNX+PBL+SRGgRyGaftri21Tst6UIleRpDBmhT20LPuWlA3Pxy", - "SeZfewN2GMaubom0xecjyW1lZ5M+h0rkfDIN7frTm9Y89x+lUvHYkhQrFqI0YPJUJamyT52SKJsyG5pD", - "nVI5JeGAKY6+CFOWfv5180s+41fQXQgONZ0UmpglbX6h4dc6qOUQ69UPoalH+yOAgEFL3y6Dlv57IrDW", - "XVI5BVOGBPPFpLil7SwWW8uFG1UMB5ihhCcmjh2IyhSsLo0B5QdwFCEFR8n11dIm7GTNemxoiq+Wno1L", - "MYEElWMEVfUKh6m/u+8/T5IEgvgMHP9z/vYNgqtK74FplpuNAEeU6VsUhSlI8jB7b8Be4GCKjNwEucoG", - "LRoOWpl2EW4ArKm0jrPdLoi4v2jQfjHTdGj4S6+nhzLS8wH6/YsZ5UCfpSQeKn5J2KD1tYMKHyZUTdNR", - "9u2TH6F17v3nJUaA2ob3b7jiMZBmIL8Gzb2BWYi45bXRHGGUc6CiHWVEGRZLK994UG8xqFV5PJFFZHwZ", - "gAV10DoYOBvqoNUZtAibwW/W0DpoffVjwArR9YmxTPEfJ2tnRLTX72+sjruz+PWI0KWG+vh9XZC+tm9N", - "8LBC16LgYRbnsvrpHTRlnIy4dQ+Sz3McusIAP0W8FSKetVwUhDfoX7wHDPlGxCi4FQlM67ORk8CWaieG", - "LCCtJWgcLkrWKBzUSXA58RbVj6o6v6hW7NadsgBAjBz97d4D/cG8eSl0mPfZfc2LI8hRmRUGflzkCJvl", - "CLHj14hfEfU9UFz/vlipTaj5kPT7WOjnFbFyX460CjfbJDP33uTPBQAxANKOYhprXfUcYOqeE6bQC/i1", - "Z//XaTyQ2fYi4pOLA2RQGPEJiiizr3GF1yJ9KVpcQicTBpD1s1EBLhFT29yf//7nvwAoyib//ue/tDRt", - "/oLjvmmyY0Di1ospwUKNCFYXB+g3QpIujuiMuMVAakUyI2KOdvogZiYCPnmKTcoBG7B3RKWCFV4tTU4k", - "aQcE1YPBeihLibRhFLohHduEDcbA7FHh3Vk2qLzXE91Z9Dk1KygsQN+KjgYgApea7LVW/2r5rWdmzSX7", - "WdVWvmAxXc1fFLlWhnq7BsA1GQyg2Hfu4INdNGqfn7/Y6CHQMQxVQFIOkJjzYazw3PvJk1bzJMNRygwF", - "sGx4U6HOeK3999i2aWYAtiP+SBbgusLp9SZgY/IggoQOXz91hSbmYD/enGnYZ589dsFz9Qbam6+3OIXz", - "JmqkCN/ePjvaW8S5+VJA2UOowKjtHLVdzb+zoxNXG2bjwYj+Xm4NvVJbUSG7OhA3lQbvTS074mwc0UCh", - "roMFkvrHJFPVygTyWNjBOws1wm5d1fR3xftts5TNpfamyxK75Ffe3d8elUnXuUbyFH05rf28SVaRzjGV", - "Add9C9TSDXBiCxYa8SU7p0UqWmWQMn7f2ZWzVFyy7Pnk2B3I+zNN2alTVr0b7oEpHlcY4gMywkoRtkJS", - "y8dEzR+yXXSJApZYrr4v0uzfnxR031YsH5k/JjNWWEGb5oImtW/tBfqKqF9NizvcaDuDZ+HnRLhT7XIQ", - "w6qzZZmuKJiS4NIsCB6kl+u+J6ZJM9XXjPcjab6AnnUkFovynyJKA2U3x9UyBffEFpa7O/0WZlhLvb29", - "d15LYB4kg7PJyFmsTc02LOcs2Pihnnrv5TYzyH6Ul9lZGkXuxWNGhEJZ/ebiHbD5BdySVsv27rQtvQ4+", - "vHvdJSzg4IeW+VD5hSj75ZYlfLNhZik/yaSJTmgidqm7z+oknG/Yf+MuiLL64P+1/dJWCP+v7ZemRvh/", - "7RyaKuEbd0Ys/ftizfctcT9i4tMCNy0jDVgTg1qhqyTUrFVDIdW1/6HkVLPotSTVDK8/hdUmwmoRXUvl", - "VbsVdyqxmjke6EkmIzYftuGT80/8wSTV+7XyWYp0+XupLD972AItXICdFz5RhlJJHqEDJc0ornhtNDRX", - "5wdy6fXhSPfkuAOI7GjUQUIhGyByT8ZrB8e9C7d23vu3XB/GIzpJeSqLsScxVsGUSBusFJEyA35sYnd+", - "PdcK3t8xlfbv8+q4d7n6J93fkcRf3VDDvM0L1CqZ37VqKvPb9lrmt8XzTezaO1eU3+ZJ2qhxKnRB1E3J", - "uBRrvujs6IPLp4ugD1pRydUFBBrEwYD9H61//K4Ijj/94oJk0n5/ew9+J2z26RcXJ8NOHakQpgS1yTsP", - "3xzDs98Eos8hv2cekleFwxQEANJzCWz+4xSk/OWzuYbkqPCnhtRIQyqga7mGZPfiblWkchKse9eRHL35", - "EG6TmPzUku5DS5LpeEwDSpjK62UtOInZcnuPMLaM2fehgnNH6aJtrCVlh3KFAJpna793x55s8vtXjlxi", - "+MfpI89NVEzo1JH8MqzXR743eujfL3O+fz3kMZOYEfgXUZdomdJXhhEyPcapAqfEPEMIeH0iYaT2bMQe", - "yqsfyjRJuFDSZIsEARjKYampFoB9mSXLySJ92SEhMS4lsjNgkEBefzax/JuXZG5yQVKeV/XPVmrzP/pi", - "r8q5OB/0GN2+jOVPNNpIxrrnY2zzKT+cjPVgrONeJK2TUpr6dnYwQKEckewk8yy4j36mbLLxqDxQDbPK", - "1lbIZ+QRtTbHtiChXwV6ycVlU6bgKZDzCHhDcYXfofalwYP8SQ+vhIF6Ys6PJpp75xsLVY8e0mmdVjlJ", - "EKWhZh2OhbjLdyx4PLQ/mgyf+lTY/Img1AV21IdmMnr2e1Cx33CFaJxERMs9JERdQ016N62w5NJkU1mo", - "EbYeE9THphhCYNJ3SVdnxGYTh+cIt2FteJlc3C4v14z4ZHXagGxyFyPvyRswYCaNN3E5vy9QxmSR4kiS", - "iAQKXU1pMIUcAvo3U5oQwvtxklxkSYM2DtArOKnF3EkweVsSoUXHgDPJI2JSA8zi+OJgMcflx9NT6GTS", - "B5hslhcHyOW1zC4IqVsVcwJkBUze2EwHbU1JgkeR2dELLWcX1rdhswXkSZ0GzJc5gJErOyAdo4tCEoGL", - "miwCjqG+5hP5UKJspz4Vn1mL4kgA4gxtEha26kzXNPLnD9jqe4s7NMxlYMC441QGC8C85pMsDWCJlHGS", - "NCVfCyZQ8SyOl9AwaheqDEoV8lT9VaqQCAGdLXXXETdq48D8Q+FLTai2SkhWpxHIz/tAY/JyeVGlmWqh", - "JIb51yyOW52WhaeQz2sNC8OKnBDVARcfEvTOFBI//LQlrJPSoczsCzkdKjeHrWNdL3Lb8tw/vEXLIir8", - "EfTS8gtADgVlTlSBveV5SZ1HFRtuKrdXZTFT0MV3Rtwqu7JQCbDZg8BCDcHvQGld9U6QFYTLqtXd94PB", - "IgSPOWxALqwGKs2ztV4SvntCur0tWVhqEwr5SZvrPzk0IswkXVIYEOoaSoRdsTzIhBtMOZcFsh+RKZ5R", - "LmzOalszKaNMMFkY7dH6G11oUr2wdbMurHh+YG1NCBc/2Tl60N16Kfl7uE95j5cFbTvj+B0nUkPePIkw", - "GglKxijBqSRaWkpjgkxNBpv6mOBg6sre9gbs/ZQgW+yuYEDIaqNSiS624osOGqUKRVhMQNsxH43vkSAB", - "j2PCQlPAcsCmBM+oVtUEirAiLJh3JYGCpjOSl3zQqrt900HwpJOVTOwgV2kTDAwXhTqaFygRBIjIqMus", - "VLRywETK/tvk+tPDXjhALxCRCo8iKqdZdv0Ah4QF3kR65983G7t9I+45UYulJh/kledGvPQhn32Ktsys", - "2O938SL0yFxbuHAVARuw+SVCr6xXDcu+Yud5ec3/wCNt1urW+EAvMxmKl53i7+NJplRf++ezjLJHMkzN", - "dKRcg/qHfWvJGApKWem5xdpkb/rgkuWOz9C8Fs/b/OL+PLmBjew74YSdJZXe/RPki/4eWK7F6o147gMZ", - "B60tqWAVe0AWbIF6OPGJiwKX+y7YsDlwGTcu8hwlMOhUpir+T2Zcevs27gE3ZcbO4rrwAF5gz5R1kwjX", - "8eW82rafAVdqsP+H+QvWVJj/ajnhQzK+/EXg3pjdScbeDMNL8Dzi+Ed/lwm4ECYEzhZwfTwpmAq2wMID", - "Uxssbp2MQ3Sc//3H09ONOi4h1FIeIdQj5hDlQpBB7Klv93ZGhKChK7Z3dHpsS/NRiUTKeuhtTKEC3iUh", - "CZTWoDyVpix/r1ihfrFsWKUEPWFKzBNOmVoJRd70boD5eqNiY/fMJ20Suh/+8Ris8I+PSQHv0OKKXcBy", - "LVJhVeuM55zTKDMVArW0hUc81aMvVNBHYxoROZeKxMYzb5xGcIggTamtYmP7mRi8DqJKIn0eOhCzlBAR", - "UykpZ3LAbMHshAg9t+4O5VJzJyOv8V7hjGueGdb3fTiwQVF98NnCqg5r5Xr6OElcPX2fk5QF7xtAegke", - "aUjO4xGPaIAiyi4lakf00igdaCZRpP/YWOrSNoR+t12j5+YnS2P6hI25t4yBodmMmH+MsI0yW3OPiI+O", - "rb0ixcPi+A9stJ+tyZV8TRAcdRWNSRYujFJFI/rZsDo9CJWKBqb2cx6sBmVrbbzagJ0SJXQbLAgKeBSR", - "QDnjymYieLA5SPv9nSChkNdhhwBwwPDqP8cw49HZB2hnSut2Bkz/AwZ+f3hmXmLH2NoICoAyoq64uEQn", - "m29XOPmeA5r+g73kzAKXRo15N/zn8936saC1Z0jWHFGeLFOAePLDu3FaCe6nteBxWgsgGD9bTXsicABC", - "sZymKuRXzG8ZmPEojfU/zB8nq1I6KBxMP0LT70baNeCsnMYt8FEcSrumkJgyKw/yQGEQ9lj9SzXi3BJA", - "iCl57nlvgUP1I1L37Rvli3j8Dp8mLUZdCaPv5mzd981nYXCZior4eCzH3FCaW4niy61PV5jWW5+eRzy4", - "lChlikZgMClImhgyJ+of80x39uEPxASIjnTFlxG5TqiAnB9g0y2MRPSKJcJIERFThqNNWLMZBHL2OSsW", - "nnEKQcpBRCFMjIYEJTyKIC/J1ZQwpFcDhio3QOGdVtqc+cU2xSdGxdGIBDwmLo/hhk91+xum6iUX5aSE", - "3wtffF/Av16PXqpe54o8jPUzflNexlN8DW7NYWqfiR1E7Vc8/9GYgjoI9mbQ2unLQauDBq3teNDSO3CE", - "wYSKFXqCYspSRWQPHRv7FoSh7vWRJAFnoXTpFJ0Fb6cv64JSDVnWRDjuQb/7FHssVQEq39lJfOxBt0O6", - "PwTYoHbxwNkzGXbg0IWIpwocuN25sq1CosA8snHvL7CFM/JTt2/Cyf9mj2+JR8Eua3ZZ2HrD2bOEeyut", - "bi6oYsplnqcPBTjBAVXzDsJRxIPcepDK7HWgm4EyEgRfah2qN2DvslR/NhACHZ196DijGQqpvDQjWLtY", - "D72dESHTUQYcAm5gLHiwGSQcMMVRgKMgjTTdkvGYBBDDENGYKlljV8tAucvCcfkkno13Hy3qHpsxyU8T", - "sHs5WcgKxW2ard4UJIgwjYtGpSpyQPSFJ10w+470oFxfw+PIPm8FgkuJ7FBdEtEJHUX2sUb20HstcuCY", - "DFgSYcaIQKk0fkca9G4iiJSpCYzRA0BlTkNRHZQnOkkEV9ZMHHEupLHsagr/eIqkIskSMntnRj6FNd9R", - "YlUzuJ3pgRSGCgz115JtgvSGGEoxCNd0pK/pB3D2MQA9dALWx3Lw3ws6mRChTwU2TNY8jZpj7dBpDn0p", - "0qM2q/h51qpZVvFs1II3d8HTeWmiiqFrOAQBep0XWM/kl7Q2l4n9tF70xW+6U8O5y17+fiDsp29c5Y9S", - "rOm84FzdNBd5TuGPLS14AfLSUS0FKKxOR9A4IuEuIwQa5x14sHQDjznLAC6FHdSlE/j+CKF/v9Fx952Y", - "+HHTVilLQKkUSU2o1Or0nd8FBd5N3s4Hjg69Qd7O7ypeCfIuPlzc6HcVqVSyA7pyCz98Zs67ClAy6Tkh", - "jUVdgJLhetaRYKmi9NG2aaYm2RF/JAnevj2vIb87tP/U+huoDAVk+U12Jjba5W0hcaLm7nGRjysPgJJ+", - "hmAMX+KHzIfg7vIt3OB5/fbIw9Fp7eP6zwpE9/Z+n5dpPTl+/GWHimeudLFs6luni0UwpTNSb3Qvn2CL", - "okSQbsITeFwJDcIsPtxdprDoTT4jO7zNVWX/hahLcUxCFFJBAhXNEWWKA0cwc/xZIsG1JgDfuZj7jOnF", - "k/tS8PjQrmbFfWjPlDWG5W++8bwbYoW7M8dtlpjQvuGl3b1ta4aHKEOvnqM2uVbCZNxFY635IDrOUEqu", - "A0JCCTS5UQR4q19j2aSfyXAyagLlktzJb21uahSkUvHY7f3JMWrjVPHuhDC9F1rUH4Mkmwg+o6Ep3Zgj", - "dcYjg9WtGoSua3fVQoUN78uVCwPcg8gwTS6kyWealNmCcV1oHbRGlGEAbmWW4vKZMgFVej5MIawhPzuO", - "clo/rzCr+bWdsqMpUSs5DomKc5Mab+PnNfeYr7miY6q700q3XbPies18VRu6kN5FwtzMj/l+zdYfvx/3", - "SiofpWelNZ3PMoW0zmz+fZFg//7uh/s2l398xO74r4hTvgumchhAj+gjmNc8wBEKyYxEPIG6e6Ztq9NK", - "RdQ6aE2VSg42NyPdbsqlOtjv7/dbXz99/f8DAAD//zlim4+EZgEA", + "H4sIAAAAAAAC/+y9/XIbOZI4+CqIutkYaYakqA/LtjY6fqeWbLe2rbbOsj232/RRYBVIolUFVAMoSrTD", + "/+4DzCPOk1wgAdQniizJlmyNvTsxI7PwmchMZCby42MQ8iTljDAlg4OPgQznJMHw56FSOJy/43GWkNfk", + "z4xIpX9OBU+JUJRAo4RnTI1TrOb6XxGRoaCpopwFB8EZVnN0NSeCoAWMguScZ3GEJgRBPxIFvYBc4ySN", + "SXAQbCVMbUVY4aAXqGWqf5JKUDYLPvUCQXDEWbw000xxFqvgYIpjSXq1aU/10AhLpLv0oU8+3oTzmGAW", + "fIIR/8yoIFFw8Ht5G+/zxnzyBwmVnvwwU/xcYRZNlmc8puGyudmXlGXXMBvCmeIJVjRE0vRBKXRCEyxJ", + "hDhDOFR0QRBlE56xCL05OkMhZ4yEejA5YnwiiViQCE0FT5CaEzTnUkEbJXB4iRSexGQwYkGvdh6E6S/R", + "eij9Y07UnAjPYqlEdhQ05QKpOZWIMv01JIPygSmRkSZkewGNYjJWNCE8U01A/cKvUMzZDLblxkVJJhWa", + "4wVBH4jg6M8Mx3S6pGzWDqQJmXJB0C/LlCSYoTTGIZGIKkSZ4m43BkYFjj1KfMhFZ4wLMo6IVJRhPf44", + "5cJQRHX1r+APHKNSW1gatEdqjpXDcsYVuiQkrW4UX+HLKhh/39npPR0Oh+97AVUkMWSFr2mSJcHB/qNH", + "u496QUKZ+fd2vnrKFJkRoZdvf8FC4GVpO5JnIiTjkEZi1U7CmBKm0NHJ8etbbiDYHg7g/7eeBL1g++nO", + "YHv/Cfx7ez8ob6sB+OrKP60mvXOFVSabPMhQ09giyriEJM1d/5YlEyIQn6IwE4IwFS8RkBSJOiBdZdtD", + "31GEnE3pLBOOBH0kVwHnHEuEmWEa/Rq/KAbrRHehZmIRv2JjQRJMmYZxYxGv3SekKRRZItJLCjlTgsex", + "ZgpKkSRV0lFRT7NxhnCaxjQE1lMhqr1kKINewLI41h9rKyxOm8R0RqFBJ9BQWTok1xcpjghTROQU3gU0", + "FbbYNnEBbu9pFHyxOxeUlIX+7bI6zBPN4QUJzXbzG6ACkQkJeUKQHrp6AjvDnf3+cK8/3H+z/fhguHcw", + "fPQ/QS+YcpFgFRwEEVakrw+8yzGt5t9HBZR0Q2QbFleVB3aDGg/uhi4xliqnaiByqpZj7FnTG5oQqXCS", + "asLWaygBs42s3YD1c3CQXwng7c8CMCPXamwh5N2PDz/IdUpCfcVwR575ja3H6yE6RRjlPECjq2GMKzfy", + "9LM2IgiWesFa7tC30+9BxmSW6ruQROM0xkqPq4UUQINxQqXUXfMfIioNYfYCh+RjxtVYZIyZhoyoKy4u", + "yy3tKGOaBr1gjuV4MUuzoLfqHqgiNUxBYpxKGM+euBgTIbgIjKy5HE+5cIekL7EChCuGakBI5neWB0JB", + "L6gAIOePbi9u3fmpehcHswAuCSOmG7kaNtNceHms5nLzpa3mlIYtG6nUHTOynWWVA0QUzxiXioayE9+E", + "21gfb8IjD+s8zodDNCJM0SklwgqqBImMwbXmBkF6EEQZymSNDnJZekwWWvkZL/bGKkybQKlpCuXDK132", + "xRVTuuby488pZQ2SVvfu1UQWmAJNHpMFNVdLVRiyRzOOBF0Q4WHf+Y1qWKFphzY0rWsWwjgjmxVIsQWN", + "KO7CDiJY05h6sOfs6ASZz+jkGG3MyXV1kp3HkydB+5AMJx5c+CVLMOtrgtDLcuND2/LYL/e8Mj9Pkmw8", + "EzxLmyOfvDo9fYvgI2IgMpZHfLLjE/3SkI5xFAkipX//7mN5bcPhcHiAdw6Gw8HQt8oFYREXrSA1n/0g", + "3R5GZMWQnUBqx2+A9Ld3J8cnh+iIi5QLUILWEk4ZPOV9ldGmeio+/P85o3HUxPqJ/pmIcX6J+AB24sSo", + "k2MnJ9h+6N0p2tA8JCKTbDajbLbZBd9DrsGhrzrfJQ5LRbaNVhOVk1Jufd+GguA10+kWnSZrklpmTnKc", + "yLbRXRPNURMax1SSkLNIluegTO3vtW+mRDDmhmpM9Uz/jBIiJZ4RtAEmFVA/DDPVgs0U05hEm92E2bbN", + "/MEnpSukgt6AFn08Cbd3dr28I8EzMo7ozNrE6leU/l2jmB5HIWjt3whc5t32AVMKMm3O9xxYN0wiyJQI", + "onH8M6dLBV8Qhq328heYN/i/tgpj4Za1FG4BMM+K5p96wZ8Zycg45ZKaFTY4l/2i0QhAjaCHf83wadVZ", + "lzBKKixW0we0+AKUWMh1a2FjzRZatMGztV3e6DZ13gmsMZclSlyglUU+00KNRzrgTNkPNfMln6GYMqNx", + "aNHOnAXIVcuU/BRzYIlfCA45+JvEr9d9C+ZlfmgZTX/r5QJ4zGdlaM4JFmpCKsBsucLsQMXqWsF/ViGf", + "2l2FJRmv5iBnlDESgb3YErZpqcVYr5oBVHRJ1XhBhPTSHCzrV6qQbdE6VMzDyymNyXiO5dwa2KKIGmPh", + "WWUnHmmtYojHoI+7AUGKAP31/JfDnUf7yE7ggaG1XOoGzZ2UeuvhTVuksJjgOPbiRju63fyObmKIHwMK", + "Y2Xb3ZNjoENMw+kCe5pWT87k3PwFvFuvCu4+zQY0esX67/eeTR8BkzBaQuvrjV8GzC3Ds5hrmC5Rxuif", + "WUXAHqCTKRiI9UVBIxL1EIYPYHfQ+t+MMCI0nyosQyUhGG2QwWzQQyMtF/a1FNzHO/3hsD8cBVUxNt7r", + "G/U+xUoRoRf4//2O+x8O+/8z7D99X/w5HvTf//0vPgToKpk7qdDuc8PRfg+5xZbF9fpC14nyt+b+5eX7", + "OI456hPNJ2560kcnTcHB7DXi4SURA8q3YjoRWCy32Iyy64MYKyJVdeer235RWMA+VgCBzTSYbgiGmtID", + "aLwR8ysiQs2BY6IRT/Y0E6ZK9hDWejMwL6Rvyf9EIWaaFoxwwQUiLEJXVM0RhnZVaCXLPk5pn5qlBr0g", + "wdcvCZupeXCwv9vAc43kG/aP/vu/uZ82/48X1UUWEw+Sv+aZomyG4HP5Wc+tIX+iWXUiDrpZDGJeQtmJ", + "6bbdfIP6vBN2G1l10kaZaz1qzYRyE9mahTTfd7WylXhUh1cLIgSN3LV8dHqMNmJ6SSy9IJExNMqGw90Q", + "GsCfxP4S8iTBLDK/bQ7Qq4QqfR1mxS1vnmxrr2sknHMQVOKY3+Q5DSRFUHBwvPIeXwUaL7SP8nGbt/4v", + "XKp+ghmeEVBHbUM0EfyS6IWaNwFKJLokSy3lLNFMD9pfUAkvPIQt0AIbq8NgxN7MuSSmifskwbZPFwQl", + "PLw0T79zDpr8AscZkT10NdciB9gECY7tz8g8jI3YXC9ShjwlkVZCTDPYGrogbHGBEpwCmWNBgMZRghUR", + "FMf0g3nCh1cGElF9w40YAcJAKdY0H4ZcRPDCxhHB4bwEhb9KdGEElgsY/oIyjdYXhjBrj9Ufg1dv3/z8", + "6u1vx+NXZ89+OzwZ//rsv/XPplNw8PvHwLhq5JLKzwQLItBfPsJ+PxnxNiIiOAgOMzXngn4w1ppPvUDD", + "QGr8wikd8JQwTAchT4Je8LfyP99/eu8EMmPGXmgy8Czsk1cYMnephyUdO2ugRNbC5N42NMg0i3px9nZL", + "384pllLNBc9m8yphWNHgRiQRUXk5pnw8SX1rovISnWy9QlpwQTHVBJoLKtvD4enPW3IU6H88cv/YHKBj", + "Q7WwfM2DuLDyk5xr9Mm9Po7O3iIcxzy0NpRp2wOvm8rH4AlTYply6lPiasypaNrkUf1+8fUGrGhrQtmW", + "1MfQD28Gd8CbW6sSz9iCCs4Src4tsKD6npZVWvnt1fGz8bPf3gUH+iKIstBaJc9evX4THAS7w+Ew8CGo", + "xqA1PPDF2Vvz6mnIRqVxNhtL+sEjShzm+0MJSbgwKrTtgzbmVUnD0C2CwxkFuy9+Nsi1/QLwyh2KfSPK", + "RzED1571Xvzsw5b5MiViQaXPzvZL/s2dfNPdp4Lb5pUsR1rA4kFJfwljnkX90pS9YEoFCcG9Qv/rT5Jo", + "QX7xofos5ennN391EmDXSKY4TikjK0TTb0REvOLiMuY46m9/YQnRPqh6XGPMh+r55i9rDiUaHmcTzKIr", + "Gqn5OOJXTC/Zw1ftF5Q3zpnrtd4Jjv/1v/98d1roWdsvJqnltNs7jz6T09Z4qx7aa0PJN5Kl/m28Tf2b", + "eHf6r//9p9vJ192EEURuJdTZ839mRqg7zVhfQmMObXkZzm/v3GFFcatQQ3fkcG/tM7CPUfMFETFelhiv", + "XVOwPQTuV1uVoOAliWw/zUYvke68hg3r0dwl/6Ku5O8M/YzWsyjPmn7WvMLeC11Wki9ke+fU/rnTXFLL", + "ii5pOgapeYxnuc13lUvo+SVNrSgOPcwxxrFhBFEGwvuEczUYMeOhos8ODphckxB4nlRYocOzE4muaByD", + "hQiYSvNq0YJ9ybUJmkul/1tkrIcmmdLSOlcEWb0JJslgLdB4QlDGsHsPr8nOdoNN9wIAyyURjMRjIxvL", + "jpAxnZDt1Aoc2OoUS+uiJlSWVuF1/OvpOdo4XjKc0BD9akY95VEWE3RuvAs2q9DrjVgqwE1BT6Lpmdp5", + "+RTxTPX5tK8EIW6JCQyW29jsY+3ixdlb+9wvNwcj9ppowBIWWUdfd+NYJ9CIs79qiiVRddjy/DWgt7l0", + "SIZTOedqnObO06u407ltXqji3Y0JvWARpln1SHd6rU6gCypUhmPNayvipPeB3zixe9QG4yNfVl8s3yuc", + "ZlX1ZbarxcWMDB7tXndZj+HESEqdDSclVb5hQnF65sdui10z/glzC1lpOCpUzc+Y69wM0nDeMT/33M5u", + "AaWTHCY1c9OXAc+hLKnmnZzPjQ+WkQgl2rjQ2rzFY62/X/TQxd8qP2jad6qFli+ukIEG8BOmfyqPXzdK", + "rDUX3Mjdu3w4WN7+PA5lq6cTWmwjJTCTxkdtjlMyQL8AE0eKJKnmZGyGqES5axdi/Oo/ETdCjes6Ynpp", + "0viJWHDkRiNJZ4yy2aYW8/XFhKPIWJammcqEbregsoBmFXWc9abh1WpWRww/hggJysI4iwi6cBaei6pc", + "2LT/NFVCaxBqaDgGJKDZgLKntpJM6en1hhOswrmGE8+UcRyzW6869dWsTOseVO1a8qe2W5z/ec4u6oEw", + "C4+KozdnH3nALFiyT7aZAa2g4jdRXpIlHLkzR+KGQbJsifTbCwWRPF4Qe+2WbZkTCPXhRnAqzJjGIGlt", + "kJr860EuPuvcuqPQ8OoM/qqq4AnxkarvNltgjJX+nU+440J6c2a+nlaMJQHgg+pxgEAcu+gZXYmABQIx", + "jSwxiqggoWoMT9lsxMCH5ML+MrCjXWgi1zLKFwmcgjgEENrLR4tKJ+vEPhhGb40nVCkS9aqywSUhqVy/", + "KS1eW8O1x7ouyJWgjpE5p+KO4hlhUy5Cklgl4fMUx2elwbxq3M2GaLp0GPiW1uziMyA6hUTGf8icB5hZ", + "K2Eb9ejFqKa1GReC6pQXOI4v0IZttIkE+QM88e1ZMc4KZH9zdOZQIH/2fnfa0xipucDFXKl0rP9LjjUV", + "X9QHs30dhReRZU+GoF/t7e3aU7VGN7Pg2rBV+5rXLaL9aJz43fqypvFCr9L6mXQR5Y+KLoUl9ZKyqOsA", + "v+q2rda5XDBymsZdG+hSQfpZOhMYXGy/pHnu1u+mAM12Dr4mjtfnJllECGZS8aTsb79Rc/GgVWeQKrAW", + "PO5HWGEwZXa0t5rlNh2Pk6UZyuhibZaY8Wzi8RuiHyAUYEZneLJU1feDbW803+c+Yru1+I6lzYHfaJAk", + "Giu+2oWZTpFr28Vj0cQbKD5eTClfHd5h/V8q8XfmOrJ6rR6in4bUmhNAxgnnxsPUAAGExnen5be7wYj1", + "4fo9QMf5BPmw+ZAYZEscmZeTDS5KizCBHGiy3EQYvTsdoDf5av8qkVZYFsRFNMyxRBNCGMrA9Ay3Yd/c", + "xeUFZBIuTVXvbm0nJvhhE54ouf02yGOOwUqTR1CDq9SE1vZjIifhoOybMGZlK1gnq9Uqx+/XZEalEjW3", + "b7Tx+vnR7u7u07r9cudRf7jd3370Znt4MNT/+Z/uHuJfPr7DN9ZhlbdY57My9zl6e3K8Y42l1XnUhz38", + "9Mn1NVZP9+mVfPohmYjZH7v4XiJA/KzsuPCaQxuZJKLv2KTGKp+vXMklrcUX7tYubnfksVY44K5qayDx", + "Rre8i9AWn9O0ddm9efBJnWGudbsuba6pyS9T0DsLKilJcNa7MaReP85jKi9/FgRfQshe895O8IzIsbnP", + "/P4MmTRONuTaWjcE52oqzbtp1eq5vfd478nu/t6T4dAT0dFEeB7ScahvoE4LeHV0gmK8JAJBH7QBD14R", + "msR8UkX0R7v7Tx4Pn27vdF2HeeLpBodc8XK90IaFyN9dnhL3pbKonZ3H+7u7u8P9/Z29Tquy9uJOi3K2", + "5YpI8nj38d72k529TlDwCfTPXIRNXYD3RVYemuh+/a++TElIpzREEKODdAe0kcAVRvLXqipNTnDk4k/9", + "d4fCNJYrPSbMZLalMbQlWaxoGhPzDQ6kky0adn4MI3kzZDCWx/vebCQbl7TWQ8DtJW+CKvFlFdCdmoDm", + "kvBESRwdGApdy+fgNIuFvW/DA7uHjtjwUqtO/ZgsSFxGAnN1mchaQVCOJ+bQKruibIFjGo0pSzMvSrSC", + "8nkmQBY1gyI84Zkyz4w2QLuYBLyeQfeYanbdTc99zsXlWv9RfRPncehrrUKHYEifWlMN3OIY2d4uRKEk", + "9OXPgebR1H6X6LXpYSxExc9pVs1q04OZrCWJIUGk4sBJrcHQDtNVuvTLLWAsde4fZr6Cd96T70t/atwF", + "vqyGLWYE8i+otRKLxpQ30P4cmnd2R9cd1xpSOsCdkav7ADr46/c12vYlw+ndQHyVM1puaygawS0saEQG", + "CKgLvGJcfGCN0s4VT1MS5fafwYhZf+78J2leUHRHAwc1J1QgLuiMVieuGtju0qvtJqjosOnW6Fju2JRQ", + "4SO4b7QTPZ4qk2vh0oVMkXL8kj2EoBec55kpLCeqguZ1nt2jAZHC1bKxxBdnb2/qm5YKPqW+fEPgC2G/", + "Ws3MeW293Bue97f/H+OBqfENRDTKjP9EwqNaIgnbvtvN8+Ls7VnbmvLUDqi8usaeco+XVcmtHETso5J9", + "lbQajEN/fbHkkxSy91OfLDsVOCGTbDolYpx4jGvP9XdkGhjXJsrQ6c9VeVbLzV215rPK4YDaPMWhjczv", + "Bn2PQa62jV4Jmu/9x/WamGu4LZ5PH5WwbWxI3wD9lifTQC/O3kpUeCl5LHXV4231lz+bLyUNcWxGNOG5", + "lJUNbICcnSXks6KjNUV65GR/DhZHCGhjMUszIMPz1/2TV++2kogsepU1gWfRnMdEr3uzxC0WLqqvcO6v", + "MIlFm6XDIIbsSkAlWOUU3BlIJXr1QEdxheOxjLnPWeON/ojgI9p499xEXekV9FBaOUr9ewkKFfze91KM", + "5kht057DhHWTaYXAvbpjNRumMa+UtleZ1EcqvxAcmySgVXxuJkDil9WD5pfrk+6YQXzznjjH8JpS4wve", + "Ojo9NgJDyJnClBGBEqKwTTlacnEBcSjoBX19R0WYJOBqN/3P1d4tLSb4cjRWqxH3qJG3404MuC3x5q+N", + "C0KEEszolEhl480rM8s53nm0f2CyYkRkuvdofzAY3DRG5VkRlNLpKLaMC38pXGUg5593DncQitJlLx+D", + "s8M3vwQHwVYmxVbMQxxvyQllB6V/5/8sPsAf5p8TyrwhLJ0SqdBpI4FK9UlT31nm94NSzkuX369TXju/", + "PgOeDRA35403Vnim9RODcZ8bWHzr1CNF/itVSjlSdgjtkH6EflhtCXWCEbSxc2ZM0bjIzNK0gd4qt45c", + "mX6gkXogJSxPOBDH5q+Qs4WmCl/2gQoDd98+6/3AermMI+rB5H9Ybc84SUBU1Xp6C7Zwmq5HW7+gmPO/", + "rllXbGy05yb66lz/Nm9s1dlfzf7rz/9Xnj3+Y/vPl+/e/ffixX8d/0b/+1189uqzIqhWh8V/1dj2LxbO", + "Dg9LlZj2rqh0ilXoEajmXKoWCNsvSHHjrzlAR6D4HYxYH72kiggcH6BRUHMRHgVog1zjUJleiDOkh7KR", + "Dpu685kx/+jOH51u+ak+RmRDGoQ9kDySSWaTiCeYss0RGzE7FnIbkfCmr/+KUIhTlQmiT0/LsPESTQSk", + "9bbqeTF5D33Eafppc8RAwyXXSugdpFioPI+HmwGQwq7K+AzY5iRygeFGQx6x/F7K48KNjWaQG0HANl/3", + "uPQDxau+cFENxXky9EXQg9eXPsiYSkXAMTvHbI1GuTsaejKssIonwyfDtQJ+jkMr0A8ooZnv3yFlB1oy", + "CAxTG8YNHmodbOmaNxkaQb+8eXOmwaD/9xy5gQpY5EdslDzjAyiNjVDFsuT9txl4s43C6XbckDGSQbe4", + "Q9TQM+Me+ublOVJEJM5hfyPU4JzSUO8Pnv+plJlGRYrR4dHps81Bh4IFANt8/SvO8U2+w3pwhzWatdkC", + "c4zX8O2hk2Nwz7UUWghw4FbznAsUGwZT0PUBeitJ1dcVjsq86puTjJeF5c3cAKNg042Y1jnFAXqdy404", + "X0qlSELVmFfQJQxrH16Mz09j9F4j/bhwepFlbeDhg1XuJK5v3HZWsJr8PRAHmrd+3SWb5s1ou2wM1ZP5", + "UaM4+y+dNeXLizu7N1Vyb5rhoRqEWQrgzZM8dM/OcBdZDpoK3zVV49ZXfKQ/2zd7p9a8O0VzLNlfFXys", + "KTfbu4875evUs3Z9/y6/fPOpWVJOli6iM3+3NbGtlzSOjTuEpDOGY/QUbZyfvPj15OXLTdRHr16d1o9i", + "VQ/f+XRI9uBo48XZWwiXwXLsnpDavSZx4XlMrqlUshnw2ukldnVyiV8qCSC8EcSbXzArhHu+bmzjPvI9", + "fE2/wG8v18TK7BCfm+LBSst3lOGhlbn6siNU+az5+cvmariT5awtL1IWKpzT9q2TI/QC6nFYPZSaBZII", + "nZwVSRYLq5YbvrYnW6tnezgcbA+72PgSHK6Y+/TwqPvkwx1jyTjAk4MwOiDTz7AxWsQ20h+Or/BSopGT", + "z0eBUQhKmkCJbK0M3+n9tpmD4nYpJ+oCxbqkEjdJItEtO8TnhuSvSrV8Xk2y3FnI+4xKJJ1cKNzVbp0n", + "bK/xTcznBIU8iyMtSE006RrFjkRW/5REFfmrgdrfskvGr1h168aKqhnAnxkRS/Tu9LRicxdkatPzdtg4", + "OF20nANPb3QMO2tk7bWruWWihvtIzlBnu6Xr7ounYigb/ZwTp8HQDsa/Qvz0PrxTZo5G48mKPdXMNhFZ", + "jLPMJ1XpTy504+3bk+MKcmC8v/1k+ORp/8lke7+/Fw23+3h7d7+/8wgPp7vh492WBPndHW9u70tTpeb2", + "UCkAPJhATSRcdKDpLXeGmWQK5Y5ympCPtHiKSnKwCQwCq8QJowqSQFI208OAkcCKySbC0+SppIwqSCkA", + "CW0o01sGa4wexLo/HaAX0BY+4QQCltwitHJUNUTgaGkMsZoxuKlT+NfqJZ/PMyj3A33kPFMIykPpbWsw", + "WHVl9RCGxxyg3zj0Ec5LlfG63mOag02g2byuI21YvyTnvwqTWYZ5gJ7nTDJns5atbkhi/zS827pWg9v4", + "ZsV5z554oLGlOLmSX1ovMBANeoEDFPivNT3Z7Lq8QRplVPS9UBAcAwstPIUyRWObJQF2QqFAEmwEw+G2", + "UbLNCEaisREB2t4bjfuJFRPyTo5RvDtFGxAP+XdklUr9r838bbJMlXs7T/ee7j/eebrfKeqhWOB6Bn8E", + "zlHNxa3l9mGajV3tkZatH529NSUOQ85klhgrgd17yck0FTzU0iplqChmUkz+dPC0HOwR8cwUdrJLspFh", + "n0rly1ZWnml5YPuTxgs6nbI/P4SXO38Immxf78udiVe5K+qkeSXhk7KptaE2kknfZHH0++MDQgnZGrLy", + "mkjYATonCgH+9BEO4ZLOfZosyrnAFgtxL2Lt7e7uPnn8aKcTXtnVlQhnDPprc5WndgUlEoOWaOP1+Tna", + "KiGcGdM5ekKCCWYFOD+dIZvQeVitBDrYHu76sKRFXiqwxo69SFpB/s4KQXZTFujgmpULSA0q90J7d3f4", + "eO/Rk0fdyNjV3RPXqzmMS+phwGPzoJRPfgPM828Oz5AeXUxxWNVQtnd29x7tP35yo1WpG60KcviY3Bs3", + "WNiTx/uP9nZ3trvFXvlM8DaqsEKwVd7lIToPUnhOwwOKJuvttd0WPsHTINhrEsaYJoehc5+p3T4mx8ZY", + "mGbFIXS5GKyRoHFxdejbSUWrFQ4yogEXqFRxcbDeHHo762Y7mzb3wXo23pShY8w0uGyQgEnleAvYpYIs", + "KM/kFxiIKxJqZJrGnIsb9W3zR3pNZBYrY4KkEr07/SswEY1cSCqSVn3tLfqtCKW45eZuRMAVnPBjdRuw", + "Op1Gl6NfteFeC5n2VvnRVsi/NWIp0qwqY+vfvo9wHGaQvAzn56l3BbEHPFPwUr80XiJxzDlD4RyzGYFk", + "8CZVIpshjOY8jgaB/6kkjsZT7xNGXmGeF/XL3SJ0N1d/f+MFL0raGVSq5ed9lBiuYjM3DTrUki9q4rZE", + "OGl4YsVLaQBMl4o2H/OZBC1Qgf/LoJ59JsXCuLVgZvLULRKjPFZDt3b0be9ZYo17+65Qc3XyqdVorYyh", + "eA5JHAouZVGY+91pdZmrHBjzevbr37Ori+2AujLlTBJ/mXhbE76Twcd3IXo8wz7nSgQcBgfQVTWgbR7D", + "BLMMMn2VEJlcp1QY9Oj2OD7nUo3zcJQbLlaqMWRxygQpYtbcfTmHAIClYXHQxnsvOtZ2G3Dl5Y1v0buB", + "Vf6h2hbYzlO9EPVDq5fjoA+NmwE5K2OAiqCiegTJTULGirQ/VMKotBSthDYYVxW2VEpds9nlocqvo+p5", + "2krKvtwbnneN5lodvHWG1fyETbknN+QNDP7WJd75LqREQPlxzlBEGCWRUx5zy7+1bYGTfSwJijJiIWcE", + "UoEtwLEhb8gByZxRjLJZjdfXJ+xihjdrWJ3kCea1Dbs8OUq/a/YbkQGsjJOARLhw0u7k8UDl2G8pbg4s", + "yCyLsUD1iMUVS5bLJKbsssvocplMeExDpDvUn3OmPI751Vh/kj/BXjY77U53GBc+hrXnGbM462FqDqQ2", + "b7GFn/QuN2v+7WB62TL9t3T/Ti+4Xr+h5zQmNqjvLaPXJUSvZkHZ2xm2hT60DFoJemgGhN6Uc1uU9VG8", + "i9U8zKsmeNzxjQdQ7VWiaois7Ne3W3AxWxXo0TTFoA33KOyyzFThWsr20skS0s3Lre7+4FazJUlYnX3v", + "yaPH+x3T7XyWrXNFVeXPsGwukhUWzZaTOu1iNnvy6MnTp7t7j57u3MhA5TxlWs6nzVumfD614ig1o9mj", + "IfzfjRZlfGX8S2rxl6kuqFLo5NYL+rSCdIsw65Znj9Y833H5JN07S9UC2s3GuEJaOqyIXKVaXhtkOiWg", + "VI4N3PrFYmru+Z3WEOIUh1QtPQYTfGVSvudNauHCXaxp1cV6QGrHthkfNOeS2aRw6Nxwk6O/GdN6DRee", + "dM7aJbNJmxn/VX1WY8QvbEDlJ6IOLzRFYYGmuSDfzxWWFa8O/XcIOZuLWm11/yHTontVaofreWHqwjPS", + "F/LuL0JdPv7acZbMvhUhuQ7xVVdoOwneSIf23Mi+MpXrvXJr/MFegLfrNZ6U8+mtTFhYSb5X3Lo3n7db", + "lblmP3OD3Xy+kgvoTTrWU4sBPto1WJAXY/cqKNGCTYqL9Rml7yBBkPEpuFWKIOuOcC9ZguzPd5IZqHEc", + "50S5tudao8/iFQmhmSJigT2GKTcEck2qdlTDiXvImvjQdrJZK1W4N/fLajaEt6P7mJYCx6kgU3q9AltM", + "A3NdV93HpYVAVE0aLtFGgq/R3mMUzrGQtbUzOpureFk1su55oic+r4gzUVp07p5evThN17H5omGPszy6", + "j2TPS7EO/rTvJBqvCnQ/yps5m3GKlyBbtiqCj3f3hsPdneGtIt2/VDb60jhtHqGlftaYU3l6LI+Q+382", + "UxZeCWqKmjkwSSUITg7AmyrFIUExmUIcWJ4qdq1O35h69eLtI6l1/M/x3x2Uq1xq7SyWxTHOQPBw49gk", + "AW4bgXulrcZ6lL83l70iWCxnM2Ejaqzuv7rfH+72h/tvtncPHu0fbG/fRWh8DqQ2F57HH7avHsc7eLoX", + "P1k+/nN7/ni2k+x6wwXuoPBBrY5grQ6C3UNKRD0XZT2HqyQxZaQvc7e39Q7IK3iBeUlaS/83sz6YHawU", + "Fs6rmyzLDFgVwKmXZLuPwCa7+pUmlPryT45XL/tWfmT1hfgRrL4UwKdui4FULdufmxckYx3vnbelhp1v", + "npW+jevuHp/TN5C295RbIO7D5wpjrFDYqhu7eat5VLgZF1TNk9XXQ94szzIAj+EfpIqqgTQDdDJjkHm2", + "/HP+9lEuDq07B70g/rBXpRn7e/eQKhtVnyOgPeqyGNDhbQASG6+GAjQpVAth3BOwIACIn7b720/hhT7+", + "sPfTsP90gP5R8hToGWiVwbftWld+HXaBYc4oQe40L+fbT2/0jO7guQqDfrX3UttFbOPtLY4XaT/dXeHc", + "pisHXHxunHEtquiOCg19WrFjJzjfLG2P6+URTQZ+2eTpm+H2jdP23OyKGHyGR/FnqXrd1LsYS9UmV7/E", + "UuX6GBKZla57UOKG1Srd27TiNhQAXDsh1v4AyZqG7/Kpy0LocVaAHppxhYoggLVSDixfZMyLD9X1F2WK", + "IT1EK0LsrEGIbmvKA/noKtI9OUap4FEWFh6wMSw6C0Mi5TSDqsuDrhLt+jfGu1TmwbVca/Trlfk27X19", + "mCm5bj/v38i1Kk2pEbb9qLeH64/6TiwAvSBLo/U8zDTqxsFulIljjU+lxx5RBXtNCipt5n0Hjv66DMGm", + "ggcFm1CoxYEsdRUENU41MUl66gbi67E3R8AxiYkivkGQKelp3T6oLLjoepa6vf/EbzLD1+MQAhIbC/mV", + "kFTL6QmXtsRmgtnSu7B6Fm20MXQVJCWC4fsmk5eFVnVxj9dKIa1H1T0hec2iayK+yvnf8wwdXzYbue25", + "tlbE3Qkqb6yq1JZRpu7rWbY7Hvb/x9gZ0XhwsPXT3//v/vu//cVfm6WiSEki+hGZwgPYJVn2TdFZrbQN", + "qvlMIdtNIBW2FU0UwQlYEcJLYqwWCb4ur/fRMKek5W84aWwBXg4TyvJ/r93Q3//S/u5WAuNbYB5rz/Gz", + "sx/dRWpZxR2P3kiImLn87c5dDMp5s3ipj0qiUv46e887sv6rzLuUC8deGNloAOWOJxTSgMoR02oODkOS", + "KhINbB4vCmsRHEiyXj7Z5tFz7t2aWDEkD7XhNrU0WR+9RYoPAkau+maGqK9xb+/Rvq0YX4bkduOIfYdu", + "IrbbKixqKHvMCC+phHAE53Vbaow2SJKqpcsS6/wiN28WQX6YD+h9Cv3C6bOGT79EttC3K9ODfof1PcsB", + "/m5Ba0P7G+ffmpPP71h1XM/UY2jS1iyrZpapqUxS9dv9rhJ9x4/BfbDpI6W/GddEmw9zltUTg28lTG3Z", + "7Lu+cIgIShKvdEYtqMxFu/eh03ofy5VSZmlnpZW0n82pE6bqZaLbAXSmQXM1J4KUDgI6FClEbwgy6yjY", + "IcjG5MhMiejXi9qZuguCgudhrgc7EOTOpE3D2OoMN6f4Op8BjKpYNp4eYB9FrrftFz9DLZXXrrgZnboh", + "YBk1UdefrqaKRV3KszcPo4xVzX2b9l7Cs7xqBfdro60achZzVFDTh4//wFQ95wKE4/aQljvPegOCd0QE", + "xPTWc9p0SghDExKNeaZW079NMm/jWSI0IVMuSCn/rlMEMCCxrXm8hhe4oItiDe99coMkYSaoWmrN0Yqk", + "E4IFEYeZIXgAJEwEPxcTQzrbT5/AhDb1eLC9IIwIGqLDsxOgR6jbr4nj3SmK6ZSEy1Ar4JCNtJF/A4S8", + "V0cnVvlyGd/gQZEqQD1Xhvjw7ASqmgqjgATDwc5gCMScEoZTGhwEu4NtqPGqEQ62uAXZ7+FP65xu4tIo", + "ZyeRlYN+Nk10L4ETooiQwcHvHidvRYTJpi9B6sSzkt6QYiqs4pDG4HpuUIXqvpD+yF2lB+Y+7hmAd7Yd", + "SbW0jngkfWWP9b3GBEM1sMWd4dDoaUzZixcX1S63/rABe8W8neQ5AI8nF1BDrncypQX5p16wN9y+0XrW", + "Fqj0TfuW4UzNuaAfCCzz0Q2BcKtJT5jxDkYmz4T1fyjTGaBQmcJ+f6/PS2ZJgsXSgauAVcplmzBMJMJQ", + "Is+UcviDTwbIWsYh/amc8yzW3AQZ12cS6QsLa54ymH1AWIRzuiAjZu9pU2wUC8gvnSB9Pxu1pUoaZmpz", + "+nlQ2s88Wtagmw+3pYfrO3toAeB6mkVJxpAtatxWp6WwhFLGoN6jJDYnZV6woOlkAQV6Zci9lYkJw0wV", + "9V5NZd5LsrTGVu+AndK6aIYHx0KgEHyer3xn0x/PANkz/aFAx/k3ZMFbFScYvFCEcRYVMpdzscViguPY", + "G/c/i/kEx7aA8SXxiKgvoIUFSjnRqBNuGI+ISRqZLtWcM/N3NsmYyszfE8GvJBFaBLLZpy2sbfVOi7pQ", + "SZ4mkAHa1LbQc26ZJW59vCTLT4MRO4wSV7dE2uLzseS2srNJn0Mlcj6ZBnf96U1bnvuPMql4YlGKlQtR", + "mmXyTKWZsk+dkiibMhuaQ51SOSfRiCmOPgpTln75aetjMeMn0F0IjjSelJqYLW19pNGntlXLMda7H0NT", + "j/ZHAACjQN8uo0D/PRNY6y6ZnIMpQ4L5YlY+0o08FlvLhZt1CIeYoZSnJo4dkMoUrK6MAeUHcBwjBaTk", + "+mppE06yZT82NMVXS8/GpZhAghoZQVW9EjEN95746UmSUBCfgeO/zl/9huCq0mdgmhVmI4ARZfoWRVEG", + "kjzMPhixZzicIyM3Qa6yUUCjUZBrF9EmrDWT1nG23wcR9ye9tJ/MND0a/TQY6KGM9HyAfv9oRjnQtJQm", + "Y8UvCRsFn3qo9GFG1Tyb5N/e+wHa5t5/XmEEaMPw/k1XPAbSDBTXoLk3MIsQt7w2XiKMCg5UtqNMKMNi", + "ZeUbD+gtBLUqj2eyDIyPI7CgjoKDkbOhjoLeKCBsAb9ZQ+so+OSHgBWi2xNjmeI/TtbOkWh/ONxcH3dn", + "4esRoSsNNfl9akhfO19M8LBCV1PwMJtzWf30CZoyTkbcugfJ52ccucIAP0S8NSKetVyUhDfoX74HDPrG", + "xCi4NQlM67Oxk8BWaicGLSCtJWgcLkrWKBzUSXAF8pbVj7o631Qr9tqoLIQlxg7/9u4B/2DeohQ6zPv0", + "vubFMeSozAsDPyx0hMNyiNjza8QviPoWMG54X6zUJtT8mvj7UPDnBbFyXwG0GjfbIgv33uTPBQAxANKO", + "YhprXfUc1tQ/J0yhZ/DrwP6v03ggs+1FzGcXB8iAMOYzFFNmX+NKr0X6UrSwhE4mDCDvZ6MCXCKmDXN/", + "/ut//wmLomz2r//9p5amzV9A7lsmOwYkbr2YEyzUhGB1cYB+JSTt45guiNsMpFYkCyKWaHcIYmYq4JOn", + "2KQcsRF7TVQmWOnV0uREknZAUD0Y7IeyjEgbRqEb0qlN2GAMzB4V3tGyAeW9UnSv6XNqdlDagL4VHQ5A", + "BC412Wut/hX4rWdmzxX7Wd1W3rCYrucvilwrg719s8AbMhgAsY/u4IPdNNo4P3+2OUCgYxisgKQcIDEX", + "w1jhefCDJ63nSYajVBkKQNnwplKd8Vb777Ft080AbEf8nizAbYXT203AxuRBBIkcvH7oCl3MwX64OdOw", + "zz577ILn2g20t99veQrnTdRJEf5y5+xwrwlz86UEsq+hAqMN56jtav6dHZ242jCbXw3p7+XW0Du1FRXy", + "qwNxU2nw3tSyI86mMQ0V6ru1QFL/hOSqWhVBHgo7eG1XjbDbVz39Xfl+26pkc2m96fLELsWVd/e3R23S", + "m1wjRYq+Atd+3CTrUOeYypDrviVs6Yc4tQULjfiS02kZi9YZpIzfd37lrBSXLHs+OXYEeX+mKTt1xup3", + "wz0wxeMaQ/yKjLBWhK2U1PIhYfPb/BRdooAVlqtvCzWH9ycF3bcVy4fmD8mMFdXAprmgSe3beoG+IOoX", + "0+IOD9rO4Nn4ORGOql0OYth1vi3TFYVzEl6aDcGD9Grd98Q06ab6mvG+J80XwHMTicWC/IeI0kHZLWC1", + "SsE9sYXl7k6/hRlupN5+uXdei2AeIIOzycRZrE3NNiyXLNz8rp567+U2M8B+kJfZWRbH7sVjQYRCef3m", + "8h2w9RHcktbL9o7aVl4Hb1+/7BMWcvBDy32o/EKU/fKFJXxzYGYrP9Cki05oInapu8/aJJzPOH/jLojy", + "+uD/sfPcVgj/j53npkb4f+wemirhm3eGLMP7Ys33LXE/YOTTAjetAg1YE4Naoesk1LxVRyHVtf+u5FSz", + "6RtJqjlcfwirXYTVMrhWyqv2KO5UYjVzfKUnmRzZfNCGT84/8TuTVO/Xymcx0uXvpbL67GELtHABdl74", + "RBnKJHmADpQ0x7jytdHRXF0Q5Mrrw6HuyXEPANnToIOEQjZA5J6M124d9y7c2nnv33J9mEzoLOOZLMee", + "JFiFcyJtsFJMqgz4oYndxfXcKnh/w1g6vM+r497l6h94f0cSf/1ADfM2L1DrZH7XqqvMb9trmd8Wzzex", + "a69dUX6bJ2mzxanQBVF3ReNKrHnT2dG3Lp8ugt5qRaVQFxBoEAcj9n+0/vG7Ijh5/5MLksmGw519+J2w", + "xfufXJwMO3WoQpgS1CbvPPztGJ79ZhB9Dvk9i5C8+jpMQQBAPZfA5t9OQSpePrtrSA4Lf2hInTSkErhW", + "a0j2LO5WRaomwbp3Hcnhmw/gNonJDy3pPrQkmU2nNKSEqaJeVsNJzJbbe4CxZcy+D5WcOyoXbWctKSfK", + "NQJoka393h178snvXzlyieEfpo88N1ExkVNHisuwXR/51vBheL/M+f71kIeMYkbgb4Iu1TKlrwwjZHpM", + "MgVOiUWGEPD6RMJI7fmIA1RUP5RZmnKhpMkWCQIwlMNScy0A+zJLVpNF+rJDQmJcSmRvxCCBvP5sYvm3", + "LsnS5IKkvKjqn+/U5n/0xV5Vc3F+VTL68jKWP9FoJxnrnsnY5lP+ejLWV2Md9yJpnVTS1G/khAEK5YTk", + "lMzz4D76gbLZ5oPyQDXMKt9bKZ+RR9TaggJsNrvulswLvbZdtKUEu7Y84b/hjdvcpE9qd7loSwBEEcUz", + "xqWioQvcrSfy/nFDd76hV0PWi81TW17Tr9A/5+Ky6xXnKff0AG668g6/QVuCXh5kA/v6JgVQts1toJHm", + "3m/BRg2vrxmCQev3Yhhnkb4I3YXoRMmp4MnY/mjy1WqqsNlAwUQR2lG/NrPRs9+Dweg3rhBN0phoKZ5E", + "qG+wSZ+mFf1d0ncqSxXvbsYMNdmUA2JMMjrpquZYFgmPa+7ANuCdvXlcXq4Z89n6JBj55C7jgycLxoiZ", + "pPTEZbC/QDmTRYojSWISKnQ1p+EcMmLo30yhTUhWgdP0Ik+BtXmAXgClljOBweQbkgitCIWcSR4Tk+hi", + "kSQXB82Mre9OT6GTSYZhcrNeHCCXpTW/IKRuVc5wkZfj+c3m7djQmCR4HJsTvdBaY2l/mzb3RZGibMR8", + "eTAYubID0im6KKXEuGjJieEY6ks++2rSVq89saTZi+JIAOAMbhIWBW0PMTT2Z8PYHnpLlXTMzGGWcceJ", + "ORqLeclneVLLCirjNO2KvnaZgMWLJFmBw2ijVDNTqohn6u9SRUQI6Gyxuw250QYOzT8UvtSIamve5FVH", + "Af28z40my5wXVJqplgq8mH8tkiToBXY9pex0N5De12Q4qQ/YfBbTJ1NKY/JD7r5JgpIqsy9lKKndHLYq", + "e7vIbYvNf/f2WQuo6HuwslTfs4pVUOZEFThbXhSIelCZDuAgG7KYKU/koxG3y74s1bXs9rzVqIj5DSit", + "61698vKGee3F+37+aq7gIQfByMZuplzUw+PXvYt984j05Y6ksdUuGPIDN29unuuEmGm2oswlVOmUYOeD", + "0o+Q1zmccy5LaD8hc7ygXNgM7NbqmmMmmCyM9mi95y40ql5Y++2FFc8PrK0J4fInO8cAulufO38P96no", + "8bykbeccv+dEasgCKRFGE0HJFKU4k0RLS1lCkKkwYhN5ExzOXRHnwYi9mRNkSzeWDAh5pV8q0cV2ctFD", + "k0yhGIsZaDvmo/GkEyTkSUJYZMqxjtic4AXVqppAMVaEhcu+JFCed0GKAiZadbcvlAgeKPMCoD3k6saC", + "geGiVBX2AqWCABIZdZlVSrCOmMjYf5rMlXrYC7fQC0SkwpOYynleKyLEEWGhNy3k+bfNxr68EfecqGbh", + "1K/yZnkrXvo1HzHLtsy8dPU38b75wBy1uHD1LTuw+RVCr2xXDauej+dFsdh/Q5I2e3V7/EovMzmIV1Hx", + "t/EkU6kW/+NZRlmSjDIzHalWVP9u31pyhoIyVnlusTbZ2z645JUQcjDfiOdtfXR/ntzCRvaNcMJeq2Lf", + "lnO72PS3wHItVG/Fc7+ScdDakkpWsa/Igu2ivp74xEWJy30TbNgQXM6NyzxHCQw6FWc/mHGdGVv3gNsy", + "Y2dxbTyAl9gzZf00xm18uagd72fA1iDwb+r9WttdiRF+dcZXvAjcG7M7ydmbYXgpXsYcf+/vMiEXwgR0", + "2nLEDyehWMkWWHpg2gCLWy/nED0XTfLu9HSzjUsItZJHCPWAOUS1rGmYeKo1vloQIWjkSkcenR5b71Uq", + "kcjYAL1KKNRzvCQkhUIxlGcSQWTuQO/PhbY2i+BVYlh7AWFKLFNOmVq7iqLp3Szm061K590zn7QpFb/7", + "x2Owwj88JgW8Q4srdgOrtUiFVasznnNOo8zUu9TSFp7wTI+uOYsrtDuDu21KYyKXUpHEeOZNsxiICJLu", + "2ppMtp+JKO0hqiTS9NCDCLyUiIRKSTmTI2bLv6dE6Ll1dyj+WzgZeY33Cudc88ywvm/DgU0vxvhsYdUG", + "NUgtAHVAg4NgC6fpFpSL9jtJ2eV9xpKeg0cakstkwmMaopiyS4k2YnpplA60kCjWf2yudGkbQ78vXXHq", + "9pSlIX3CptxblMPgbI7M30cQUpWtuUfEB8fWXpAysTj+AwftZ2tyLV8TBMd9RROSB7+jTNGYfjCsTg9C", + "paKhiaspQi+hCLONvhyxU6KEboMFQSGPYxIqZ1zZSgUPt0bZcLgbphSylOwSWBwwvPbPCcx4dPYW2plC", + "0b0R0/+Agd8cnpmX2Cm2NoLSQhlRV1xcopOtV2ucfM8BTP/GXnJmgytjIL0H/uP57uaRza00JFtIlKer", + "FCCefvdunFaC+2EteJjWAkgtke9mYyZwCEKxnGcq4lfMbxlY8DhL9D/MHyfrEpQoHM7fQdNvRto1y1k7", + "jdvggyBKu6eImKJBX+WBwgDsofqXasC5LYAQU/Hc894Ch+p7xO4vb5Qvw/EbfJq0EHUFub4Z2rrvm8+u", + "weXdKsPjoZC5wTS3E8VXW5+uMG23Pv0c8/BSoowpGleSGmi9DfKA6h+LvI324Q/EBIiOdKXEEblOqYAM", + "NrX0CIjoHUuEkSIioQzHW7BnMwhkoHRWLLzgFIKUw5hCmBiNCEp5HEOWnas5YUjvBgxVboDSO620FSDK", + "bcpPjIqjCQl5QlxWzk2f6vYPTNVzLqopNr8VvvimBH+9H71Vvc81WUXbZ/ysLKOn+BrcmqPMPhO7FW28", + "4MWPxhTUQ3A2o2B3KEdBD42CnWQU6BM4wmBCxQo9QgllmSJygI6NfQvCUPeHSJKQs0i65KDOgrc7lG1B", + "qQYtWyIc96HffYo9FqsAlK/tJD72oNsh3R8CbNBGmeAsTUY9ILoI8UyBA7ejK9sqIgrMI5v3/gJbopEf", + "un0XTv4PS74VHgWnrNll6egNZ8/TR661urmgijmXRdZJFOIUh1QtewjHMQ8L60Em89eBfr6UiSD4UutQ", + "gxF7nSeutIEQ6Ojsbc8ZzVBE5aUZwdrFBujVggiZTfLFIeAGxoIHh0GiEVMchTgOs1jjLZlOSQgxDDFN", + "qJItdrV8KXdZBrGYxHPw7mOetuZhGZP8OAGnV6CFrGHcljnqLUHCGNOkbFSqAwdEX3jSBbPvRA/K9TU8", + "je3zVii4lMgO1ScxndFJbB9r5AC90SIHTsiIpTFmjAiUSeN3pJfeTwWRMjOBMXoAqDNrMKqHikQnqeDK", + "moljzoU0ll2N4e9OkVQkXYFmr83Ip7DnO0oTbAa3M30lhaG2hvZryTZB+kAMphiAazzS1/RXcPYxC/ra", + "6YQfCuG/EXQ2I0JTBTZM1jyNGrJ24DREX4n0aM2Rf5636pYjPx+15M1d8nRemahi7BqOQYC+yQusZ/JL", + "2prLxH66WfTFr7pTx7mrXv7+RdhPn7nL76X02HnJubprZv0Cwx9akvvSyiukWglQWJ+OoHNEwl1GCHTO", + "O/DV0g085CwDuBJ20JZO4NtDhOH9Rsfdd5rth41blSwBlcI6LaFS69N3fhMYeDd5O79ydOgt8nZ+U/FK", + "kHfx68WNflORShU7oCse8t1n5ryrACWTnhPSWLQFKBmuZx0JVipK72ybbmqSHfF7kuDt2/MN5HcH9h9a", + "fweVoQQsv8nOxEa7vC0kSdXSPS7yae0BUNIPEIzhS/yQ+xDcXb6FWzyvfzn0cHja+rj+o57Wvb3fF0WH", + "T44ffhGtMs1VLpYtfev0sQjndEHaje5VCrYgSgXppzyFx5XIAMzCw91lCovB7AOyw9tcVfZfiLoUxyRC", + "ERUkVPESUaY4cAQzx18lElxrAvCdi6XPmF6m3OeCJ4d2N2vuQ0tT1hhWvPkmy36EFe4vHLdZYUL7jJd2", + "97atGR6iDL34GW2QayVMxl001ZoPotMcpOQ6JCSSgJOb5QVvD1ssm/QDGc8mXVa5InfyK5ubGoWZVDxx", + "Z39yjDag2MKMMH0WWtSfgiSbCr6gkSlEWgB1wWMD1e0WgN7U7qqFirxShlMuzOK+igzT5UKafaBplS0Y", + "14XgIJhQhmFxa7MUV2nKBFTp+TCFsIaCdhzmBD+uMKv5bThlR2OiVnIcEBXnJjXe5o9r7iFfc2XHVHen", + "VW67bqUiu/mqdnQhvYuEubkf8/2ard99O+6VVD5Iz0prOl/kCmmb2fzbQsHh/d0P920uf/eA3fFfEKd8", + "l0zlMIAe0YcwL3mIYxSRBYl5ClUkTdugF2QiDg6CuVLpwdZWrNvNuVQHT4ZPhsGn95/+/wAAAP//tsVd", + "yWFyAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go index f73bb47c..2a1063b2 100644 --- a/lib/providers/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -5,14 +5,21 @@ package providers import ( "context" "log/slog" - "time" "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/instances" + "go.opentelemetry.io/otel" ) +type autoStandbyRuntimeManager interface { + GetAutoStandbyRuntime(ctx context.Context, id string) (*autostandby.Runtime, error) + SetAutoStandbyRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error + SubscribeLifecycleEvents() (<-chan instances.LifecycleEvent, func()) +} + type autoStandbyInstanceStore struct { - manager instances.Manager + manager instances.Manager + runtimeManager autoStandbyRuntimeManager } func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostandby.Instance, error) { @@ -23,6 +30,11 @@ func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostan out := make([]autostandby.Instance, 0, len(insts)) for _, inst := range insts { + runtime, err := s.runtimeManager.GetAutoStandbyRuntime(ctx, inst.Id) + if err != nil { + return nil, err + } + out = append(out, autostandby.Instance{ ID: inst.Id, Name: inst.Name, @@ -31,6 +43,7 @@ func (s autoStandbyInstanceStore) ListInstances(ctx context.Context) ([]autostan IP: inst.IP, HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", AutoStandby: inst.AutoStandby, + Runtime: runtime, }) } return out, nil @@ -41,16 +54,60 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string return err } +func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { + return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime) +} + +func (s autoStandbyInstanceStore) SubscribeInstanceEvents() (<-chan autostandby.InstanceEvent, func(), error) { + src, unsub := s.runtimeManager.SubscribeLifecycleEvents() + dst := make(chan autostandby.InstanceEvent, 32) + go func() { + defer close(dst) + for event := range src { + dst <- autostandby.InstanceEvent{ + Action: autostandby.InstanceEventAction(event.Action), + InstanceID: event.InstanceID, + Instance: toAutoStandbyInstance(event.Instance), + } + } + }() + return dst, unsub, nil +} + +func toAutoStandbyInstance(inst *instances.Instance) *autostandby.Instance { + if inst == nil { + return nil + } + return &autostandby.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + HasVGPU: inst.GPUProfile != "" || inst.GPUMdevUUID != "", + AutoStandby: inst.AutoStandby, + } +} + // ProvideAutoStandbyController provides the Linux auto-standby controller. func ProvideAutoStandbyController(instanceManager instances.Manager, log *slog.Logger) *autostandby.Controller { if instanceManager == nil || log == nil { return nil } + runtimeManager, ok := instanceManager.(autoStandbyRuntimeManager) + if !ok { + return nil + } + return autostandby.NewController( - autoStandbyInstanceStore{manager: instanceManager}, + autoStandbyInstanceStore{manager: instanceManager, runtimeManager: runtimeManager}, autostandby.NewConntrackSource(), - log.With("controller", "auto_standby"), - 5*time.Second, + autostandby.ControllerOptions{ + Log: log.With("controller", "auto_standby"), + Meter: otel.GetMeterProvider().Meter("hypeman/autostandby"), + Tracer: otel.GetTracerProvider().Tracer("hypeman/autostandby"), + }, ) } + diff --git a/openapi.yaml b/openapi.yaml index fd55aedb..b1884bac 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -235,6 +235,71 @@ components: maximum: 65535 example: [22, 9000] + AutoStandbyStatus: + type: object + required: [supported, configured, enabled, eligible, status, reason, active_inbound_connections, tracking_mode] + properties: + supported: + type: boolean + description: Whether the current host platform supports auto-standby diagnostics. + example: true + configured: + type: boolean + description: Whether the instance has any auto-standby policy configured. + example: true + enabled: + type: boolean + description: Whether the configured auto-standby policy is enabled. + example: true + eligible: + type: boolean + description: Whether the instance is currently eligible to enter standby. + example: true + status: + type: string + enum: [unsupported, disabled, ineligible, active, idle_countdown, ready_for_standby, standby_requested, error] + example: idle_countdown + reason: + type: string + enum: [unsupported_platform, policy_missing, policy_disabled, instance_not_running, network_disabled, missing_ip, has_vgpu, active_inbound_connections, idle_timeout_not_elapsed, observer_error, ready_for_standby] + example: idle_timeout_not_elapsed + active_inbound_connections: + type: integer + description: Number of currently tracked qualifying inbound TCP connections. + example: 0 + idle_timeout: + type: string + nullable: true + description: Configured idle timeout from the auto-standby policy. + example: "5m0s" + idle_since: + type: string + format: date-time + nullable: true + description: When the controller most recently observed the instance become idle. + example: "2026-04-06T17:04:05Z" + last_inbound_activity_at: + type: string + format: date-time + nullable: true + description: Timestamp of the most recent qualifying inbound TCP activity the controller observed. + example: "2026-04-06T17:01:05Z" + next_standby_at: + type: string + format: date-time + nullable: true + description: When the controller expects to attempt standby next, if a countdown is active. + example: "2026-04-06T17:09:05Z" + countdown_remaining: + type: string + nullable: true + description: Remaining time before the controller attempts standby, when applicable. + example: "4m0s" + tracking_mode: + type: string + description: Diagnostic identifier for the runtime tracking mode in use. + example: conntrack_events_v4_tcp + UpdateInstanceRequest: type: object properties: @@ -2963,6 +3028,39 @@ paths: schema: $ref: "#/components/schemas/Error" + /instances/{id}/auto-standby/status: + get: + summary: Get auto-standby diagnostic status + operationId: getAutoStandbyStatus + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Instance ID or name + responses: + 200: + description: Current auto-standby diagnostic status for the instance + content: + application/json: + schema: + $ref: "#/components/schemas/AutoStandbyStatus" + 404: + description: Instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /instances/{id}/stats: get: summary: Get instance resource utilization stats diff --git a/stainless.yaml b/stainless.yaml index fa68eb7c..8f9645cd 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -73,6 +73,7 @@ resources: instances: models: auto_standby_policy: "#/components/schemas/AutoStandbyPolicy" + auto_standby_status: "#/components/schemas/AutoStandbyStatus" snapshot_policy: "#/components/schemas/SnapshotPolicy" snapshot_schedule: "#/components/schemas/SnapshotSchedule" snapshot_schedule_retention: "#/components/schemas/SnapshotScheduleRetention" @@ -101,6 +102,9 @@ resources: # Subresources define resources that are nested within another for more powerful # logical groupings, e.g. `cards.payments`. subresources: + auto_standby: + methods: + status: get /instances/{id}/auto-standby/status volumes: methods: attach: post /instances/{id}/volumes/{volumeId} From 16e6c206b6ba94a2c2c3ef123654a485e439f299 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 13:40:11 -0400 Subject: [PATCH 08/13] Add periodic auto-standby snapshot sync --- lib/autostandby/README.md | 1 + lib/autostandby/controller.go | 81 +++++++++++++++++++++--------- lib/autostandby/controller_test.go | 38 ++++++++++++++ 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/lib/autostandby/README.md b/lib/autostandby/README.md index feccf8a6..d3ee41c4 100644 --- a/lib/autostandby/README.md +++ b/lib/autostandby/README.md @@ -25,6 +25,7 @@ Hypeman seeds its controller from a conntrack snapshot on startup, then keeps st - new inbound TCP flows are tracked from conntrack `NEW` events - TCP teardown is treated as inactivity once conntrack reports a terminal state or the flow disappears - connections that were already open when Hypeman started are reconciled against fresh conntrack snapshots until they drain, so restart-seeded traffic can still age out correctly +- Hypeman also performs a full snapshot sync every 5 minutes by default as a low-frequency consistency check; the controller interval is configurable When the active inbound TCP connection count reaches zero, Hypeman starts an idle timer for that instance. diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 3294fb59..995ea089 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -19,6 +19,7 @@ const ( trackingModeConntrackEventsV4TCP = "conntrack_events_v4_tcp" defaultReconnectDelay = 2 * time.Second defaultReconcileDelay = 2 * time.Second + defaultSnapshotSyncInterval = 5 * time.Minute ) // InstanceEventAction identifies an instance lifecycle change relevant to auto-standby. @@ -80,12 +81,13 @@ type ConnectionSource interface { // ControllerOptions configures logging, timing, and observability. type ControllerOptions struct { - Log *slog.Logger - Meter metric.Meter - Tracer trace.Tracer - Now func() time.Time - ReconnectDelay time.Duration - ReconcileDelay time.Duration + Log *slog.Logger + Meter metric.Meter + Tracer trace.Tracer + Now func() time.Time + ReconnectDelay time.Duration + ReconcileDelay time.Duration + SnapshotSyncInterval time.Duration } type controllerState struct { @@ -110,11 +112,12 @@ type Controller struct { tracer trace.Tracer metrics *Metrics - reconnectDelay time.Duration - reconcileDelay time.Duration - timerFired chan string - reconcileFired chan string - streamReady chan ConnectionStream + reconnectDelay time.Duration + reconcileDelay time.Duration + snapshotSyncInterval time.Duration + timerFired chan string + reconcileFired chan string + streamReady chan ConnectionStream mu sync.RWMutex states map[string]*controllerState @@ -140,19 +143,24 @@ func NewController(store InstanceStore, source ConnectionSource, opts Controller if reconcileDelay <= 0 { reconcileDelay = defaultReconcileDelay } + snapshotSyncInterval := opts.SnapshotSyncInterval + if snapshotSyncInterval <= 0 { + snapshotSyncInterval = defaultSnapshotSyncInterval + } c := &Controller{ - store: store, - source: source, - log: log, - now: now, - tracer: opts.Tracer, - reconnectDelay: reconnectDelay, - reconcileDelay: reconcileDelay, - timerFired: make(chan string, 128), - reconcileFired: make(chan string, 128), - streamReady: make(chan ConnectionStream, 4), - states: make(map[string]*controllerState), + store: store, + source: source, + log: log, + now: now, + tracer: opts.Tracer, + reconnectDelay: reconnectDelay, + reconcileDelay: reconcileDelay, + snapshotSyncInterval: snapshotSyncInterval, + timerFired: make(chan string, 128), + reconcileFired: make(chan string, 128), + streamReady: make(chan ConnectionStream, 4), + states: make(map[string]*controllerState), } c.metrics = newMetrics(opts.Meter, opts.Tracer, c) return c @@ -161,7 +169,7 @@ func NewController(store InstanceStore, source ConnectionSource, opts Controller // Run starts the controller and blocks until ctx is cancelled. func (c *Controller) Run(ctx context.Context) error { log := c.log.With("tracking_mode", trackingModeConntrackEventsV4TCP) - log.Info("auto-standby controller started") + log.Info("auto-standby controller started", "snapshot_sync_interval", c.snapshotSyncInterval) var stream ConnectionStream if c.source != nil { @@ -190,6 +198,8 @@ func (c *Controller) Run(ctx context.Context) error { if stream != nil { defer stream.Close() } + snapshotTicker := time.NewTicker(c.snapshotSyncInterval) + defer snapshotTicker.Stop() for { var streamEvents <-chan ConnectionEvent @@ -245,6 +255,11 @@ func (c *Controller) Run(ctx context.Context) error { c.handleStandbyTimer(ctx, id) case id := <-c.reconcileFired: c.handleActiveReconcile(ctx, id) + case <-snapshotTicker.C: + if err := c.periodicSnapshotSync(ctx); err != nil { + c.recordControllerError("snapshot_sync") + log.Warn("auto-standby periodic snapshot sync failed", "error", err) + } } } } @@ -384,6 +399,26 @@ func (c *Controller) startupResync(ctx context.Context) error { return nil } +func (c *Controller) periodicSnapshotSync(ctx context.Context) error { + instances, err := c.store.ListInstances(ctx) + if err != nil { + return err + } + conns, err := c.source.ListConnections(ctx) + if err != nil { + return err + } + + now := c.now().UTC() + c.log.Debug("auto-standby periodic snapshot sync completed", "instance_count", len(instances), "current_connection_count", len(conns)) + for _, inst := range instances { + if err := c.seedInstanceState(ctx, inst, conns, now); err != nil { + c.log.Warn("auto-standby periodic snapshot sync failed for instance", "instance_id", inst.ID, "instance_name", inst.Name, "error", err) + } + } + return nil +} + func (c *Controller) seedInstanceState(ctx context.Context, inst Instance, conns []Connection, now time.Time) error { c.mu.Lock() defer c.mu.Unlock() diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index cf763cde..7bb50823 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -152,6 +152,44 @@ func TestStartupResyncResumesPersistedIdleCountdown(t *testing.T) { assert.Equal(t, idleSince.Add(10*time.Minute), *status.NextStandbyAt) } +func TestPeriodicSnapshotSyncRefreshesTrackedState(t *testing.T) { + t.Parallel() + + store := newFakeInstanceStore([]Instance{{ + ID: "inst-periodic", + Name: "inst-periodic", + State: StateRunning, + NetworkEnabled: true, + IP: "192.168.100.21", + AutoStandby: &Policy{Enabled: true, IdleTimeout: "10m"}, + }}) + source := &fakeConnectionSource{} + now := time.Date(2026, 4, 6, 11, 0, 0, 0, time.UTC) + controller := NewController(store, source, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + status := controller.Describe(store.instances[0]) + require.Equal(t, StatusIdleCountdown, status.Status) + + source.connections = []Connection{{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 51235, + OriginalDestinationIP: mustAddr("192.168.100.21"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }} + now = now.Add(time.Minute) + + require.NoError(t, controller.periodicSnapshotSync(context.Background())) + + status = controller.Describe(store.instances[0]) + require.Equal(t, StatusActive, status.Status) + require.Equal(t, 1, status.ActiveInboundCount) +} + func TestConnectionEventsClearIdleAndStartCountdown(t *testing.T) { t.Parallel() From 767b8a2a8c5dbaa3dae45f813d171bd34766c84f Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 13:43:30 -0400 Subject: [PATCH 09/13] Fix auto-standby review follow-ups --- lib/autostandby/conntrack_events_linux.go | 2 +- .../conntrack_events_linux_test.go | 22 +++++++++++ lib/autostandby/controller.go | 39 +++++++++++++------ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/lib/autostandby/conntrack_events_linux.go b/lib/autostandby/conntrack_events_linux.go index 6510cdf1..627d0bc1 100644 --- a/lib/autostandby/conntrack_events_linux.go +++ b/lib/autostandby/conntrack_events_linux.go @@ -134,7 +134,7 @@ func connectionEventFromNetlinkMessage(msg syscall.NetlinkMessage) (ConnectionEv return ConnectionEvent{}, false, nil case unix.NLMSG_ERROR: if len(msg.Data) >= 4 { - errno := -int32(binary.LittleEndian.Uint32(msg.Data[:4])) + errno := -int32(nl.NativeEndian().Uint32(msg.Data[:4])) if errno != 0 { return ConnectionEvent{}, false, unix.Errno(errno) } diff --git a/lib/autostandby/conntrack_events_linux_test.go b/lib/autostandby/conntrack_events_linux_test.go index 65604e50..935ddaf9 100644 --- a/lib/autostandby/conntrack_events_linux_test.go +++ b/lib/autostandby/conntrack_events_linux_test.go @@ -3,11 +3,14 @@ package autostandby import ( + "encoding/binary" "syscall" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" ) func TestConnectionEventFromNetlinkMessageParsesIPv4TCPEvent(t *testing.T) { @@ -45,3 +48,22 @@ func TestConnectionEventFromNetlinkMessageParsesIPv4TCPEvent(t *testing.T) { assert.Equal(t, uint16(3333), event.Connection.OriginalDestinationPort) assert.Equal(t, TCPStateClose, event.Connection.TCPState) } + +func TestConnectionEventFromNetlinkMessageParsesNativeEndianNLMSGError(t *testing.T) { + t.Parallel() + + data := make([]byte, 4) + nl.NativeEndian().PutUint32(data, uint32(int32(-unix.EPERM))) + + _, ok, err := connectionEventFromNetlinkMessage(syscall.NetlinkMessage{ + Header: syscall.NlMsghdr{Type: unix.NLMSG_ERROR}, + Data: data, + }) + require.ErrorIs(t, err, unix.EPERM) + require.False(t, ok) + + // Sanity-check the fixture is using native byte order rather than an accidental little-endian match. + if nl.NativeEndian() != binary.LittleEndian { + require.NotEqual(t, binary.LittleEndian.Uint32(data), nl.NativeEndian().Uint32(data)) + } +} diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 995ea089..6b47dda0 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -314,25 +314,42 @@ func (c *Controller) Describe(inst Instance) StatusSnapshot { snapshot.Eligible = true + var ( + activeInboundCount int + idleSince *time.Time + lastInboundAt *time.Time + nextStandbyAt *time.Time + standbyRequested bool + hasState bool + ) + c.mu.RLock() state := c.states[inst.ID] observerConnected := c.observerConnected lastObserverErr := c.lastObserverErr + if state != nil { + hasState = true + activeInboundCount = len(state.activeInbound) + idleSince = cloneTimePtr(state.idleSince) + lastInboundAt = cloneTimePtr(state.lastInboundAt) + nextStandbyAt = cloneTimePtr(state.nextStandbyAt) + standbyRequested = state.standbyRequested + } c.mu.RUnlock() - if state != nil { - snapshot.ActiveInboundCount = len(state.activeInbound) - snapshot.IdleSince = cloneTimePtr(state.idleSince) - snapshot.LastInboundActivityAt = cloneTimePtr(state.lastInboundAt) - snapshot.NextStandbyAt = cloneTimePtr(state.nextStandbyAt) - if state.nextStandbyAt != nil { - remaining := state.nextStandbyAt.Sub(c.now().UTC()) + if hasState { + snapshot.ActiveInboundCount = activeInboundCount + snapshot.IdleSince = idleSince + snapshot.LastInboundActivityAt = lastInboundAt + snapshot.NextStandbyAt = nextStandbyAt + if nextStandbyAt != nil { + remaining := nextStandbyAt.Sub(c.now().UTC()) if remaining < 0 { remaining = 0 } snapshot.CountdownRemaining = &remaining } - if state.standbyRequested { + if standbyRequested { snapshot.Status = StatusStandbyRequested snapshot.Reason = ReasonReadyForStandby return snapshot @@ -344,13 +361,13 @@ func (c *Controller) Describe(inst Instance) StatusSnapshot { snapshot.Reason = ReasonObserverError return snapshot } - if state != nil && len(state.activeInbound) > 0 { + if hasState && activeInboundCount > 0 { snapshot.Status = StatusActive snapshot.Reason = ReasonActiveInbound return snapshot } - if state != nil && state.nextStandbyAt != nil { - if state.nextStandbyAt.After(c.now().UTC()) { + if hasState && nextStandbyAt != nil { + if nextStandbyAt.After(c.now().UTC()) { snapshot.Status = StatusIdleCountdown snapshot.Reason = ReasonIdleTimeoutNotElapsed return snapshot From 73758cd0879c81ef5403405bed780701c8bc4532 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 13:45:41 -0400 Subject: [PATCH 10/13] Format auto-standby files for CI --- cmd/api/api/auto_standby_status.go | 1 - lib/autostandby/metrics.go | 15 +++++++-------- lib/autostandby/status.go | 15 +++++++-------- lib/autostandby/types_test.go | 1 - lib/instances/lifecycle_events.go | 1 - lib/instances/metadata_clone.go | 2 +- lib/providers/auto_standby_linux.go | 1 - 7 files changed, 15 insertions(+), 21 deletions(-) diff --git a/cmd/api/api/auto_standby_status.go b/cmd/api/api/auto_standby_status.go index a9fda3c3..5393a1e8 100644 --- a/cmd/api/api/auto_standby_status.go +++ b/cmd/api/api/auto_standby_status.go @@ -77,4 +77,3 @@ func toOAPIAutoStandbyStatus(status autostandby.StatusSnapshot) oapi.AutoStandby } return out } - diff --git a/lib/autostandby/metrics.go b/lib/autostandby/metrics.go index 91d34213..9d29a76f 100644 --- a/lib/autostandby/metrics.go +++ b/lib/autostandby/metrics.go @@ -11,13 +11,13 @@ import ( ) type Metrics struct { - conntrackEventsTotal metric.Int64Counter - startupResyncDuration metric.Float64Histogram - standbyAttemptsTotal metric.Int64Counter - controllerErrorsTotal metric.Int64Counter - trackedInstancesGauge metric.Int64ObservableGauge - activeConnectionsGauge metric.Int64ObservableGauge - tracer trace.Tracer + conntrackEventsTotal metric.Int64Counter + startupResyncDuration metric.Float64Histogram + standbyAttemptsTotal metric.Int64Counter + controllerErrorsTotal metric.Int64Counter + trackedInstancesGauge metric.Int64ObservableGauge + activeConnectionsGauge metric.Int64ObservableGauge + tracer trace.Tracer } func newMetrics(meter metric.Meter, tracer trace.Tracer, controller *Controller) *Metrics { @@ -161,4 +161,3 @@ func (c *Controller) metricSnapshot() (active, countdown, ready, ineligible, tot } return } - diff --git a/lib/autostandby/status.go b/lib/autostandby/status.go index 689493c0..bb48be57 100644 --- a/lib/autostandby/status.go +++ b/lib/autostandby/status.go @@ -5,14 +5,14 @@ import "time" type Status string const ( - StatusUnsupported Status = "unsupported" - StatusDisabled Status = "disabled" - StatusIneligible Status = "ineligible" - StatusActive Status = "active" - StatusIdleCountdown Status = "idle_countdown" - StatusReadyForStandby Status = "ready_for_standby" + StatusUnsupported Status = "unsupported" + StatusDisabled Status = "disabled" + StatusIneligible Status = "ineligible" + StatusActive Status = "active" + StatusIdleCountdown Status = "idle_countdown" + StatusReadyForStandby Status = "ready_for_standby" StatusStandbyRequested Status = "standby_requested" - StatusError Status = "error" + StatusError Status = "error" ) type Reason string @@ -47,4 +47,3 @@ type StatusSnapshot struct { CountdownRemaining *time.Duration TrackingMode string } - diff --git a/lib/autostandby/types_test.go b/lib/autostandby/types_test.go index 39b856b8..a77bdb0d 100644 --- a/lib/autostandby/types_test.go +++ b/lib/autostandby/types_test.go @@ -12,4 +12,3 @@ func TestTCPStateConstantsMatchExpectedKernelValues(t *testing.T) { assert.Equal(t, TCPState(10), TCPStateIgnore) assert.Equal(t, TCPState(11), TCPStateRetrans) } - diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go index 149aa4f5..ec32a1cf 100644 --- a/lib/instances/lifecycle_events.go +++ b/lib/instances/lifecycle_events.go @@ -71,4 +71,3 @@ func trySendLifecycleEvent(ch chan LifecycleEvent, event LifecycleEvent) { default: } } - diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go index fdb53796..d6efa4b5 100644 --- a/lib/instances/metadata_clone.go +++ b/lib/instances/metadata_clone.go @@ -8,7 +8,7 @@ func deepCopyMetadata(src *metadata) *metadata { } return &metadata{ - StoredMetadata: cloneStoredMetadata(src.StoredMetadata), + StoredMetadata: cloneStoredMetadata(src.StoredMetadata), AutoStandbyRuntime: cloneAutoStandbyRuntime(src.AutoStandbyRuntime), } } diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go index 2a1063b2..f5961888 100644 --- a/lib/providers/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -110,4 +110,3 @@ func ProvideAutoStandbyController(instanceManager instances.Manager, log *slog.L }, ) } - From e9bf3339a8d2ad3f3740e9c059433754806ea307 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 15:13:42 -0400 Subject: [PATCH 11/13] Add scope for auto-standby status route --- lib/scopes/scopes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/scopes/scopes.go b/lib/scopes/scopes.go index e92ecde0..f41aafb1 100644 --- a/lib/scopes/scopes.go +++ b/lib/scopes/scopes.go @@ -243,6 +243,7 @@ var RouteScopes = map[string]Scope{ "POST /instances/{id}/start": InstanceWrite, "GET /instances/{id}/stat": InstanceRead, "GET /instances/{id}/stats": InstanceRead, + "GET /instances/{id}/auto-standby/status": InstanceRead, "GET /instances/{id}/wait": InstanceRead, "POST /instances/{id}/stop": InstanceWrite, "PATCH /instances/{id}": InstanceWrite, From 53f201799c87ad268ebddd4872eaa2314ea6f927 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 15:57:07 -0400 Subject: [PATCH 12/13] Consolidate instance lifecycle subscriptions --- lib/builds/manager_test.go | 4 +- lib/instances/README.md | 4 +- lib/instances/lifecycle_events.go | 136 +++++++++++++++++++++++++ lib/instances/lifecycle_events_test.go | 125 +++++++++++++++++++++++ lib/instances/manager.go | 69 +++++++++---- lib/instances/metrics.go | 62 +++++++++++ lib/instances/metrics_test.go | 101 ++++++++++++++++++ lib/instances/subscribe.go | 83 --------------- lib/instances/subscribe_test.go | 97 ------------------ lib/instances/wait.go | 24 +++-- lib/instances/wait_test.go | 113 +++++++++++++++++--- 11 files changed, 589 insertions(+), 229 deletions(-) create mode 100644 lib/instances/lifecycle_events.go create mode 100644 lib/instances/lifecycle_events_test.go delete mode 100644 lib/instances/subscribe.go delete mode 100644 lib/instances/subscribe_test.go diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index e0709e6f..3b6a3e34 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -171,8 +171,8 @@ func (m *mockInstanceManager) GetVsockDialer(ctx context.Context, instanceID str return nil, nil } -func (m *mockInstanceManager) Subscribe(instanceID string) (<-chan instances.StateChange, func()) { - ch := make(chan instances.StateChange, 1) +func (m *mockInstanceManager) SubscribeLifecycleEvents(consumer instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) { + ch := make(chan instances.LifecycleEvent, 1) return ch, func() { close(ch) } } diff --git a/lib/instances/README.md b/lib/instances/README.md index 77fdd3ae..b56e276d 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -130,9 +130,9 @@ Any State → Stopped - If an instance is deleted, its schedule is retained so retention can continue cleaning existing scheduled snapshots. - Once the deleted instance has no scheduled snapshots left, the scheduler removes that schedule automatically. -## WaitForState (wait.go, subscribe.go) +## WaitForState (wait.go, lifecycle_events.go) -Waits for an instance to reach a target state using channel-based subscriptions with a polling fallback. State-changing manager methods (start, stop, standby, restore, fork, delete) broadcast via `notifyStateChange`, which fans out to subscriber channels. `WaitForState` uses a 3-way select: subscription events, 5s polling fallback (logs at debug level if it detects the state change), and context cancellation/timeout. Returns early on terminal (`Stopped`, `Standby`, `Shutdown`) or error (`Unknown`) states. Used by `GET /instances/{id}/wait`. +Waits for an instance to reach a target state using the shared lifecycle event bus with a polling fallback. State-changing manager methods publish lifecycle events after successful mutations, and `WaitForState` filters them by `instance_id`. A 5s polling fallback guards against missed or dropped events. Returns early on terminal (`Stopped`, `Standby`, `Shutdown`) or error (`Unknown`) states. Used by `GET /instances/{id}/wait`. ## Reference Handling diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go new file mode 100644 index 00000000..ad14cb00 --- /dev/null +++ b/lib/instances/lifecycle_events.go @@ -0,0 +1,136 @@ +package instances + +import ( + "context" + "sync" +) + +const lifecycleEventBufferSize = 32 + +// LifecycleEventConsumer identifies the internal consumer of lifecycle events. +// Keep this set bounded for observability label safety. +type LifecycleEventConsumer string + +const ( + LifecycleEventConsumerWaitForState LifecycleEventConsumer = "wait_for_state" + LifecycleEventConsumerAutoStandby LifecycleEventConsumer = "auto_standby" +) + +var allLifecycleEventConsumers = []LifecycleEventConsumer{ + LifecycleEventConsumerWaitForState, + LifecycleEventConsumerAutoStandby, +} + +// LifecycleEventAction identifies which instance lifecycle action occurred. +type LifecycleEventAction string + +const ( + LifecycleEventCreate LifecycleEventAction = "create" + LifecycleEventUpdate LifecycleEventAction = "update" + LifecycleEventStart LifecycleEventAction = "start" + LifecycleEventStop LifecycleEventAction = "stop" + LifecycleEventStandby LifecycleEventAction = "standby" + LifecycleEventRestore LifecycleEventAction = "restore" + LifecycleEventDelete LifecycleEventAction = "delete" + LifecycleEventFork LifecycleEventAction = "fork" +) + +// LifecycleEvent is a global instance change event stream used by internal +// consumers such as wait-for-state and background controllers. +type LifecycleEvent struct { + Action LifecycleEventAction + InstanceID string + Instance *Instance +} + +type lifecycleSubscriber struct { + consumer LifecycleEventConsumer + ch chan LifecycleEvent +} + +type lifecycleConsumerStats struct { + Subscribers int64 + MaxQueueDepth int64 +} + +type lifecycleSubscribers struct { + mu sync.Mutex + subs []lifecycleSubscriber + onDrop func(context.Context, LifecycleEventConsumer) +} + +func newLifecycleSubscribers() *lifecycleSubscribers { + return &lifecycleSubscribers{} +} + +func (s *lifecycleSubscribers) Subscribe(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) { + ch := make(chan LifecycleEvent, lifecycleEventBufferSize) + + s.mu.Lock() + s.subs = append(s.subs, lifecycleSubscriber{ + consumer: consumer, + ch: ch, + }) + s.mu.Unlock() + + return ch, func() { + s.mu.Lock() + defer s.mu.Unlock() + for i, sub := range s.subs { + if sub.ch == ch { + s.subs = append(s.subs[:i], s.subs[i+1:]...) + close(ch) + break + } + } + } +} + +func (s *lifecycleSubscribers) Notify(ctx context.Context, event LifecycleEvent) { + s.mu.Lock() + subs := append([]lifecycleSubscriber(nil), s.subs...) + s.mu.Unlock() + + for _, sub := range subs { + if trySendLifecycleEvent(sub.ch, event) && s.onDrop != nil { + s.onDrop(ctx, sub.consumer) + } + } +} + +func (s *lifecycleSubscribers) Stats() map[LifecycleEventConsumer]lifecycleConsumerStats { + s.mu.Lock() + defer s.mu.Unlock() + + stats := make(map[LifecycleEventConsumer]lifecycleConsumerStats, len(allLifecycleEventConsumers)) + for _, consumer := range allLifecycleEventConsumers { + stats[consumer] = lifecycleConsumerStats{} + } + for _, sub := range s.subs { + stat := stats[sub.consumer] + stat.Subscribers++ + queueDepth := int64(len(sub.ch)) + if queueDepth > stat.MaxQueueDepth { + stat.MaxQueueDepth = queueDepth + } + stats[sub.consumer] = stat + } + return stats +} + +// trySendLifecycleEvent attempts a non-blocking send. +// Returns true when the event was dropped because the channel buffer was full. +func trySendLifecycleEvent(ch chan LifecycleEvent, event LifecycleEvent) (dropped bool) { + defer func() { + if recover() != nil { + dropped = false + } + }() + + select { + case ch <- event: + return false + default: + return true + } +} diff --git a/lib/instances/lifecycle_events_test.go b/lib/instances/lifecycle_events_test.go new file mode 100644 index 00000000..0cf0c1c7 --- /dev/null +++ b/lib/instances/lifecycle_events_test.go @@ -0,0 +1,125 @@ +package instances + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLifecycleSubscribers_NotifyDelivers(t *testing.T) { + s := newLifecycleSubscribers() + ch, unsub := s.Subscribe(LifecycleEventConsumerWaitForState) + defer unsub() + + s.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStart, + InstanceID: "inst-1", + Instance: &Instance{State: StateRunning}, + }) + + select { + case event := <-ch: + assert.Equal(t, LifecycleEventStart, event.Action) + require.NotNil(t, event.Instance) + assert.Equal(t, StateRunning, event.Instance.State) + case <-time.After(time.Second): + t.Fatal("timed out waiting for lifecycle event") + } +} + +func TestLifecycleSubscribers_MultipleSubscribers(t *testing.T) { + s := newLifecycleSubscribers() + ch1, unsub1 := s.Subscribe(LifecycleEventConsumerWaitForState) + defer unsub1() + ch2, unsub2 := s.Subscribe(LifecycleEventConsumerAutoStandby) + defer unsub2() + + s.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStop, + InstanceID: "inst-1", + Instance: &Instance{State: StateStopped}, + }) + + for _, ch := range []<-chan LifecycleEvent{ch1, ch2} { + select { + case event := <-ch: + assert.Equal(t, LifecycleEventStop, event.Action) + require.NotNil(t, event.Instance) + assert.Equal(t, StateStopped, event.Instance.State) + case <-time.After(time.Second): + t.Fatal("timed out waiting for lifecycle event") + } + } +} + +func TestLifecycleSubscribers_UnsubscribeStopsDelivery(t *testing.T) { + s := newLifecycleSubscribers() + ch, unsub := s.Subscribe(LifecycleEventConsumerWaitForState) + unsub() + + _, ok := <-ch + assert.False(t, ok, "channel should be closed after unsubscribe") +} + +func TestLifecycleSubscribers_StatsByConsumer(t *testing.T) { + s := newLifecycleSubscribers() + wait1, unsub1 := s.Subscribe(LifecycleEventConsumerWaitForState) + defer unsub1() + wait2, unsub2 := s.Subscribe(LifecycleEventConsumerWaitForState) + defer unsub2() + auto, unsub3 := s.Subscribe(LifecycleEventConsumerAutoStandby) + defer unsub3() + + for i := 0; i < 3; i++ { + s.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventUpdate, + InstanceID: "inst-1", + Instance: &Instance{State: StateRunning}, + }) + } + <-wait1 + <-wait2 + + stats := s.Stats() + assert.Equal(t, int64(2), stats[LifecycleEventConsumerWaitForState].Subscribers) + assert.Equal(t, int64(2), stats[LifecycleEventConsumerWaitForState].MaxQueueDepth) + assert.Equal(t, int64(1), stats[LifecycleEventConsumerAutoStandby].Subscribers) + assert.Equal(t, int64(3), stats[LifecycleEventConsumerAutoStandby].MaxQueueDepth) + assert.Equal(t, 3, len(auto)) +} + +func TestLifecycleSubscribers_DropCallbackOnBackpressure(t *testing.T) { + s := newLifecycleSubscribers() + + drops := make(chan LifecycleEventConsumer, 1) + s.onDrop = func(ctx context.Context, consumer LifecycleEventConsumer) { + drops <- consumer + } + + _, unsub := s.Subscribe(LifecycleEventConsumerWaitForState) + defer unsub() + + for i := 0; i < lifecycleEventBufferSize; i++ { + s.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventUpdate, + InstanceID: "inst-1", + Instance: &Instance{State: StateRunning}, + }) + } + + s.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStart, + InstanceID: "inst-1", + Instance: &Instance{State: StateRunning}, + }) + + select { + case consumer := <-drops: + assert.Equal(t, LifecycleEventConsumerWaitForState, consumer) + case <-time.After(time.Second): + t.Fatal("expected drop callback") + } +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 4ef8d9bf..accb9741 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -57,9 +57,8 @@ type Manager interface { SetResourceValidator(v ResourceValidator) // GetVsockDialer returns a VsockDialer for the specified instance. GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) - // Subscribe returns a channel that receives state change events for the - // given instance, plus an unsubscribe function the caller must defer. - Subscribe(instanceID string) (<-chan StateChange, func()) + // SubscribeLifecycleEvents returns the shared internal lifecycle event stream. + SubscribeLifecycleEvents(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) } // ImageUsageRecorder records newly used images before instance metadata is persisted. @@ -115,8 +114,8 @@ type manager struct { nativeCodecPaths map[string]string imageUsageRecorder ImageUsageRecorder - // State change subscriptions for waitForState - stateSubscribers *subscribers + // Shared lifecycle event subscriptions for internal consumers. + lifecycleEvents *lifecycleSubscribers // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -169,7 +168,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste snapshotDefaults: snapshotDefaults, compressionJobs: make(map[string]*compressionJob), nativeCodecPaths: make(map[string]string), - stateSubscribers: newSubscribers(), + lifecycleEvents: newLifecycleSubscribers(), } m.deleteSnapshotFn = m.deleteSnapshot @@ -180,6 +179,9 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste m.metrics = metrics } } + m.lifecycleEvents.onDrop = func(ctx context.Context, consumer LifecycleEventConsumer) { + m.recordLifecycleEventDropped(ctx, consumer, lifecycleEventDropReasonBufferFull) + } return m } @@ -195,15 +197,25 @@ func (m *manager) SetImageUsageRecorder(recorder ImageUsageRecorder) { m.imageUsageRecorder = recorder } -func (m *manager) Subscribe(instanceID string) (<-chan StateChange, func()) { - return m.stateSubscribers.Subscribe(instanceID) +func (m *manager) SubscribeLifecycleEvents(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) { + return m.lifecycleEvents.Subscribe(consumer) +} + +func (m *manager) notifyLifecycleEvent(ctx context.Context, action LifecycleEventAction, inst *Instance) { + if inst == nil { + return + } + m.lifecycleEvents.Notify(ctx, LifecycleEvent{ + Action: action, + InstanceID: inst.Id, + Instance: inst, + }) } -// notifyStateChange broadcasts a state change to all subscribers for the instance. -func (m *manager) notifyStateChange(instanceID string, inst *Instance) { - m.stateSubscribers.Notify(instanceID, StateChange{ - State: inst.State, - StateError: inst.StateError, +func (m *manager) notifyLifecycleDelete(ctx context.Context, instanceID string) { + m.lifecycleEvents.Notify(ctx, LifecycleEvent{ + Action: LifecycleEventDelete, + InstanceID: instanceID, }) } @@ -270,7 +282,11 @@ func (m *manager) CreateInstance(ctx context.Context, req CreateInstanceRequest) // 1. ULID generation is unique // 2. Filesystem mkdir is atomic per instance directory // 3. Concurrent creates of different instances don't conflict - return m.createInstance(ctx, req) + inst, err := m.createInstance(ctx, req) + if err == nil { + m.notifyLifecycleEvent(ctx, LifecycleEventCreate, inst) + } + return inst, err } // DeleteInstance stops and deletes an instance @@ -281,7 +297,7 @@ func (m *manager) DeleteInstance(ctx context.Context, id string) error { err := m.deleteInstance(ctx, id) if err == nil { - m.stateSubscribers.CloseAll(id) + m.notifyLifecycleDelete(ctx, id) // Clean up the lock after successful deletion m.instanceLocks.Delete(id) } @@ -332,11 +348,16 @@ func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceR return nil, fmt.Errorf("wait for fork guest agent readiness: %w", err) } } + m.notifyLifecycleEvent(ctx, LifecycleEventFork, inst) return inst, nil } func (m *manager) ForkSnapshot(ctx context.Context, snapshotID string, req ForkSnapshotRequest) (*Instance, error) { - return m.forkSnapshot(ctx, snapshotID, req) + inst, err := m.forkSnapshot(ctx, snapshotID, req) + if err == nil { + m.notifyLifecycleEvent(ctx, LifecycleEventFork, inst) + } + return inst, err } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) @@ -346,7 +367,7 @@ func (m *manager) StandbyInstance(ctx context.Context, id string, req StandbyIns defer lock.Unlock() inst, err := m.standbyInstance(ctx, id, req, false) if err == nil { - m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(ctx, LifecycleEventStandby, inst) } return inst, err } @@ -358,7 +379,7 @@ func (m *manager) RestoreInstance(ctx context.Context, id string) (*Instance, er defer lock.Unlock() inst, err := m.restoreInstance(ctx, id) if err == nil { - m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(ctx, LifecycleEventRestore, inst) } return inst, err } @@ -369,7 +390,7 @@ func (m *manager) RestoreSnapshot(ctx context.Context, id string, snapshotID str defer lock.Unlock() inst, err := m.restoreSnapshot(ctx, id, snapshotID, req) if err == nil { - m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(ctx, LifecycleEventRestore, inst) } return inst, err } @@ -381,7 +402,7 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error defer lock.Unlock() inst, err := m.stopInstance(ctx, id) if err == nil { - m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(ctx, LifecycleEventStop, inst) } return inst, err } @@ -393,7 +414,7 @@ func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanc defer lock.Unlock() inst, err := m.startInstance(ctx, id, req) if err == nil { - m.notifyStateChange(id, inst) + m.notifyLifecycleEvent(ctx, LifecycleEventStart, inst) } return inst, err } @@ -403,7 +424,11 @@ func (m *manager) UpdateInstance(ctx context.Context, id string, req UpdateInsta lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.updateInstance(ctx, id, req) + inst, err := m.updateInstance(ctx, id, req) + if err == nil { + m.notifyLifecycleEvent(ctx, LifecycleEventUpdate, inst) + } + return inst, err } // ListInstances returns instances, optionally filtered by the given criteria. diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index bee79d1a..1b652fca 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -61,6 +61,10 @@ const ( snapshotCodecFallbackReasonNotExecutable snapshotCodecFallbackReason = "not_executable" ) +type lifecycleEventDropReason string + +const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full" + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -78,6 +82,7 @@ type Metrics struct { snapshotRestoreMemoryPrepareTotal metric.Int64Counter snapshotRestoreMemoryPrepareDuration metric.Float64Histogram snapshotCompressionPreemptionsTotal metric.Int64Counter + lifecycleEventsDroppedTotal metric.Int64Counter tracer trace.Tracer } @@ -220,6 +225,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + lifecycleEventsDroppedTotal, err := meter.Int64Counter( + "hypeman_instances_lifecycle_events_dropped_total", + metric.WithDescription("Total number of lifecycle events dropped because subscriber buffers were full"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -246,6 +259,22 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + lifecycleSubscribersTotal, err := meter.Int64ObservableGauge( + "hypeman_instances_lifecycle_subscribers_total", + metric.WithDescription("Current number of lifecycle event subscribers by consumer"), + ) + if err != nil { + return nil, err + } + + lifecycleSubscriberQueueDepth, err := meter.Int64ObservableGauge( + "hypeman_instances_lifecycle_subscriber_queue_depth", + metric.WithDescription("Maximum buffered lifecycle events across subscribers for each consumer"), + ) + if err != nil { + return nil, err + } + _, err = meter.RegisterCallback( func(ctx context.Context, o metric.Observer) error { instances, err := m.listInstances(ctx) @@ -332,6 +361,27 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + stats := make(map[LifecycleEventConsumer]lifecycleConsumerStats, len(allLifecycleEventConsumers)) + if m.lifecycleEvents != nil { + stats = m.lifecycleEvents.Stats() + } + for _, consumer := range allLifecycleEventConsumers { + stat := stats[consumer] + attrs := metric.WithAttributes(attribute.String("consumer", string(consumer))) + o.ObserveInt64(lifecycleSubscribersTotal, stat.Subscribers, attrs) + o.ObserveInt64(lifecycleSubscriberQueueDepth, stat.MaxQueueDepth, attrs) + } + return nil + }, + lifecycleSubscribersTotal, + lifecycleSubscriberQueueDepth, + ) + if err != nil { + return nil, err + } + return &Metrics{ createDuration: createDuration, restoreDuration: restoreDuration, @@ -348,6 +398,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M snapshotRestoreMemoryPrepareTotal: snapshotRestoreMemoryPrepareTotal, snapshotRestoreMemoryPrepareDuration: snapshotRestoreMemoryPrepareDuration, snapshotCompressionPreemptionsTotal: snapshotCompressionPreemptionsTotal, + lifecycleEventsDroppedTotal: lifecycleEventsDroppedTotal, tracer: tracer, }, nil } @@ -534,3 +585,14 @@ func (m *manager) recordSnapshotCodecFallback(ctx context.Context, algorithm sna attribute.String("reason", string(reason)), )) } + +func (m *manager) recordLifecycleEventDropped(ctx context.Context, consumer LifecycleEventConsumer, reason lifecycleEventDropReason) { + if m.metrics == nil { + return + } + + m.metrics.lifecycleEventsDroppedTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("consumer", string(consumer)), + attribute.String("reason", string(reason)), + )) +} diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 88def215..b5681fd0 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -1,6 +1,7 @@ package instances import ( + "context" "os" "path/filepath" "testing" @@ -115,6 +116,106 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assert.Equal(t, "standby", metricLabel(t, active.DataPoints[0].Attributes, "source")) } +func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + lifecycleEvents: newLifecycleSubscribers(), + } + + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + m.lifecycleEvents.onDrop = func(ctx context.Context, consumer LifecycleEventConsumer) { + m.recordLifecycleEventDropped(ctx, consumer, lifecycleEventDropReasonBufferFull) + } + + waitCh, waitUnsub := m.SubscribeLifecycleEvents(LifecycleEventConsumerWaitForState) + defer waitUnsub() + autoCh, autoUnsub := m.SubscribeLifecycleEvents(LifecycleEventConsumerAutoStandby) + defer autoUnsub() + + for i := 0; i < 3; i++ { + m.lifecycleEvents.Notify(t.Context(), LifecycleEvent{ + Action: LifecycleEventUpdate, + InstanceID: "inst-1", + Instance: &Instance{State: StateRunning}, + }) + } + <-waitCh + + for i := 0; i < lifecycleEventBufferSize; i++ { + m.lifecycleEvents.Notify(t.Context(), LifecycleEvent{ + Action: LifecycleEventUpdate, + InstanceID: "inst-1", + Instance: &Instance{State: StateRunning}, + }) + } + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + assertMetricNames(t, rm, []string{ + "hypeman_instances_lifecycle_subscribers_total", + "hypeman_instances_lifecycle_subscriber_queue_depth", + "hypeman_instances_lifecycle_events_dropped_total", + }) + + subscribersMetric := findMetric(t, rm, "hypeman_instances_lifecycle_subscribers_total") + subscribers, ok := subscribersMetric.Data.(metricdata.Gauge[int64]) + require.True(t, ok) + require.Len(t, subscribers.DataPoints, len(allLifecycleEventConsumers)) + for _, point := range subscribers.DataPoints { + switch metricLabel(t, point.Attributes, "consumer") { + case string(LifecycleEventConsumerWaitForState): + assert.Equal(t, int64(1), point.Value) + case string(LifecycleEventConsumerAutoStandby): + assert.Equal(t, int64(1), point.Value) + default: + t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) + } + } + + queueDepthMetric := findMetric(t, rm, "hypeman_instances_lifecycle_subscriber_queue_depth") + queueDepth, ok := queueDepthMetric.Data.(metricdata.Gauge[int64]) + require.True(t, ok) + require.Len(t, queueDepth.DataPoints, len(allLifecycleEventConsumers)) + for _, point := range queueDepth.DataPoints { + switch metricLabel(t, point.Attributes, "consumer") { + case string(LifecycleEventConsumerWaitForState): + assert.Equal(t, int64(lifecycleEventBufferSize), point.Value) + case string(LifecycleEventConsumerAutoStandby): + assert.Equal(t, int64(lifecycleEventBufferSize), point.Value) + default: + t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) + } + } + + droppedMetric := findMetric(t, rm, "hypeman_instances_lifecycle_events_dropped_total") + dropped, ok := droppedMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.NotEmpty(t, dropped.DataPoints) + + var waitDrops int64 + var autoDrops int64 + for _, point := range dropped.DataPoints { + assert.Equal(t, string(lifecycleEventDropReasonBufferFull), metricLabel(t, point.Attributes, "reason")) + switch metricLabel(t, point.Attributes, "consumer") { + case string(LifecycleEventConsumerWaitForState): + waitDrops += point.Value + case string(LifecycleEventConsumerAutoStandby): + autoDrops += point.Value + } + } + assert.Greater(t, waitDrops, int64(0)) + assert.Greater(t, autoDrops, int64(0)) + assert.Equal(t, lifecycleEventBufferSize, len(autoCh)) +} + func TestInstanceOldestInStateMetric_ObserveOldestAgePerState(t *testing.T) { t.Parallel() diff --git a/lib/instances/subscribe.go b/lib/instances/subscribe.go deleted file mode 100644 index f401f89d..00000000 --- a/lib/instances/subscribe.go +++ /dev/null @@ -1,83 +0,0 @@ -package instances - -import "sync" - -// StateChange represents a state transition for an instance. -type StateChange struct { - State State - StateError *string -} - -// subscribers manages per-instance state change subscriptions. -type subscribers struct { - mu sync.Mutex - subs map[string][]chan StateChange -} - -func newSubscribers() *subscribers { - return &subscribers{ - subs: make(map[string][]chan StateChange), - } -} - -// Subscribe returns a channel that receives state changes for the given -// instance and an unsubscribe function. The channel is buffered (16) to -// avoid blocking the notifier on slow consumers. -func (s *subscribers) Subscribe(instanceID string) (<-chan StateChange, func()) { - ch := make(chan StateChange, 16) - s.mu.Lock() - s.subs[instanceID] = append(s.subs[instanceID], ch) - s.mu.Unlock() - - return ch, func() { - s.mu.Lock() - defer s.mu.Unlock() - chans := s.subs[instanceID] - for i, c := range chans { - if c == ch { - s.subs[instanceID] = append(chans[:i], chans[i+1:]...) - close(ch) - break - } - } - if len(s.subs[instanceID]) == 0 { - delete(s.subs, instanceID) - } - } -} - -// Notify fans out a state change to all subscribers for the given instance. -// Non-blocking: drops the event if a subscriber's buffer is full. -// Safe to call concurrently with CloseAll. -func (s *subscribers) Notify(instanceID string, sc StateChange) { - s.mu.Lock() - chans := make([]chan StateChange, len(s.subs[instanceID])) - copy(chans, s.subs[instanceID]) - s.mu.Unlock() - - for _, ch := range chans { - trySend(ch, sc) - } -} - -// trySend attempts a non-blocking send on ch, recovering from a panic if the -// channel was closed by a concurrent CloseAll. -func trySend(ch chan StateChange, sc StateChange) { - defer func() { recover() }() - select { - case ch <- sc: - default: - } -} - -// CloseAll closes and removes all subscriber channels for the given instance. -func (s *subscribers) CloseAll(instanceID string) { - s.mu.Lock() - chans := s.subs[instanceID] - delete(s.subs, instanceID) - s.mu.Unlock() - - for _, ch := range chans { - close(ch) - } -} diff --git a/lib/instances/subscribe_test.go b/lib/instances/subscribe_test.go deleted file mode 100644 index 667e09ab..00000000 --- a/lib/instances/subscribe_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package instances - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSubscribers_NotifyDelivers(t *testing.T) { - s := newSubscribers() - ch, unsub := s.Subscribe("inst-1") - defer unsub() - - s.Notify("inst-1", StateChange{State: StateRunning}) - - select { - case sc := <-ch: - assert.Equal(t, StateRunning, sc.State) - case <-time.After(time.Second): - t.Fatal("timed out waiting for state change") - } -} - -func TestSubscribers_MultipleSubscribers(t *testing.T) { - s := newSubscribers() - ch1, unsub1 := s.Subscribe("inst-1") - defer unsub1() - ch2, unsub2 := s.Subscribe("inst-1") - defer unsub2() - - s.Notify("inst-1", StateChange{State: StateStopped}) - - for _, ch := range []<-chan StateChange{ch1, ch2} { - select { - case sc := <-ch: - assert.Equal(t, StateStopped, sc.State) - case <-time.After(time.Second): - t.Fatal("timed out waiting for state change") - } - } -} - -func TestSubscribers_UnsubscribeStopsDelivery(t *testing.T) { - s := newSubscribers() - ch, unsub := s.Subscribe("inst-1") - unsub() - - // Channel should be closed after unsubscribe. - _, ok := <-ch - assert.False(t, ok, "channel should be closed after unsubscribe") -} - -func TestSubscribers_DifferentInstancesIsolated(t *testing.T) { - s := newSubscribers() - ch1, unsub1 := s.Subscribe("inst-1") - defer unsub1() - ch2, unsub2 := s.Subscribe("inst-2") - defer unsub2() - - s.Notify("inst-1", StateChange{State: StateRunning}) - - select { - case sc := <-ch1: - assert.Equal(t, StateRunning, sc.State) - case <-time.After(time.Second): - t.Fatal("inst-1 subscriber should have received event") - } - - select { - case <-ch2: - t.Fatal("inst-2 subscriber should not have received event") - case <-time.After(50 * time.Millisecond): - // expected - } -} - -func TestSubscribers_CloseAll(t *testing.T) { - s := newSubscribers() - ch1, _ := s.Subscribe("inst-1") - ch2, _ := s.Subscribe("inst-1") - - s.CloseAll("inst-1") - - _, ok1 := <-ch1 - _, ok2 := <-ch2 - assert.False(t, ok1, "ch1 should be closed") - assert.False(t, ok2, "ch2 should be closed") -} - -func TestSubscribers_NotifyNoSubscribersNoPanic(t *testing.T) { - s := newSubscribers() - require.NotPanics(t, func() { - s.Notify("no-such-instance", StateChange{State: StateRunning}) - }) -} diff --git a/lib/instances/wait.go b/lib/instances/wait.go index ccb52c06..23ebeb44 100644 --- a/lib/instances/wait.go +++ b/lib/instances/wait.go @@ -22,13 +22,13 @@ type WaitForStateResult struct { TimedOut bool } -// WaitForState subscribes to state change events for the instance and waits -// until it reaches targetState, a terminal/error state is detected, the timeout +// WaitForState subscribes to lifecycle events for the instance and waits until +// it reaches targetState, a terminal/error state is detected, the timeout // expires, or the context is cancelled. A polling fallback (every 5s) guards -// against missed subscription events. +// against missed or dropped events. func WaitForState(ctx context.Context, mgr Manager, inst *Instance, targetState State, timeout time.Duration) (*WaitForStateResult, error) { // Subscribe first to avoid missing events between initial check and loop. - ch, unsub := mgr.Subscribe(inst.Id) + ch, unsub := mgr.SubscribeLifecycleEvents(LifecycleEventConsumerWaitForState) defer unsub() // Already in target state — return immediately. @@ -80,14 +80,20 @@ func WaitForState(ctx context.Context, mgr Manager, inst *Instance, targetState TimedOut: latest.State != targetState, }, nil - case sc, ok := <-ch: + case event, ok := <-ch: if !ok { - // Channel closed — instance was deleted. return nil, ErrNotFound } - latest = &Instance{} - latest.State = sc.State - latest.StateError = sc.StateError + if event.InstanceID != id { + continue + } + if event.Action == LifecycleEventDelete { + return nil, ErrNotFound + } + if event.Instance == nil { + continue + } + latest = event.Instance if latest.State == targetState { return &WaitForStateResult{ diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index 8fe3291b..bab6f06d 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -13,12 +13,12 @@ import ( // stubManager is a minimal Manager implementation for unit-testing WaitForState. type stubManager struct { - subs *subscribers + subs *lifecycleSubscribers getInstance func(ctx context.Context, id string) (*Instance, error) } -func (s *stubManager) Subscribe(instanceID string) (<-chan StateChange, func()) { - return s.subs.Subscribe(instanceID) +func (s *stubManager) SubscribeLifecycleEvents(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) { + return s.subs.Subscribe(consumer) } func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, error) { @@ -28,7 +28,7 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er return nil, ErrNotFound } -// Unused interface methods — only GetInstance and Subscribe are needed. +// Unused interface methods — only GetInstance and SubscribeLifecycleEvents are needed. func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } @@ -87,7 +87,7 @@ func (s *stubManager) GetVsockDialer(context.Context, string) (hypervisor.VsockD func TestWaitForState_SubscriptionDelivers(t *testing.T) { t.Parallel() - subs := newSubscribers() + subs := newLifecycleSubscribers() mgr := &stubManager{subs: subs} inst := &Instance{} @@ -97,7 +97,11 @@ func TestWaitForState_SubscriptionDelivers(t *testing.T) { // Simulate a state change via subscription after 100ms. go func() { time.Sleep(100 * time.Millisecond) - subs.Notify("test-sub", StateChange{State: StateRunning}) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStart, + InstanceID: "test-sub", + Instance: &Instance{State: StateRunning}, + }) }() start := time.Now() @@ -113,7 +117,7 @@ func TestWaitForState_SubscriptionDelivers(t *testing.T) { func TestWaitForState_ChannelClosedOnDelete(t *testing.T) { t.Parallel() - subs := newSubscribers() + subs := newLifecycleSubscribers() mgr := &stubManager{subs: subs} inst := &Instance{} @@ -123,7 +127,10 @@ func TestWaitForState_ChannelClosedOnDelete(t *testing.T) { // Simulate instance deletion (close all subscriber channels). go func() { time.Sleep(100 * time.Millisecond) - subs.CloseAll("test-close") + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventDelete, + InstanceID: "test-close", + }) }() start := time.Now() @@ -137,7 +144,7 @@ func TestWaitForState_ChannelClosedOnDelete(t *testing.T) { func TestWaitForState_TerminalViaSubscription(t *testing.T) { t.Parallel() - subs := newSubscribers() + subs := newLifecycleSubscribers() mgr := &stubManager{subs: subs} inst := &Instance{} @@ -146,7 +153,11 @@ func TestWaitForState_TerminalViaSubscription(t *testing.T) { go func() { time.Sleep(100 * time.Millisecond) - subs.Notify("test-terminal-sub", StateChange{State: StateStopped}) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStop, + InstanceID: "test-terminal-sub", + Instance: &Instance{State: StateStopped}, + }) }() result, err := WaitForState(context.Background(), mgr, inst, StateRunning, 30*time.Second) @@ -159,7 +170,7 @@ func TestWaitForState_TerminalViaSubscription(t *testing.T) { func TestWaitForState_ShutdownIsTerminal(t *testing.T) { t.Parallel() - subs := newSubscribers() + subs := newLifecycleSubscribers() mgr := &stubManager{subs: subs} inst := &Instance{} @@ -168,7 +179,11 @@ func TestWaitForState_ShutdownIsTerminal(t *testing.T) { go func() { time.Sleep(100 * time.Millisecond) - subs.Notify("test-shutdown", StateChange{State: StateShutdown}) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStop, + InstanceID: "test-shutdown", + Instance: &Instance{State: StateShutdown}, + }) }() start := time.Now() @@ -184,7 +199,7 @@ func TestWaitForState_ShutdownIsTerminal(t *testing.T) { func TestWaitForState_PausedIsTerminal(t *testing.T) { t.Parallel() - subs := newSubscribers() + subs := newLifecycleSubscribers() mgr := &stubManager{subs: subs} inst := &Instance{} @@ -193,7 +208,11 @@ func TestWaitForState_PausedIsTerminal(t *testing.T) { go func() { time.Sleep(100 * time.Millisecond) - subs.Notify("test-paused", StateChange{State: StatePaused}) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStandby, + InstanceID: "test-paused", + Instance: &Instance{State: StatePaused}, + }) }() start := time.Now() @@ -206,3 +225,69 @@ func TestWaitForState_PausedIsTerminal(t *testing.T) { assert.False(t, result.TimedOut) assert.Less(t, elapsed, 2*time.Second, "paused should be detected as terminal immediately") } + +func TestWaitForState_IgnoresEventsForOtherInstances(t *testing.T) { + t.Parallel() + subs := newLifecycleSubscribers() + mgr := &stubManager{subs: subs} + + inst := &Instance{} + inst.Id = "target-instance" + inst.State = StateInitializing + + go func() { + time.Sleep(50 * time.Millisecond) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStart, + InstanceID: "other-instance", + Instance: &Instance{State: StateRunning}, + }) + time.Sleep(50 * time.Millisecond) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStart, + InstanceID: "target-instance", + Instance: &Instance{State: StateRunning}, + }) + }() + + result, err := WaitForState(context.Background(), mgr, inst, StateRunning, 30*time.Second) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, StateRunning, result.State) +} + +func TestWaitForState_IgnoresNilInstancePayloadAndUsesPollingFallback(t *testing.T) { + t.Parallel() + subs := newLifecycleSubscribers() + mgr := &stubManager{ + subs: subs, + getInstance: func(ctx context.Context, id string) (*Instance, error) { + return &Instance{ + StoredMetadata: StoredMetadata{Id: id}, + State: StateRunning, + }, nil + }, + } + + inst := &Instance{} + inst.Id = "test-nil-event" + inst.State = StateInitializing + + go func() { + time.Sleep(100 * time.Millisecond) + subs.Notify(context.Background(), LifecycleEvent{ + Action: LifecycleEventStart, + InstanceID: "test-nil-event", + }) + }() + + start := time.Now() + result, err := WaitForState(context.Background(), mgr, inst, StateRunning, 6*time.Second) + elapsed := time.Since(start) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, StateRunning, result.State) + assert.GreaterOrEqual(t, elapsed, WaitForStatePollInterval) +} From 134628d677cbb38284330c93bb7f9dc5c741c368 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 16:13:54 -0400 Subject: [PATCH 13/13] Add config for lifecycle event buffer size --- cmd/api/config/config.go | 13 ++++++++++ cmd/api/config/config_test.go | 34 ++++++++++++++++++++++++++ lib/instances/lifecycle_events.go | 24 +++++++++++++----- lib/instances/lifecycle_events_test.go | 2 +- lib/instances/manager.go | 21 +++++++++++++++- lib/instances/metrics_test.go | 8 +++--- lib/providers/providers.go | 5 +++- 7 files changed, 94 insertions(+), 13 deletions(-) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index ce7c90a0..55079fd6 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -144,6 +144,11 @@ type BuildConfig struct { DockerSocket string `koanf:"docker_socket"` } +// InstancesConfig holds instance-manager internal settings. +type InstancesConfig struct { + LifecycleEventBufferSize int `koanf:"lifecycle_event_buffer_size"` +} + // RegistryConfig holds OCI registry settings. type RegistryConfig struct { URL string `koanf:"url"` @@ -240,6 +245,7 @@ type Config struct { Logging LoggingConfig `koanf:"logging"` Images ImagesConfig `koanf:"images"` Build BuildConfig `koanf:"build"` + Instances InstancesConfig `koanf:"instances"` Registry RegistryConfig `koanf:"registry"` Limits LimitsConfig `koanf:"limits"` Oversubscription OversubscriptionConfig `koanf:"oversubscription"` @@ -348,6 +354,10 @@ func defaultConfig() *Config { DockerSocket: "/var/run/docker.sock", }, + Instances: InstancesConfig{ + LifecycleEventBufferSize: 256, + }, + Registry: RegistryConfig{ URL: "localhost:8080", Insecure: false, @@ -532,6 +542,9 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } + if c.Instances.LifecycleEventBufferSize <= 0 { + return fmt.Errorf("instances.lifecycle_event_buffer_size must be positive, got %d", c.Instances.LifecycleEventBufferSize) + } if err := validateDuration("images.auto_delete.unused_for", c.Images.AutoDelete.UnusedFor); err != nil { return err } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index bb451957..21703eb5 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -40,6 +40,9 @@ func TestDefaultConfigIncludesMetricsSettings(t *testing.T) { if len(cfg.Images.AutoDelete.Allowed) != 0 { t.Fatalf("expected default images.auto_delete.allowed to be empty, got %v", cfg.Images.AutoDelete.Allowed) } + if cfg.Instances.LifecycleEventBufferSize != 256 { + t.Fatalf("expected default instances.lifecycle_event_buffer_size to be 256, got %d", cfg.Instances.LifecycleEventBufferSize) + } } func TestLoadEnvOverridesMetricsAndOtelInterval(t *testing.T) { @@ -49,6 +52,7 @@ func TestLoadEnvOverridesMetricsAndOtelInterval(t *testing.T) { t.Setenv("METRICS__RESOURCE_REFRESH_INTERVAL", "30s") t.Setenv("OTEL__METRIC_EXPORT_INTERVAL", "15s") t.Setenv("OTEL__SUCCESSFUL_GET_SAMPLE_RATIO", "0.25") + t.Setenv("INSTANCES__LIFECYCLE_EVENT_BUFFER_SIZE", "512") tmp := t.TempDir() cfgPath := filepath.Join(tmp, "config.yaml") @@ -79,6 +83,9 @@ func TestLoadEnvOverridesMetricsAndOtelInterval(t *testing.T) { if cfg.Otel.SuccessfulGetSampleRatio != 0.25 { t.Fatalf("expected otel.successful_get_sample_ratio override, got %v", cfg.Otel.SuccessfulGetSampleRatio) } + if cfg.Instances.LifecycleEventBufferSize != 512 { + t.Fatalf("expected instances.lifecycle_event_buffer_size override, got %d", cfg.Instances.LifecycleEventBufferSize) + } } func TestValidateRejectsInvalidMetricsPort(t *testing.T) { @@ -147,6 +154,33 @@ func TestValidateRejectsInvalidResourceRefreshInterval(t *testing.T) { } } +func TestLoadUsesConfiguredLifecycleEventBufferSize(t *testing.T) { + tmp := t.TempDir() + cfgPath := filepath.Join(tmp, "config.yaml") + if err := os.WriteFile(cfgPath, []byte("instances:\n lifecycle_event_buffer_size: 384\n"), 0600); err != nil { + t.Fatalf("write temp config: %v", err) + } + + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + + if cfg.Instances.LifecycleEventBufferSize != 384 { + t.Fatalf("expected instances.lifecycle_event_buffer_size from config file, got %d", cfg.Instances.LifecycleEventBufferSize) + } +} + +func TestValidateRejectsInvalidLifecycleEventBufferSize(t *testing.T) { + cfg := defaultConfig() + cfg.Instances.LifecycleEventBufferSize = 0 + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for invalid lifecycle event buffer size") + } +} + func TestLoadUsesDefaultImageAutoDeleteRetentionWindow(t *testing.T) { tmp := t.TempDir() cfgPath := filepath.Join(tmp, "config.yaml") diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go index ad14cb00..5ae42024 100644 --- a/lib/instances/lifecycle_events.go +++ b/lib/instances/lifecycle_events.go @@ -5,7 +5,7 @@ import ( "sync" ) -const lifecycleEventBufferSize = 32 +const defaultLifecycleEventBufferSize = 256 // LifecycleEventConsumer identifies the internal consumer of lifecycle events. // Keep this set bounded for observability label safety. @@ -54,17 +54,29 @@ type lifecycleConsumerStats struct { } type lifecycleSubscribers struct { - mu sync.Mutex - subs []lifecycleSubscriber - onDrop func(context.Context, LifecycleEventConsumer) + mu sync.Mutex + subs []lifecycleSubscriber + onDrop func(context.Context, LifecycleEventConsumer) + bufferSize int } func newLifecycleSubscribers() *lifecycleSubscribers { - return &lifecycleSubscribers{} + return &lifecycleSubscribers{ + bufferSize: defaultLifecycleEventBufferSize, + } +} + +func newLifecycleSubscribersWithBufferSize(bufferSize int) *lifecycleSubscribers { + if bufferSize <= 0 { + bufferSize = defaultLifecycleEventBufferSize + } + return &lifecycleSubscribers{ + bufferSize: bufferSize, + } } func (s *lifecycleSubscribers) Subscribe(consumer LifecycleEventConsumer) (<-chan LifecycleEvent, func()) { - ch := make(chan LifecycleEvent, lifecycleEventBufferSize) + ch := make(chan LifecycleEvent, s.bufferSize) s.mu.Lock() s.subs = append(s.subs, lifecycleSubscriber{ diff --git a/lib/instances/lifecycle_events_test.go b/lib/instances/lifecycle_events_test.go index 0cf0c1c7..a46366e7 100644 --- a/lib/instances/lifecycle_events_test.go +++ b/lib/instances/lifecycle_events_test.go @@ -102,7 +102,7 @@ func TestLifecycleSubscribers_DropCallbackOnBackpressure(t *testing.T) { _, unsub := s.Subscribe(LifecycleEventConsumerWaitForState) defer unsub() - for i := 0; i < lifecycleEventBufferSize; i++ { + for i := 0; i < s.bufferSize; i++ { s.Notify(context.Background(), LifecycleEvent{ Action: LifecycleEventUpdate, InstanceID: "inst-1", diff --git a/lib/instances/manager.go b/lib/instances/manager.go index accb9741..9f6492ac 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -78,6 +78,19 @@ type ResourceLimits struct { MaxMemoryPerInstance int64 // Maximum memory in bytes per instance (0 = unlimited) } +// ManagerConfig holds non-resource manager behavior settings. +type ManagerConfig struct { + LifecycleEventBufferSize int +} + +// Normalize applies defaults to manager config values. +func (c ManagerConfig) Normalize() ManagerConfig { + if c.LifecycleEventBufferSize <= 0 { + c.LifecycleEventBufferSize = defaultLifecycleEventBufferSize + } + return c +} + // ResourceValidator validates if resources can be allocated type ResourceValidator interface { // ValidateAllocation checks if the requested resources are available. @@ -130,6 +143,11 @@ var platformStarters = make(map[hypervisor.Type]hypervisor.VMStarter) // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, snapshotDefaults SnapshotPolicy, meter metric.Meter, tracer trace.Tracer, memoryPolicy ...guestmemory.Policy) Manager { + return NewManagerWithConfig(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, ManagerConfig{}, meter, tracer, memoryPolicy...) +} + +// NewManagerWithConfig creates a new instances manager with additional manager settings. +func NewManagerWithConfig(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, snapshotDefaults SnapshotPolicy, managerConfig ManagerConfig, meter metric.Meter, tracer trace.Tracer, memoryPolicy ...guestmemory.Policy) Manager { // Validate and default the hypervisor type if defaultHypervisor == "" { defaultHypervisor = hypervisor.TypeCloudHypervisor @@ -140,6 +158,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste policy = memoryPolicy[0] } policy = policy.Normalize() + managerConfig = managerConfig.Normalize() // Initialize VM starters from platform-specific init functions vmStarters := make(map[hypervisor.Type]hypervisor.VMStarter, len(platformStarters)) @@ -168,7 +187,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste snapshotDefaults: snapshotDefaults, compressionJobs: make(map[string]*compressionJob), nativeCodecPaths: make(map[string]string), - lifecycleEvents: newLifecycleSubscribers(), + lifecycleEvents: newLifecycleSubscribersWithBufferSize(managerConfig.LifecycleEventBufferSize), } m.deleteSnapshotFn = m.deleteSnapshot diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index b5681fd0..37e251dd 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -148,7 +148,7 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T } <-waitCh - for i := 0; i < lifecycleEventBufferSize; i++ { + for i := 0; i < m.lifecycleEvents.bufferSize; i++ { m.lifecycleEvents.Notify(t.Context(), LifecycleEvent{ Action: LifecycleEventUpdate, InstanceID: "inst-1", @@ -187,9 +187,9 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T for _, point := range queueDepth.DataPoints { switch metricLabel(t, point.Attributes, "consumer") { case string(LifecycleEventConsumerWaitForState): - assert.Equal(t, int64(lifecycleEventBufferSize), point.Value) + assert.Equal(t, int64(m.lifecycleEvents.bufferSize), point.Value) case string(LifecycleEventConsumerAutoStandby): - assert.Equal(t, int64(lifecycleEventBufferSize), point.Value) + assert.Equal(t, int64(m.lifecycleEvents.bufferSize), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } @@ -213,7 +213,7 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T } assert.Greater(t, waitDrops, int64(0)) assert.Greater(t, autoDrops, int64(0)) - assert.Equal(t, lifecycleEventBufferSize, len(autoCh)) + assert.Equal(t, m.lifecycleEvents.bufferSize, len(autoCh)) } func TestInstanceOldestInStateMetric_ObserveOldestAgePerState(t *testing.T) { diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 4c112915..205176d8 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -134,7 +134,10 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima ReclaimEnabled: cfg.Hypervisor.Memory.ReclaimEnabled, VZBalloonRequired: cfg.Hypervisor.Memory.VZBalloonRequired, } - return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, meter, tracer, memoryPolicy), nil + managerConfig := instances.ManagerConfig{ + LifecycleEventBufferSize: cfg.Instances.LifecycleEventBufferSize, + } + return instances.NewManagerWithConfig(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, managerConfig, meter, tracer, memoryPolicy), nil } func snapshotDefaultsFromConfig(cfg *config.Config) instances.SnapshotPolicy {