From d96eeb648282dc7c3f4d6b9f749bd4930f12e134 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Sat, 4 Apr 2026 15:56:40 -0400 Subject: [PATCH 1/6] Add standby compression start delay --- cmd/api/api/instances.go | 32 ++ cmd/api/api/instances_test.go | 150 ++++++ lib/instances/create.go | 5 + lib/instances/delete.go | 4 +- lib/instances/fork.go | 3 + lib/instances/manager.go | 1 + lib/instances/metrics.go | 69 ++- lib/instances/metrics_test.go | 136 +++++- lib/instances/snapshot.go | 20 +- lib/instances/snapshot_compression.go | 213 ++++++++- lib/instances/snapshot_compression_test.go | 147 ++++++ lib/instances/standby.go | 20 + lib/instances/types.go | 6 +- lib/oapi/oapi.go | 512 +++++++++++---------- lib/otel/README.md | 11 + lib/snapshot/README.md | 3 + openapi.yaml | 11 +- 17 files changed, 1044 insertions(+), 299 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index b7248207..a28d257e 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -534,6 +534,16 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn } standbyReq.Compression = compression } + if request.Body != nil && request.Body.CompressionDelay != nil { + compressionDelay, err := parseOptionalDuration(*request.Body.CompressionDelay, "compression_delay") + if err != nil { + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_compression_delay", + Message: err.Error(), + }, nil + } + standbyReq.CompressionDelay = compressionDelay + } result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { @@ -1135,6 +1145,13 @@ func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPo } out.Compression = compression } + if policy.StandbyCompressionDelay != nil { + delay, err := parseOptionalDuration(*policy.StandbyCompressionDelay, "standby_compression_delay") + if err != nil { + return nil, err + } + out.StandbyCompressionDelay = delay + } return out, nil } @@ -1159,5 +1176,20 @@ func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) oapi.SnapshotPolicy { compression := toOAPISnapshotCompressionConfig(*policy.Compression) out.Compression = &compression } + if policy.StandbyCompressionDelay != nil { + delay := policy.StandbyCompressionDelay.String() + out.StandbyCompressionDelay = &delay + } return out } + +func parseOptionalDuration(value string, field string) (*time.Duration, error) { + duration, err := time.ParseDuration(value) + if err != nil { + return nil, fmt.Errorf("%s must be a valid duration: %w", field, err) + } + if duration < 0 { + return nil, fmt.Errorf("%s cannot be negative", field) + } + return &duration, nil +} diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index b3f74ea2..f7252a09 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -477,6 +477,81 @@ func TestCreateInstance_MapsNetworkEgressEnforcementMode(t *testing.T) { assert.Equal(t, instances.EgressEnforcementModeHTTPHTTPSOnly, mockMgr.lastReq.NetworkEgress.EnforcementMode) } +func TestCreateInstance_MapsStandbyCompressionDelayInSnapshotPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + delay := "2m30s" + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-standby-compression-delay", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &oapi.SnapshotPolicy{ + StandbyCompressionDelay: &delay, + }, + }, + }) + require.NoError(t, err) + _, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.SnapshotPolicy) + require.NotNil(t, mockMgr.lastReq.SnapshotPolicy.StandbyCompressionDelay) + assert.Equal(t, 150*time.Second, *mockMgr.lastReq.SnapshotPolicy.StandbyCompressionDelay) +} + +func TestCreateInstance_InvalidStandbyCompressionDelayInSnapshotPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + delay := "not-a-duration" + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-invalid-standby-delay", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &oapi.SnapshotPolicy{ + StandbyCompressionDelay: &delay, + }, + }, + }) + require.NoError(t, err) + + badReq, ok := resp.(oapi.CreateInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_snapshot_policy", badReq.Code) + assert.Contains(t, badReq.Message, "standby_compression_delay") +} + +func TestInstanceToOAPI_EmitsStandbyCompressionDelayInSnapshotPolicy(t *testing.T) { + t.Parallel() + + delay := 90 * time.Second + inst := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-standby-delay", + Name: "inst-standby-delay", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + SnapshotPolicy: &instances.SnapshotPolicy{ + StandbyCompressionDelay: &delay, + }, + }, + State: instances.StateStandby, + } + + oapiInst := instanceToOAPI(inst) + require.NotNil(t, oapiInst.SnapshotPolicy) + require.NotNil(t, oapiInst.SnapshotPolicy.StandbyCompressionDelay) + assert.Equal(t, "1m30s", *oapiInst.SnapshotPolicy.StandbyCompressionDelay) +} + func TestUpdateInstance_MapsEnvPatch(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -753,6 +828,81 @@ func TestStandbyInstance_InvalidRequest(t *testing.T) { assert.Contains(t, badReq.Message, "invalid snapshot compression level") } +func TestStandbyInstance_MapsCompressionDelay(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "standby-delay-src", + Name: "standby-delay-src", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateRunning, + } + + mockMgr := &captureStandbyManager{ + Manager: svc.InstanceManager, + result: &source, + } + svc.InstanceManager = mockMgr + + delay := "45s" + resp, err := svc.StandbyInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.StandbyInstanceRequestObject{ + Id: source.Id, + Body: &oapi.StandbyInstanceRequest{ + CompressionDelay: &delay, + }, + }, + ) + require.NoError(t, err) + _, ok := resp.(oapi.StandbyInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.CompressionDelay) + assert.Equal(t, 45*time.Second, *mockMgr.lastReq.CompressionDelay) +} + +func TestStandbyInstance_InvalidCompressionDelay(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "standby-invalid-delay-src", + Name: "standby-invalid-delay-src", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateRunning, + } + + delay := "-5s" + resp, err := svc.StandbyInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.StandbyInstanceRequestObject{ + Id: source.Id, + Body: &oapi.StandbyInstanceRequest{ + CompressionDelay: &delay, + }, + }, + ) + require.NoError(t, err) + + badReq, ok := resp.(oapi.StandbyInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_compression_delay", badReq.Code) + assert.Contains(t, badReq.Message, "compression_delay") +} + func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/lib/instances/create.go b/lib/instances/create.go index ca4be904..5ac2d39a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -570,6 +570,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { return err } } + if req.SnapshotPolicy != nil && req.SnapshotPolicy.StandbyCompressionDelay != nil { + if _, err := normalizeStandbyCompressionDelay(req.SnapshotPolicy.StandbyCompressionDelay); err != nil { + return err + } + } // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 974224e8..283e0b19 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -39,8 +39,8 @@ func (m *manager) deleteInstance( if err != nil { return fmt.Errorf("wait for instance compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteInstance, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteInstance, target.Target) } // 2. Get network allocation BEFORE killing VMM (while we can still query it) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 2c75c94e..9e846ab9 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -485,6 +485,9 @@ func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { if src.Credentials != nil { dst.Credentials = cloneCredentialPolicies(src.Credentials) } + if src.SnapshotPolicy != nil { + dst.SnapshotPolicy = cloneSnapshotPolicy(src.SnapshotPolicy) + } if src.Tags != nil { dst.Tags = make(map[string]string, len(src.Tags)) for k, v := range src.Tags { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 4ef8d9bf..b2b7a042 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -111,6 +111,7 @@ type manager struct { snapshotDefaults SnapshotPolicy compressionMu sync.Mutex compressionJobs map[string]*compressionJob + compressionTimerFactory func(time.Duration) compressionTimer nativeCodecMu sync.Mutex nativeCodecPaths map[string]string imageUsageRecorder ImageUsageRecorder diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index bee79d1a..bd504402 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -25,10 +25,18 @@ type snapshotCompressionResult string const ( snapshotCompressionResultSuccess snapshotCompressionResult = "success" + snapshotCompressionResultSkipped snapshotCompressionResult = "skipped" snapshotCompressionResultCanceled snapshotCompressionResult = "canceled" snapshotCompressionResultFailed snapshotCompressionResult = "failed" ) +type snapshotCompressionWaitOutcome string + +const ( + snapshotCompressionWaitOutcomeStarted snapshotCompressionWaitOutcome = "started" + snapshotCompressionWaitOutcomeSkipped snapshotCompressionWaitOutcome = "skipped" +) + type snapshotMemoryPreparePath string const ( @@ -72,6 +80,7 @@ type Metrics struct { stateTransitions metric.Int64Counter snapshotCompressionJobsTotal metric.Int64Counter snapshotCompressionDuration metric.Float64Histogram + snapshotCompressionWaitDuration metric.Float64Histogram snapshotCompressionSavedBytes metric.Int64Histogram snapshotCompressionRatio metric.Float64Histogram snapshotCodecFallbacksTotal metric.Int64Counter @@ -169,6 +178,16 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCompressionWaitDuration, err := meter.Float64Histogram( + "hypeman_snapshot_compression_wait_duration_seconds", + metric.WithDescription("Time a delayed snapshot compression job waits before compression starts or is skipped"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(hypotel.CommonDurationHistogramBuckets()...), + ) + if err != nil { + return nil, err + } + snapshotCompressionSavedBytes, err := meter.Int64Histogram( "hypeman_snapshot_compression_saved_bytes", metric.WithDescription("Bytes saved by compressing snapshot memory"), @@ -240,7 +259,15 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M snapshotCompressionActiveTotal, err := meter.Int64ObservableGauge( "hypeman_snapshot_compression_active_total", - metric.WithDescription("Total number of in-flight snapshot compression jobs"), + metric.WithDescription("Total number of actively running snapshot compression jobs"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionPendingTotal, err := meter.Int64ObservableGauge( + "hypeman_snapshot_compression_pending_total", + metric.WithDescription("Total number of delayed snapshot compression jobs waiting to start"), ) if err != nil { return nil, err @@ -302,7 +329,8 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M source string } - counts := make(map[compressionKey]int64) + activeCounts := make(map[compressionKey]int64) + pendingCounts := make(map[compressionKey]int64) m.compressionMu.Lock() for _, job := range m.compressionJobs { key := compressionKey{ @@ -310,11 +338,16 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M algorithm: string(job.target.Policy.Algorithm), source: string(job.target.Source), } - counts[key]++ + switch job.state { + case compressionJobStatePendingDelay: + pendingCounts[key]++ + case compressionJobStateRunning: + activeCounts[key]++ + } } m.compressionMu.Unlock() - for key, count := range counts { + for key, count := range activeCounts { attrs := []attribute.KeyValue{ attribute.String("algorithm", key.algorithm), attribute.String("source", key.source), @@ -324,9 +357,20 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M } o.ObserveInt64(snapshotCompressionActiveTotal, count, metric.WithAttributes(attrs...)) } + for key, count := range pendingCounts { + attrs := []attribute.KeyValue{ + attribute.String("algorithm", key.algorithm), + attribute.String("source", key.source), + } + if key.hypervisor != "" { + attrs = append(attrs, attribute.String("hypervisor", key.hypervisor)) + } + o.ObserveInt64(snapshotCompressionPendingTotal, count, metric.WithAttributes(attrs...)) + } return nil }, snapshotCompressionActiveTotal, + snapshotCompressionPendingTotal, ) if err != nil { return nil, err @@ -342,6 +386,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M stateTransitions: stateTransitions, snapshotCompressionJobsTotal: snapshotCompressionJobsTotal, snapshotCompressionDuration: snapshotCompressionDuration, + snapshotCompressionWaitDuration: snapshotCompressionWaitDuration, snapshotCompressionSavedBytes: snapshotCompressionSavedBytes, snapshotCompressionRatio: snapshotCompressionRatio, snapshotCodecFallbacksTotal: snapshotCodecFallbacksTotal, @@ -473,7 +518,7 @@ func snapshotCompressionAttributes(hvType hypervisor.Type, algorithm snapshotsto return attrs } -func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compressionTarget, result snapshotCompressionResult, start time.Time, uncompressedSize, compressedSize int64) { +func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compressionTarget, result snapshotCompressionResult, compressionStart *time.Time, uncompressedSize, compressedSize int64) { if m.metrics == nil { return } @@ -483,7 +528,9 @@ func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compr attrsWithResult = append(attrsWithResult, attribute.String("result", string(result))) m.metrics.snapshotCompressionJobsTotal.Add(ctx, 1, metric.WithAttributes(attrsWithResult...)) - m.metrics.snapshotCompressionDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrsWithResult...)) + if compressionStart != nil { + m.metrics.snapshotCompressionDuration.Record(ctx, time.Since(*compressionStart).Seconds(), metric.WithAttributes(attrsWithResult...)) + } if result != snapshotCompressionResultSuccess || uncompressedSize <= 0 || compressedSize < 0 { return @@ -497,6 +544,16 @@ func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compr m.metrics.snapshotCompressionRatio.Record(ctx, float64(compressedSize)/float64(uncompressedSize), metric.WithAttributes(attrs...)) } +func (m *manager) recordSnapshotCompressionWait(ctx context.Context, target compressionTarget, outcome snapshotCompressionWaitOutcome, start time.Time) { + if m.metrics == nil { + return + } + + attrs := snapshotCompressionAttributes(target.HypervisorType, target.Policy.Algorithm, target.Source) + attrs = append(attrs, attribute.String("outcome", string(outcome))) + m.metrics.snapshotCompressionWaitDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) +} + func (m *manager) recordSnapshotRestoreMemoryPrepare(ctx context.Context, hvType hypervisor.Type, path snapshotMemoryPreparePath, result snapshotCompressionResult, start time.Time) { if m.metrics == nil { return diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 88def215..859c6b9c 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -26,7 +26,8 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { paths: paths.New(t.TempDir()), compressionJobs: map[string]*compressionJob{ "job-1": { - done: make(chan struct{}), + done: make(chan struct{}), + state: compressionJobStateRunning, target: compressionTarget{ Key: "job-1", HypervisorType: hypervisor.TypeCloudHypervisor, @@ -37,6 +38,19 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { }, }, }, + "job-2": { + done: make(chan struct{}), + state: compressionJobStatePendingDelay, + target: compressionTarget{ + Key: "job-2", + HypervisorType: hypervisor.TypeQEMU, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + }, + }, + }, }, } @@ -45,7 +59,10 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { m.metrics = metrics target := m.compressionJobs["job-1"].target - m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, time.Now().Add(-2*time.Second), 1024, 256) + startedAt := time.Now().Add(-2 * time.Second) + m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, &startedAt, 1024, 256) + m.recordSnapshotCompressionJob(t.Context(), m.compressionJobs["job-2"].target, snapshotCompressionResultSkipped, nil, 0, 0) + m.recordSnapshotCompressionWait(t.Context(), m.compressionJobs["job-2"].target, snapshotCompressionWaitOutcomeSkipped, time.Now().Add(-1500*time.Millisecond)) m.recordSnapshotCodecFallback(t.Context(), snapshotstore.SnapshotCompressionAlgorithmLz4, snapshotCodecOperationCompress, snapshotCodecFallbackReasonMissingBinary) m.recordSnapshotRestoreMemoryPrepare(t.Context(), hypervisor.TypeCloudHypervisor, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, time.Now().Add(-250*time.Millisecond)) m.recordSnapshotCompressionPreemption(t.Context(), snapshotCompressionPreemptionRestoreInstance, target) @@ -56,6 +73,7 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assertMetricNames(t, rm, []string{ "hypeman_snapshot_compression_jobs_total", "hypeman_snapshot_compression_duration_seconds", + "hypeman_snapshot_compression_wait_duration_seconds", "hypeman_snapshot_compression_saved_bytes", "hypeman_snapshot_compression_ratio", "hypeman_snapshot_codec_fallbacks_total", @@ -63,17 +81,29 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { "hypeman_snapshot_restore_memory_prepare_duration_seconds", "hypeman_snapshot_compression_preemptions_total", "hypeman_snapshot_compression_active_total", + "hypeman_snapshot_compression_pending_total", }) jobsMetric := findMetric(t, rm, "hypeman_snapshot_compression_jobs_total") jobs, ok := jobsMetric.Data.(metricdata.Sum[int64]) require.True(t, ok) - require.Len(t, jobs.DataPoints, 1) - assert.Equal(t, int64(1), jobs.DataPoints[0].Value) - assert.Equal(t, "cloud-hypervisor", metricLabel(t, jobs.DataPoints[0].Attributes, "hypervisor")) - assert.Equal(t, "lz4", metricLabel(t, jobs.DataPoints[0].Attributes, "algorithm")) - assert.Equal(t, "standby", metricLabel(t, jobs.DataPoints[0].Attributes, "source")) - assert.Equal(t, "success", metricLabel(t, jobs.DataPoints[0].Attributes, "result")) + require.Len(t, jobs.DataPoints, 2) + for _, point := range jobs.DataPoints { + switch metricLabel(t, point.Attributes, "result") { + case "success": + assert.Equal(t, int64(1), point.Value) + assert.Equal(t, "cloud-hypervisor", metricLabel(t, point.Attributes, "hypervisor")) + assert.Equal(t, "lz4", metricLabel(t, point.Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, point.Attributes, "source")) + case "skipped": + assert.Equal(t, int64(1), point.Value) + assert.Equal(t, "qemu", metricLabel(t, point.Attributes, "hypervisor")) + assert.Equal(t, "zstd", metricLabel(t, point.Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, point.Attributes, "source")) + default: + t.Fatalf("unexpected compression job result datapoint: %s", metricLabel(t, point.Attributes, "result")) + } + } savedBytesMetric := findMetric(t, rm, "hypeman_snapshot_compression_saved_bytes") savedBytes, ok := savedBytesMetric.Data.(metricdata.Histogram[int64]) @@ -113,6 +143,20 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assert.Equal(t, int64(1), active.DataPoints[0].Value) assert.Equal(t, "lz4", metricLabel(t, active.DataPoints[0].Attributes, "algorithm")) assert.Equal(t, "standby", metricLabel(t, active.DataPoints[0].Attributes, "source")) + + pendingMetric := findMetric(t, rm, "hypeman_snapshot_compression_pending_total") + pending, ok := pendingMetric.Data.(metricdata.Gauge[int64]) + require.True(t, ok) + require.Len(t, pending.DataPoints, 1) + assert.Equal(t, int64(1), pending.DataPoints[0].Value) + assert.Equal(t, "zstd", metricLabel(t, pending.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, pending.DataPoints[0].Attributes, "source")) + + waitMetric := findMetric(t, rm, "hypeman_snapshot_compression_wait_duration_seconds") + waitDurations, ok := waitMetric.Data.(metricdata.Histogram[float64]) + require.True(t, ok) + require.Len(t, waitDurations.DataPoints, 1) + assert.Equal(t, "skipped", metricLabel(t, waitDurations.DataPoints[0].Attributes, "outcome")) } func TestInstanceOldestInStateMetric_ObserveOldestAgePerState(t *testing.T) { @@ -338,6 +382,71 @@ func TestLifecycleDurationMetrics_RecordCompressionLabels(t *testing.T) { assert.Equal(t, "none", metricLabel(t, standby.DataPoints[0].Attributes, "level")) } +func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + mgr, _ := setupTestManager(t) + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, mgr) + require.NoError(t, err) + mgr.metrics = metrics + + delay := 30 * time.Second + timer := newFakeCompressionTimer() + mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { + require.Equal(t, delay, got) + return timer + } + + snapshotDir := t.TempDir() + rawPath := filepath.Join(snapshotDir, "memory") + require.NoError(t, os.WriteFile(rawPath, []byte("pending raw snapshot"), 0o644)) + + target := compressionTarget{ + Key: "instance:pending", + OwnerID: "pending", + SnapshotDir: snapshotDir, + HypervisorType: hypervisor.TypeCloudHypervisor, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + Delay: delay, + } + + mgr.startCompressionJob(t.Context(), target) + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + job, ok := mgr.compressionJobs[target.Key] + return ok && job.state == compressionJobStatePendingDelay + }, time.Second, 10*time.Millisecond) + + require.NoError(t, mgr.ensureSnapshotMemoryReady(t.Context(), snapshotDir, target.Key, hypervisor.TypeCloudHypervisor)) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + jobsMetric := findMetric(t, rm, "hypeman_snapshot_compression_jobs_total") + jobs, ok := jobsMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, jobs.DataPoints, 1) + assert.Equal(t, "skipped", metricLabel(t, jobs.DataPoints[0].Attributes, "result")) + + waitMetric := findMetric(t, rm, "hypeman_snapshot_compression_wait_duration_seconds") + waitDurations, ok := waitMetric.Data.(metricdata.Histogram[float64]) + require.True(t, ok) + require.Len(t, waitDurations.DataPoints, 1) + assert.Equal(t, "skipped", metricLabel(t, waitDurations.DataPoints[0].Attributes, "outcome")) + + assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() @@ -353,6 +462,17 @@ func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []s } } +func metricExists(rm metricdata.ResourceMetrics, name string) bool { + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return true + } + } + } + return false +} + func findMetric(t *testing.T, rm metricdata.ResourceMetrics, name string) metricdata.Metrics { t.Helper() diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index be982f74..3c7f94bf 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -107,8 +107,8 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err != nil { return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, target.Target) } if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), "", stored.HypervisorType); err != nil { return nil, fmt.Errorf("prepare source snapshot memory for copy: %w", err) @@ -224,8 +224,8 @@ func (m *manager) deleteSnapshot(ctx context.Context, snapshotID string) error { if err != nil { return fmt.Errorf("wait for snapshot compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteSnapshot, target.Target) } if err := m.snapshotStore().Delete(snapshotID); err != nil { if errors.Is(err, snapshotstore.ErrNotFound) { @@ -274,15 +274,15 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if err != nil { return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, target.Target) } target, err = m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) if err != nil { return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, target.Target) } if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { @@ -403,8 +403,8 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, target.Target) } if err := m.ensureSnapshotMemoryReady(ctx, m.paths.SnapshotGuestDir(snapshotID), "", rec.StoredMetadata.HypervisorType); err != nil { return nil, fmt.Errorf("prepare snapshot memory for fork: %w", err) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 5c930002..78d35312 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -35,9 +35,17 @@ const ( type compressionJob struct { cancel context.CancelFunc done chan struct{} + state compressionJobState target compressionTarget } +type compressionJobState string + +const ( + compressionJobStatePendingDelay compressionJobState = "pending_delay" + compressionJobStateRunning compressionJobState = "running" +) + type compressionTarget struct { Key string OwnerID string @@ -46,6 +54,29 @@ type compressionTarget struct { HypervisorType hypervisor.Type Source snapshotCompressionSource Policy snapshotstore.SnapshotCompressionConfig + Delay time.Duration +} + +type canceledCompressionJob struct { + Target compressionTarget + State compressionJobState +} + +type compressionTimer interface { + Chan() <-chan time.Time + Stop() bool +} + +type realCompressionTimer struct { + timer *time.Timer +} + +func (t *realCompressionTimer) Chan() <-chan time.Time { + return t.timer.C +} + +func (t *realCompressionTimer) Stop() bool { + return t.timer.Stop() } type nativeCodecRuntime struct { @@ -80,13 +111,32 @@ func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snaps return &cloned } +func cloneDurationPtr(v *time.Duration) *time.Duration { + if v == nil { + return nil + } + cloned := *v + return &cloned +} + func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { if policy == nil { return nil } return &SnapshotPolicy{ - Compression: cloneCompressionConfig(policy.Compression), + Compression: cloneCompressionConfig(policy.Compression), + StandbyCompressionDelay: cloneDurationPtr(policy.StandbyCompressionDelay), + } +} + +func normalizeStandbyCompressionDelay(delay *time.Duration) (*time.Duration, error) { + if delay == nil { + return nil, nil + } + if *delay < 0 { + return nil, fmt.Errorf("%w: standby compression delay cannot be negative", ErrInvalidRequest) } + return cloneDurationPtr(delay), nil } func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { @@ -147,6 +197,28 @@ func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, overri return m.resolveConfiguredCompressionPolicy(stored, override) } +func (m *manager) resolveStandbyCompressionDelay(stored *StoredMetadata, override *time.Duration) (time.Duration, error) { + candidates := []*time.Duration{override} + if stored != nil && stored.SnapshotPolicy != nil { + candidates = append(candidates, stored.SnapshotPolicy.StandbyCompressionDelay) + } + + for _, candidate := range candidates { + if candidate == nil { + continue + } + normalized, err := normalizeStandbyCompressionDelay(candidate) + if err != nil { + return 0, err + } + if normalized != nil { + return *normalized, nil + } + } + + return 0, nil +} + func (m *manager) resolveConfiguredCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { candidates := []*snapshotstore.SnapshotCompressionConfig{override} if stored != nil && stored.SnapshotPolicy != nil { @@ -256,6 +328,26 @@ func invalidateNativeCodecPath(runtime nativeCodecRuntime, algorithm snapshotsto } } +func (m *manager) newCompressionTimer(delay time.Duration) compressionTimer { + if m != nil && m.compressionTimerFactory != nil { + return m.compressionTimerFactory(delay) + } + return &realCompressionTimer{timer: time.NewTimer(delay)} +} + +func stopCompressionTimer(timer compressionTimer) { + if timer == nil { + return + } + if timer.Stop() { + return + } + select { + case <-timer.Chan(): + default: + } +} + func recordNativeCodecFallback(ctx context.Context, runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, reason snapshotCodecFallbackReason, nativeBinary string, err error) { logger.FromContext(ctx).WarnContext( ctx, @@ -302,6 +394,11 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar return } + initialState := compressionJobStateRunning + if target.Delay > 0 { + initialState = compressionJobStatePendingDelay + } + m.compressionMu.Lock() if _, exists := m.compressionJobs[target.Key]; exists { m.compressionMu.Unlock() @@ -311,6 +408,7 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar job := &compressionJob{ cancel: cancel, done: make(chan struct{}), + state: initialState, target: target, } parentSpanContext := trace.SpanContextFromContext(ctx) @@ -318,12 +416,97 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar m.compressionMu.Unlock() go func() { - start := time.Now() result := snapshotCompressionResultSuccess var uncompressedSize int64 var compressedSize int64 var spanErr error + log := logger.FromContext(ctx) metricsCtx := context.Background() + var compressionStart *time.Time + + defer func() { + m.recordSnapshotCompressionJob(metricsCtx, target, result, compressionStart, uncompressedSize, compressedSize) + m.compressionMu.Lock() + delete(m.compressionJobs, target.Key) + m.compressionMu.Unlock() + close(job.done) + }() + + if target.Delay > 0 { + waitStart := time.Now() + waitSpanOptions := []trace.SpanStartOption{ + trace.WithNewRoot(), + trace.WithAttributes( + attribute.String("operation", "snapshot_compression_wait"), + attribute.String("owner_id", target.OwnerID), + attribute.String("snapshot_id", target.SnapshotID), + attribute.String("hypervisor", string(target.HypervisorType)), + attribute.String("source", string(target.Source)), + attribute.String("algorithm", string(target.Policy.Algorithm)), + attribute.Float64("compression_delay_seconds", target.Delay.Seconds()), + ), + } + if parentSpanContext.IsValid() { + waitSpanOptions = append(waitSpanOptions, trace.WithLinks(trace.Link{SpanContext: parentSpanContext})) + } + waitCtx, waitSpan := m.tracerOrDefault().Start(context.Background(), "instances.snapshot_compression_wait", waitSpanOptions...) + timer := m.newCompressionTimer(target.Delay) + log.InfoContext(ctx, "snapshot compression queued with standby delay", + "owner_id", target.OwnerID, + "snapshot_id", target.SnapshotID, + "source", string(target.Source), + "algorithm", string(target.Policy.Algorithm), + "compression_delay", target.Delay.String(), + ) + + select { + case <-timer.Chan(): + m.recordSnapshotCompressionWait(metricsCtx, target, snapshotCompressionWaitOutcomeStarted, waitStart) + waitSpan.SetAttributes( + attribute.String("wait_outcome", string(snapshotCompressionWaitOutcomeStarted)), + attribute.Float64("wait_duration_seconds", time.Since(waitStart).Seconds()), + ) + waitSpan.End() + case <-jobCtx.Done(): + stopCompressionTimer(timer) + result = snapshotCompressionResultSkipped + m.recordSnapshotCompressionWait(metricsCtx, target, snapshotCompressionWaitOutcomeSkipped, waitStart) + waitSpan.SetAttributes( + attribute.String("wait_outcome", string(snapshotCompressionWaitOutcomeSkipped)), + attribute.Float64("wait_duration_seconds", time.Since(waitStart).Seconds()), + ) + waitSpan.End() + if target.SnapshotID != "" { + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateNone, "error", err) + } + } + log.InfoContext(waitCtx, "snapshot compression skipped before start", + "owner_id", target.OwnerID, + "snapshot_id", target.SnapshotID, + "source", string(target.Source), + "algorithm", string(target.Policy.Algorithm), + "compression_delay", target.Delay.String(), + ) + return + } + + m.compressionMu.Lock() + if currentJob, ok := m.compressionJobs[target.Key]; ok && currentJob == job { + currentJob.state = compressionJobStateRunning + } + m.compressionMu.Unlock() + log.InfoContext(waitCtx, "standby compression delay elapsed; starting compression", + "owner_id", target.OwnerID, + "snapshot_id", target.SnapshotID, + "source", string(target.Source), + "algorithm", string(target.Policy.Algorithm), + "compression_delay", target.Delay.String(), + ) + } + + startedAt := time.Now() + compressionStart = &startedAt spanOptions := []trace.SpanStartOption{ trace.WithNewRoot(), trace.WithAttributes( @@ -338,17 +521,9 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar if parentSpanContext.IsValid() { spanOptions = append(spanOptions, trace.WithLinks(trace.Link{SpanContext: parentSpanContext})) } - metricsCtx, span := m.tracerOrDefault().Start(metricsCtx, "instances.snapshot_compression", spanOptions...) - log := logger.FromContext(ctx) - - defer func() { - m.recordSnapshotCompressionJob(metricsCtx, target, result, start, uncompressedSize, compressedSize) - finishInstancesSpan(span, spanErr) - m.compressionMu.Lock() - delete(m.compressionJobs, target.Key) - m.compressionMu.Unlock() - close(job.done) - }() + compressionCtx, span := m.tracerOrDefault().Start(context.Background(), "instances.snapshot_compression", spanOptions...) + metricsCtx = compressionCtx + defer func() { finishInstancesSpan(span, spanErr) }() rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) if !ok { @@ -414,7 +589,7 @@ func (m *manager) waitCompressionJobContext(ctx context.Context, key string) err } } -func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) (*compressionTarget, error) { +func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) (*canceledCompressionJob, error) { if key == "" { return nil, nil } @@ -432,8 +607,10 @@ func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) ( select { case <-job.done: - target := job.target - return &target, nil + return &canceledCompressionJob{ + Target: job.target, + State: job.state, + }, nil case <-ctx.Done(): return nil, ctx.Err() } @@ -447,8 +624,8 @@ func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jo if err != nil { return err } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, target.Target) } } diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 274f9c50..ea4fdff6 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "os" + "path/filepath" + "sync" "testing" "time" @@ -183,6 +185,45 @@ func TestResolveStandbyCompressionPolicyInvalidConfiguredDefaultIsInvalidRequest assert.True(t, errors.Is(err, ErrInvalidRequest)) } +func TestResolveStandbyCompressionDelayPrecedence(t *testing.T) { + t.Parallel() + + instanceDelay := 2 * time.Minute + overrideDelay := 15 * time.Second + m := &manager{} + + delay, err := m.resolveStandbyCompressionDelay(&StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + StandbyCompressionDelay: &instanceDelay, + }, + }, &overrideDelay) + require.NoError(t, err) + assert.Equal(t, overrideDelay, delay) + + delay, err = m.resolveStandbyCompressionDelay(&StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + StandbyCompressionDelay: &instanceDelay, + }, + }, nil) + require.NoError(t, err) + assert.Equal(t, instanceDelay, delay) + + delay, err = m.resolveStandbyCompressionDelay(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Zero(t, delay) +} + +func TestResolveStandbyCompressionDelayRejectsNegativeDuration(t *testing.T) { + t.Parallel() + + m := &manager{} + negative := -1 * time.Second + + _, err := m.resolveStandbyCompressionDelay(&StoredMetadata{}, &negative) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + func TestCompressionMetadataForExistingArtifactUsesActualAlgorithm(t *testing.T) { t.Parallel() @@ -225,6 +266,23 @@ func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { assert.True(t, errors.Is(err, ErrInvalidRequest)) } +func TestValidateCreateRequestRejectsNegativeStandbyCompressionDelay(t *testing.T) { + t.Parallel() + + negative := -1 * time.Second + req := &CreateInstanceRequest{ + Name: "compression-delay-test", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &SnapshotPolicy{ + StandbyCompressionDelay: &negative, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { t.Parallel() @@ -320,6 +378,7 @@ func installCancelableCompressionJob(mgr *manager, target compressionTarget) *co mgr.compressionJobs[target.Key] = &compressionJob{ cancel: cancel, done: done, + state: compressionJobStateRunning, target: target, } mgr.compressionMu.Unlock() @@ -345,3 +404,91 @@ func assertCompressionJobCanceled(t *testing.T, mgr *manager, target *compressio return !ok }, time.Second, 10*time.Millisecond) } + +func TestStartCompressionJobDelayedCancellationRecordsSkipped(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + delay := 45 * time.Second + timer := newFakeCompressionTimer() + mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { + require.Equal(t, delay, got) + return timer + } + + snapshotDir := t.TempDir() + rawPath := filepath.Join(snapshotDir, "memory") + require.NoError(t, os.WriteFile(rawPath, []byte("delayed standby snapshot"), 0o644)) + + target := compressionTarget{ + Key: "instance:delayed", + OwnerID: "delayed", + SnapshotDir: snapshotDir, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + Delay: delay, + } + + mgr.startCompressionJob(context.Background(), target) + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + job, ok := mgr.compressionJobs[target.Key] + return ok && job.state == compressionJobStatePendingDelay + }, time.Second, 10*time.Millisecond) + + canceled, err := mgr.cancelAndWaitCompressionJob(context.Background(), target.Key) + require.NoError(t, err) + require.NotNil(t, canceled) + assert.Equal(t, compressionJobStatePendingDelay, canceled.State) + + _, err = os.Stat(rawPath) + require.NoError(t, err, "raw snapshot should remain available when delay is skipped") + _, err = os.Stat(rawPath + ".zst") + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +type fakeCompressionTimer struct { + ch chan time.Time + mu sync.Mutex + stopped bool + fired bool +} + +func newFakeCompressionTimer() *fakeCompressionTimer { + return &fakeCompressionTimer{ch: make(chan time.Time, 1)} +} + +func (t *fakeCompressionTimer) Chan() <-chan time.Time { + return t.ch +} + +func (t *fakeCompressionTimer) Stop() bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.stopped || t.fired { + return false + } + t.stopped = true + return true +} + +func (t *fakeCompressionTimer) Fire() bool { + t.mu.Lock() + if t.stopped || t.fired { + t.mu.Unlock() + return false + } + t.fired = true + ch := t.ch + t.mu.Unlock() + + ch <- time.Now() + return true +} diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 433422f2..00c9532a 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -63,6 +63,7 @@ func (m *manager) standbyInstance( // Resolve/validate compression policy early so invalid request/config // fails before any state transition side effects. var compressionPolicy *snapshotstore.SnapshotCompressionConfig + var compressionDelay time.Duration if !skipCompression { policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) if err != nil { @@ -72,6 +73,16 @@ func (m *manager) standbyInstance( return nil, err } compressionPolicy = policy + if compressionPolicy != nil { + delay, err := m.resolveStandbyCompressionDelay(stored, req.CompressionDelay) + if err != nil { + if !errors.Is(err, ErrInvalidRequest) { + err = fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return nil, err + } + compressionDelay = delay + } } // 3. Get network allocation BEFORE killing VMM (while we can still query it) @@ -220,10 +231,18 @@ func (m *manager) standbyInstance( finalInst := m.toInstance(ctx, meta) if compressionPolicy != nil { + log.InfoContext(ctx, "enqueueing standby snapshot compression", + "instance_id", id, + "operation", "enqueue_snapshot_compression", + "source", string(snapshotCompressionSourceStandby), + "algorithm", string(compressionPolicy.Algorithm), + "compression_delay", compressionDelay.String(), + ) compressionCtx, compressionSpanEnd := m.startLifecycleStep(ctx, "enqueue_snapshot_compression", attribute.String("instance_id", id), attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "enqueue_snapshot_compression"), + attribute.Float64("compression_delay_seconds", compressionDelay.Seconds()), ) m.startCompressionJob(compressionCtx, compressionTarget{ Key: m.snapshotJobKeyForInstance(stored.Id), @@ -232,6 +251,7 @@ func (m *manager) standbyInstance( HypervisorType: stored.HypervisorType, Source: snapshotCompressionSourceStandby, Policy: *compressionPolicy, + Delay: compressionDelay, }) compressionSpanEnd(nil) } diff --git a/lib/instances/types.go b/lib/instances/types.go index fb8f355d..0df715fe 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -268,7 +268,8 @@ type CreateSnapshotRequest struct { // StandbyInstanceRequest is the domain request for putting an instance into standby. type StandbyInstanceRequest struct { - Compression *snapshot.SnapshotCompressionConfig // Optional compression override + Compression *snapshot.SnapshotCompressionConfig // Optional compression override + CompressionDelay *time.Duration // Optional standby-only compression delay override } // RestoreSnapshotRequest is the domain request for restoring a snapshot in-place. @@ -286,7 +287,8 @@ type ForkSnapshotRequest struct { // SnapshotPolicy defines default snapshot behavior for an instance. type SnapshotPolicy struct { - Compression *snapshot.SnapshotCompressionConfig + Compression *snapshot.SnapshotCompressionConfig + StandbyCompressionDelay *time.Duration } // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index d4d336f0..e4138528 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1167,6 +1167,9 @@ type SnapshotKind string // SnapshotPolicy defines model for SnapshotPolicy. type SnapshotPolicy struct { Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // StandbyCompressionDelay Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Applies only to standby compression and defaults to immediate start when omitted. + StandbyCompressionDelay *string `json:"standby_compression_delay,omitempty"` } // SnapshotSchedule defines model for SnapshotSchedule. @@ -1220,6 +1223,9 @@ type SnapshotTargetState string // StandbyInstanceRequest defines model for StandbyInstanceRequest. type StandbyInstanceRequest struct { Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // CompressionDelay Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Overrides the instance default for this standby operation only. + CompressionDelay *string `json:"compression_delay,omitempty"` } // Tags User-defined key-value tags. @@ -15316,258 +15322,260 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZIw+ioIntloaoakqItlWRMde2TJdmvbsnUs2312mv4osAok0aoCqgEUJdrh", - "v/MA84jzJF8gAdSNKLIoW5I19sbGtMzCNZGZyEzk5VMr4HHCGWFKtg4+tWQwJTGGPw+VwsH0PY/SmLwh", - "f6ZEKv1zInhChKIEGsU8ZWqYYDXV/wqJDARNFOWsddA6w2qKrqZEEDSDUZCc8jQK0Ygg6EfCVqdFrnGc", - "RKR10NqMmdoMscKtTkvNE/2TVIKySetzpyUIDjmL5maaMU4j1ToY40iSTmXaUz00whLpLl3ok4034jwi", - "mLU+w4h/plSQsHXwe3EbH7LGfPQHCZSe/HCGaYRHETkmMxqQRTAEqRCEqWEo6IyIRVAcme/RHI14ykJk", - "2qE2S6MI0TFinJGNEjDYjIZUQ0I30VO3DpRIiQcyIaxpSEPPCRydIPMZnRyj9pRclyfZfjzab9UPyXBM", - "Fgf9JY0x62rg6mW58aFtceyXu76RKY/jdDgRPE0WRz55fXr6DsFHxNJ4RERxxP3tbDzKFJkQoQdMAjrE", - "YSiIlP79u4/FtfX7/f4B3j7o93t93ypnhIVc1ILUfPaDdKsfkiVDNgKpHX8BpK/enxyfHKIjLhIuMPRd", - "mKmC2EXwFPdVRJvyqfjw/2lKo3AR60f6ZyKGlEmFWQ0OntiPGlx8jNSUINsPvT9F7TEXKCSjdDKhbLLR", - "BN81w4qIIuEQq8XpYKnItqGcIUVjIhWOk1anNeYi1p1aIVakq780mlAQvGI63aLRZIuklpqTHMaybnTX", - "BFGGYhpFVJKAs1AW56BM7e3Wb6ZAMEQI7uFQz/TPKCZS4glBbc02Ne9mSCqsUomoRGNMIxI2OiMfIpjN", - "/MFHiIaEKTqmZfo26NTFo2Bre8fLO2I8IcOQTuxNVB7+GH7XKKbHUQha+zeiCW3ebB8wpSDjxfmeA+uG", - "SQQZE0E0jn/hdIngM8I0tej5/gLztv6fzfyK3rT38yYA8yxv/rnT+jMlKRkmXFKzwgXOZb9oNAJQI+jh", - "XzN8WnbWBYySCovl9AEtvgIlmvU1gs25afq501J4srLLW92myjuBNdopS1yglkU+mxHmEZICzpT9UIbO", - "Sz5BEWUE2Rb2LDRP1BP8HHFgiV8JDhn4F4lfr/sGzMv8UDOa/tZpEZbGGpgRnxShOSVYqBEpAbPmCrMD", - "5aurBf9ZiXwqdxWWZLicg5xRxkiIdEtL2KYlSiVIqgvbByq6pGo4I0J6aQ6W9StVyLaoHSriweWYRmQ4", - "xXJqVozDEOgVR2elnXiktZL4ixPNBN2AIEVIpDg6/+Vw+9EeshN4YCh5KgKzgsWdFHrr4U1bpLAY4Sjy", - "4kY9uq1/Ry9iiB8DzjPCqLt7Mgx0iGk4Xcueph6+00pSOTV/Ae/Wq4K7T7MBjV6R/vuDZ9NHwCSMllCr", - "M/llwNeJOWw0ibiG6RyljP6ZlgTsHjrRuoJC+qKgIQk7CMMHzbJxqnh3QhgRmk+hseAxSFsFIRi1SW/S", - "66CBlgu7Wgru4u1uv9/tD1plMTba7U6SVIMCK0WEXuD/+R13Px52/9HvPvmQ/znsdT/87S8+BGgqmTup", - "0O6z7Wi/g9xii+J6daGrRPkbc//i8n0cxxz1ieYT65700cmi4GD2GvLgkoge5ZsRHQks5ptsQtn1QYQV", - "kaq88+VtvyosYB9LgMAmGkxrgqGi9AAatyN+RUSgOXBENOLJjmbCVMkOwlpvBuaF9C35dxRgpmnBCBdc", - "IMJCdEXVFGFoV4ZWPO/ihHapWWqr04rx9UvCJmraOtjbWcBzjeRt+0f3w1/dTxv/7UV1kUbEg+RveKoo", - "myD4bG71KZUoXwNVJF55Ig66aQRiXkzZiem2la0EC4HnX37CbiPLTtooc7VHHcQeyf/1jAhBQ3erHp0e", - "o3ZEL4lFdyRShgZpv78TQAP4k9hfAh7HmIXmt40eeh1TpW+zNL+kjTWoVzzu31skmHKQM6KI6w1loK4R", - "YnIYBoKAfoKjpdfwMhB7gXWUjbt4af/CperGmOEJAW3SNkQjwS+JXihKeEQDSiS6JHMtpMzRRA/anVFJ", - "NfkQNkMzbIwGvQF7O+WSmCbuk1ZEAkJnBMU8uERJhAMy5aCIz3CUEtlBV1MtMWhmLAiO7M9IkBhTNmBT", - "vUgZ8ISEWocwzWBr6IKw2QWKcQJUigUBEkUxVkRQHNGPJETcdIlJSPUFNWAE8BolWJNsEHChb199tgQH", - "0wIUfpLowsgbFzD8BWUaKy8MXfUGrHjyn1qv3719+vrdq+Ph67Nnrw5Phr8++1/9s+nUOvj9U8vYNzNB", - "4ynBggj0l0+w389GOg2JaB20DlM15YJ+NMaWz52WhoHU+IUT2uMJYZj2Ah63Oq2/Fv/54fMHJ0/pqQib", - "aTLwLOyzV5YxV6GHoxw7Y55E1kAEoh0GUy1wmBdn7zb15ZpgKdVU8HQyLROGvdnXIomQyssh5cNR4lsT", - "lZfoZPM10nIHiqgm0EzO2Or3T59uykFL/+OR+8dGDx0bqoXlaxbChRV/5FSjjxbCAWWOzt4hHEU8sCaQ", - "sdaVxnSSChL2KpY3GN3HnwlTYp5w6tPBKswpb7rIo7rd/OsarGhzRNmm1MfQDdaDO+DNjTWBZ2xGBWex", - "1sZmWFB9zcoyrbx6ffxs+OzV+9aB5uNhGlij4tnrN29bB62dfr/f8iGoxqAVPPDF2bsjOClDNiqJ0slQ", - "0o8eSeAw2x+KScyF0YBtH9SelgUFQ7cIDmfQ2nnx1CDX1gvAK3coIZXQ2o1iBi5jzPaLpz5smc4TImZU", - "+sxkv2Tf3MkXrnXD7su4LYmYEZEhLWBxr6B+BBFPw25hyk5rTAUJBNZo1+q0/iSxlsNnHzXq5Gv39PNb", - "rxrJnysESxwllJElkuU3IuFdcXEZcRx2t76ygMeI0mMvbvGV+VA+X4sTJEOJVmfBGsHCKxqq6TDkV0wv", - "2cNX7ReUNc6Y67XeCY7+/c9/vT/N1aStF6PEctqt7UdfyGkrvFUP7TWBZBtJE/823iX+Tbw//fc//+V2", - "cr+bMILIjYQ6e/7PzAjAsjWuh6VnSmPNLIPltylRUyIKt7dDFv2T0YehO3K4V9hKyTxafNNcYNR8RkSE", - "5wXGa9fU2uoD96usSlAFtGr7aTZ6iXTnFWxYj+Yu+RdVHX2772e0nkV51vRU8wp7LzRZSbaQre1T++f2", - "4pJqVnRJkyFIzUM8yUy2y16bzy9pYkVx6GGOMYoMIwhTEN5HnKvegP02JQzB2cEBk2sSAM+TCit0eHYi", - "0RWNIjDwAFNZvFq0YJ+zFdNcKv2/ImUdNEqVlta5IsjqTTBJCmuBxiOCUobdc3ZFdrYbrOKVBcslEYxE", - "QyMby4aQMZ2Q7VQLHNjqGEtFhOH2aVKG1/Gvp+eofTxnOKYB+tWMesrDNCLoPE00P9goQ68zYIkgM61C", - "sAkYG6mdl48RT1WXj7tKEOKWGMNgmYnMvrXOXpy9s6/1cqM3YG+IBixhIQlhze7GkUhNsUIhZz9piiVh", - "edji/BWg+2m505IMJ3LK1RA0v/kq7nRum5+Z1mvZAjqtWZCk5SPdrh7nK3iQ18CbUaFSHGleWxInve/z", - "xvPDozYYx5Ki+mL5XobdWJUfVpsaTMzI4AayKFT77R5GUmps9yio8gsWEKdnfmq22BXjnzC3kKV2n1zV", - "/IK5zs0gVRDZsTtuZzeA0kkGkzKs8NcBz6EsqOa1dvWQSEWZQSfdFlmJUKL2hdbmLR5r/f2igy7+WvpB", - "075TLbR8cYUMNICfMP1TcfyqUWKluaC5Ulg5HCxvfh6HstZRCc22kBKYSX23ahkrIT30CzBxpEicaE7G", - "JohKJA3zJSFi/OrviBuhxnUdML00adw8LDgyo5GkE0bZZEOL+fpiwmFoLEvjVKVCt5tRmUOzjDrOelPd", - "wFuzOmL4cZxKfSMHURoSdOEsPBdluXDR/rOoElqD0IKGY0ACmg0oe2ozTpWeXm84xiqYajjxVBm/L7t1", - "WV5A2cq06j3UriV7KbvB+Z9n7KIMVGtvqDB+vTn7RgNmwYJ9ss4MaAUVv4nykszhyJ05Ei8YJIuWSL+9", - "UBDJoxmx127RljnCwaW5SozrhTVjGoOktUFq8q+QqNc6t+ooNLwag7+sKiyiEpiA7WZzjLHSv7H/zjMu", - "pDdn5utoxVgSAD6oHgcIxLGLjtGVCFggENPIEqGQChKoheEpmwwYuIBc2F96drQLTeRaRvERoU/Z8cqC", - "BW3H9CkdLSqcrBP7YBi9NR5TpUjYKcsGl4QkcvWmtHhtDdce67ogV4I6RmYNRmFD8YywMRcBia2S8GWK", - "47PCYF41br0hFj0yDHwLa7b4hHCSRJSExv3HnAeYWaU9J7CxVl1+w4rWZjwAylNe4Ci6QG3baAMJovci", - "3VkxznJkf3t05lAge7V+f9rRGKm5wMVUqWSo/0cONRVfVAezfR2F6+H0nSTRfh/0q93dHXuq1uhmFlwZ", - "tmxf83o11B+NE7/rH8Z4rGnRuYk0EeWP8i65JfWSsrDpAL/qtrXWuUwwcprGbRvoEkG6aTIRGDxkv6Z5", - "7sbPngDNeg6+wvnd5+WYQTVIpeJxwdcRtSseGrTsy1EG1oxH3RArDKbMhvZWs9xFv+F4boYyulidJWY4", - "GXncfuhHzXXRhE7waK7K7wdbfZ/G96Vv0G4tvmOp8783GiQJh4ov90CmY+TaNnE4hPtkqPhwNqaekbNr", - "LXdfoRIFFWd/q9fqIbpJQK05AWScYGocRA0QQGh8f1p8u+sNWBeu3wN0nE2QDZsNiUG2xKF5OWlzUVgE", - "Ba8zNJpvIIzen/bQ22y1P0mkFZYZcQEJUyzRiBCGUjA9w23YNXdxcQGphEtTVbtb24mJXdiAJ0puv/XQ", - "L/OExNjaoTQpxFjRADydRrSyH7iOzEHZN2HMilawRlarZX7bb8iESiUqXtuo/eb50c7OzpOq/XL7Ube/", - "1d169Harf9DX//+P5g7eXz88wzfWYZm3WN+xIvc5endyvG2NpeV51Mdd/GT/+hqrJ3v0Sj75GI/E5I8d", - "fCcBHH5Wdpw7vaF2KonoOjapscrn6lbwKKtxZbuxh9otOZzl/rPL2hpIvNUtbyMyxefzbD1u148dqTLM", - "lV7Thc0tavLzBPTOnEoKEpx1Tgyo1w3zmMrLp4Lgy5BfMc+9HeMJkUNzn/n9GVJpnGzItbVuCM7VWJp3", - "07LVc2v38e7+zt7ufr/vCchYRHge0GGgb6BGC3h9dIIiPCcCQR/UhgevEI0iPioj+qOdvf3H/Sdb203X", - "YZ54msEhU7xcL9S2EPmbC+5zX0qL2t5+vLezs9Pf29vebbQqay9utChnWy6JJI93Hu9u7W/vNoKCT6B/", - "5gJkqgJ86HNd0PqTeWzsyoQEdEwDBCE2SHdA7RiuMJK9VpVpcoTDoTWe+O8OhWkkl3pMmMlsS2Noi9NI", - "0SQi5hscSCNbNOz8GEbyeaNQxogYZvFDa4xkw4pWegi4vWRNUCk8rAS6UypBCsmFJ0qi8MBQ6Eo+B6eZ", - "L+xDHR7YPTTEhpdadepGZEaiIhKYq0svNuaCoAxPzKGVdkXZDEc0HFKWpF6UqAXl81SALGoGRXjEU2We", - "GeHAipOA0zLoHmPNrpvpuc+5uFzp/qlv4qFIGdPDrLQKHYIhfWxNNXCLY2R7uwiDgtCXPQeaR1P7XaI3", - "poexEOU/J6lClCmutVMWjuYdmMlakhgSRCoOnNQaDO0wTaVLv9wCxlLn/mHmy3nnHfm+dMfGXeDrathi", - "QtRQKqxWSiwaU95C+3No3tibXHdcaUhpAHdGru4C6OBu39Vo25UMJ7cD8WXOaJmtIW8Et7CgIekhoC7w", - "inHhfRVKO1c8SUiY2X96A3ZuSCX7SZoXFN3RwEFNCRWICzqh5YnLBrbb9GpbBxUdNt0YHYsdFyVU+Aju", - "G/VEj8eKCANBF7lcDD+yh9DqtCzsW52W5URl0LgfPRDJXS0Xlvji7N26vmmJ4GMaebYLvhD2q9XMnNfW", - "y93+eXfr/zMemBrfQESjzPhPxDwkvUpyAGjf7OZ5cfburG5NWWYGVFzdwp4yjxcP58j8GhxE7KOSfZW0", - "GoxDf32xZJPksvcTnyw7Fjgmo3Q8JmIYe4xrz/V3ZBoY1ybK0OnTsjyr5eamWvNZ6XBAbR7jwAbWN4O+", - "xyBX2UanAM0P/uN6Q8w1XBeOp49K2DY2Iq+HXmW5MNCLs3cS5V5KHktd+Xhr/eXPpnNJAxyZEU10LWVF", - "AxsgZ2MJ+SzvaE2RHjk59sqGjhBQezZJUiDD8zfdk9fvN+OQzDqlNYFn0ZRHRK97o8AtZi4oL3fuLzGJ", - "WZ2lwyCGbEpABVhlFNwYSAV69UBHcYWjoYy4z1njrf6I4CNqv39ugqb0CjooKR2l/r0AhRJ+73kpRnOk", - "umnPYcKqybRE4F7dsZxCxphXCtsrTeojlV8IjkzmnDI+5/Hd7uD5Zfmg+eVK6rWD+OY9cY7hDYK3jk6P", - "jcAQcKYwZUSgmChs8/QUXFxAHGp1Wl19R4WYxOBqN/77cu+WGhN8MRqr1oh7tJB241YMuDXh4m+MC0KI", - "YszomEhlw8VLM8sp3n60d2CSWoRkvPtor9frrRuj8iwPSml0FJvGhb8QrtKT0y87h1sIRWmyl0+ts8O3", - "v7QOWpupFJsRD3C0KUeUHRT+nf0z/wB/mH+OKPOGsDTKg0LHC/lPyk+a+s4yvx/onTDrEqZxiYMCv/KJ", - "qUafAc8GiJvzhgsrPNH6icG4L40LvnHmkDx9lSpkDCk6hDbIHkI/LreEOsEI2tg5U6ZolCdWWbSB3ig1", - "jlyaPWAhc0BCWJYvIIrMXwFnM00VvuQBJQbuvn3R+4H1chmG1IPJv1ltzzhJQFTVanprbeIkWY22fkEx", - "439Nk6bY0GbPTXTvXP8mb2zl2V9P/ufP/1+ePf5j68+X79//7+zF/xy/ov/7Pjp7/UURVMuj2u81NP2r", - "RaPDw1IpJL0pKp1iFXgEqimXqgbC9gtS3Phr9tARKH4HA9ZFL6kiAkcHaNCquAgPWqhNrnGgTC/EGdJD", - "2UiHDd35zJh/dOdPTrf8XB0jtCENwh5IFskk01HIY0zZxoANmB0LuY1IeNPXf4UowIlKBdGnp2XYaI5G", - "Agd5KEM+eQd9wknyeWPAQMMl10roHSRYqCwNh5sBkMKuyvgM2OYkdIHhRkMesOxeyuLCjY2mlxlBwDZf", - "9bj0A8WrvnBRDsXZ7/si6MHrSx9kRKUi4JidYbZGo8wdDe33S6xiv7/fXyngZzi0BP2AEhaTZDqkbEBL", - "BoFhasO4wUOtgS1d8yZDI+iXt2/PNBj0f8+RGyiHRXbERskzPoDS2AhVJAvefxstn+nbnG7DDRkjGXSL", - "GkQNPTPuoW9fniNFROwc9tuBBueYBnp/8PxPpUw1KlKMDo9On230GmT5BNhm619yjm+zHVaDO6zRrM4W", - "mGG8hm8HnRyDe66l0FyAA7ea51ygyDCYnK4P0DtJyr6ucFTmVd+cZDTPLW/mBhi0NtyISZVTHKA3mdyI", - "s6VkjpY5Mrghc7qEYe3Di/H5WRi94pcL3kxWL7KsDTx8sMqcxPWNW88KlpO/B+JA89avu2DTXI+2i8ZQ", - "PZkfNfKzv3VpZWddHXXdBA3lGMpC/G2Wo6F5coXbSFKwqK9dUzWsfYRH+rN9cndayftTNMWS/aTgY0U3", - "2dp53Chbpp616fN18eGaj82SMqpyAZnZs6sJTb2kUWS8GSSdMByhJ6h9fvLi15OXLzdQF71+fVo9imU9", - "fOfTIFeDQ+0XZ+8g2gXLoXsBqnd6xLnjMLmmUsnFeNVGD6nLc0P8Usrf4A0A3viKSR3c6/PCNu4iXcN9", - "uvV9e6kiliZ3+NIMDVbYvaUEDbXM1ZfcoMxnzc9fN9XCrSynFPvj4w9FmcD5XN84t0GnRT3+podSs0AS", - "opOzPMVhbpRyw1f29GS7t7W339vq93tb/SYmuhgHS+Y+PTxqPnl/2xgiDvDoIAgPyPgLTIQWsY3whqMr", - "PJdo4MTrQcvI8wVBvkC2VgRv9Py6mELiZhkjqgLFqpwQ6+SAaJbc4Usj6pclOj4vpzhuLOQ9+scXZUMm", - "Ta926/tgew3XsX4TFPA0CrUgNdKka/QyElr1URKVZ48Gan/HLhm/YuWtGyOoZgB/pkTM0fvT05LJXJCx", - "TY7bYOPgM1FzDjxZ6xi2V8jaK1dzwzwLd5Fbocp2C9fdV8+kULTZOR9Mg6ENbHe5+Ol9N6fMHI3GkyV7", - "qlhdQjIbpqlPqtKfXOTFu3cnxyXkwHhva7+//6S7P9ra6+6G/a0u3trZ624/wv3xTvB4pyY9fXO/mZu7", - "wpSpuT7SCQAPFkwTyBYeaHrLfFlGqUKZn5sm5CMtnqKCHGziesCocMKoghyOlE30MKDjWzHZBGiaNJOU", - "UQUZASAfDWV6y2BM0YNY76UD9ALawiccQ7yRW4RWjsp2BBzOjR1VMwY3dQL/Wr7k82mqtNwGfeQ0VUj/", - "C7atwWDVleVDGB5zgF5x6COckynjVb3HNAfnrcXmVR2pbd2KnPspTGYZ5gF6njHJjM1attqWxP5peLf1", - "jAav742S75098ZbGlvzkCm5lnZaBaKvTcoAC97NFRzS7Lm+MRREVfQ8MBEfAQnNHn1TRyCY5gJ1QqWhg", - "tEYMh1tHyTahFwmHRgSoey403iNWTMg6OUbx/hS1IZzxb8gqlfpfG9nTYpEqd7ef7D7Ze7z9ZK9R0EK+", - "wNUM/gh8mxYXt5LbB0k6dJU/arZ+dPYO7j59r8o0NlYCu/eCj2gieKClVcpQXkokn/xJ70kxViPk6Sgq", - "WJ1sYBcEBDSp+1LzPvYnjWZ0PGZ/fgwut/8QNN663pPbI69yl03kl4RPipbSBbWRjLomCaPfnR4QSsja", - "iJM3RMIO0DlRCPCni3AAl3TmkmRRzsWlWIh7EWt3Z2dn//Gj7UZ4ZVdXIJwh6K+Lqzy1KyiQGLRE7Tfn", - "52izgHBmTOenCfkhmBXg/HSGbD7mfsmFU+tOOz4sqZGXcqyxY8/iWpC/t0KQ3ZQFOnhWZQLSApV7ob2z", - "03+8+2j/UTMythrbUFwv5zAuJ4cBj01jUjz5NljX3x6eIT26GOOgrKFsbe/sPtp7vL/WqtRaq4IUPCZ1", - "xhoL23+892h3Z3urWeiUz4JugwJLBFvmXR6i8yCF5zQ8oFhkvZ2628IneBoEe0OCCNP4MHDeL5Xbx6TI", - "GArTLD+EJheDNRIsXFwN+jZS0Sple4xowAVKWZaYqbfaHHoz62Y9mzb3wWo2vihDR5hpcFkff5OJ8Qaw", - "SwSZUZ7KrzAQVyTQyDSOOBdr9a1zJ3pDZBopY4KkEr0//QmYiEYuJBVJyq7yFv2WRELccHNrEXAJJ/xY", - "XQesRqfR5OiXbbhTQ6adZW6wJfKvDTgKNatK2eqn6yMcBSnkHsPZeepdQegATxU8tM+Nk0cUcc5QMMVs", - "QiCXu8l0yCYIoymPwl7L/1QShcOx9wmDX6GIm1QJl4QkNi2XWYTupmUWOiOo/YLnBeUMKlXS6z6KDVex", - "iZfK2PgorinOKX2Og1mAkoYnVrwQxW+6lLT5iE8kaIEK3Fd61eQxCRbGKwUzk2ZuFhvlsRx5ta1ve88S", - "K9zbd4Waq5OPrUZrZQzFM0jiQHApEYnoBFKavT8tL3OZ/2FMGY01n139HF1ebAPUlQln0pcXBe402Tgb", - "pe9C9Dh2fcmVCDgM/pvepwPzkG9d8VGMWQqJugqITK4TKgx6NHscn3Kphlk0yZqLlWoISZhSQfKQM3df", - "TsF/f25YHLTx3ouOtd0EXNZr4ka9F7DKP1TdAut5qheifmh1Mhz0ofFiPM3SEJ48JqgaALJOxFeetYdK", - "GJUWgo1Qm3FVYkuFzDMbTR6q/DqqnqeuoOvL3f5502Cs5bFXZ1hNT9iYe1I7rmHwtx7tznchISKmkIYM", - "hYRREjrlMbP8W9sW+MhHkqAwJRZyRiAV2AIcG/KGFI7MGcUom1R4fXXCJmZ4s4blOZpgXtuwyZOj9HtW", - "vxUpwMo4CUiEcx/rRh4PVA79luLFgQWZpBEWqBpwuGTJch5HlF02GV3O4xGPaIB0h+pzzphHEb8a6k/y", - "Z9jLRqPd6Q7D3EWw8jxjFmcdRM2BVObNt/Cz3uVGxT0dTC+bpv8mVOxu8oLr9Rt6TiNiY/LeMXpdQPRy", - "EpPd7X5d5ELNoKWYhcV4znU5t0VZH8W7UMvDrOiBxz/NeABVXiXKhsjSfn27BRezZXEai6YY1HaPwi5J", - "TBmuhWQtjSwhzbzcqu4PbjWbkgTl2Xf3Hz3ea5gt54tsnUtqGn+BZXMWL7Fo1pzUaROz2f6j/SdPdnYf", - "Pdley0DlPGVqzqfOW6Z4PpXaJhWj2aM+/N9aizK+Mv4l1fjLlBdUqlNy4wV9XkK6eZR0zbNHbZruqHiS", - "7p2lbAFtZmNcIi0dlkSuQimuNhmPCSiVQwO3br6Yind9ozUEOMEBVXOPwQRfmYztWZNKtG8Ta1p5sR6Q", - "2rFtwgbNuWQ6yh06225y9FdjWq/gwn7jpFsyHdWZ8V9XZzVG/NwGVHwiavBCk9cFWDQXZPu5wrLk1aH/", - "DiDlcl5qreo/ZFo0rwntcD0rC517Rvoi1v0loIvHXznOgtm3JCRXIb7sCq0nwbV0aM+N7Ksyudort8If", - "7AV4s17DUTEd3tJ8g6Xcefmtu/68zYrELfYzN9j68xVcQNfpWM0MBvho12BBno/dKaFEDTYpLlYnhL6F", - "/D7Gp+BGGX6sO8KdJPmxP99KYp+F4zgnyrU91xp9Gi3J58wUETPsMUy5IZBrUrajGk7cQdbEh7bijUql", - "wd2pX1azEbgN3ce0FDhMBBnT6yXYYhqY67rsPi4tBMJyzm+J2jG+RruPUTDFQlbWzuhkqqJ52ci664me", - "+LISykRp0bl5dvT8NF3HxRcNe5zF0X0ke16IdfBnbSfhcFmc+lHWzNmMEzwH2bJWEXy8s9vv72z3bxSo", - "/rWSyRfGqfMILfSzxpzS02NxhMz/czHj4JWgpiaZA5NUguD4ALypEhwQFJExhHFlmV5X6vQLUy9fvH0k", - "tY7/Gf67g3KFR62dxbI4xhkIHm4cG+PvttFyr7TlWI/i98VlLwkWy9hMsBA1VvVf3ev2d7r9vbdbOweP", - "9g62tm4jsj0DUp0Lz+OPW1ePo2083o3254//3Jo+nmzHO95wgVuoW1ApA1gpY2D3kBBRTSVZTcEqSUQZ", - "6crM7W21A/ISXmBeklbS/3rWB7ODpcLCeXmTRZkBqxw41YpqdxHYZFe/1IRSXf7J8fJl38iPrLoQP4JV", - "lwL41GwxkGll60vTeqSs4b3zrtCw8c2z1Ldx1d3jc/oG0vaecg3EffhcYowlClt2Yy/eah4VbsIFVdN4", - "+fWQNcuSBMBj+EepwnIgTQ+dTBgkji3+nL19FGs7686tTiv6uFumGft785AqGxSfIaA96qIY0OBtAPIS", - "L4cCNMlVC2HcE7AgAIift7pbT+CFPvq4+3O/+6SHfit4CnQMtIrg23KtS7/2m8AwY5Qgd5qX860naz2j", - "O3guw6Bf7b1UdxHbcHmL43nWTndXOLfp0gHnnxfOuBJVdEt1gj4v2bETnNeLY3e9PKJJzy+bPHnb31o7", - "6856V0TvCzyKv0jVa6beRViqOrn6JZYq08eQSK103YEKNaxSqN5mBbehAODaCbH2B0hWNHyXDl3mQo+z", - "AnTQhCuUBwGslHJg+SJlXnworz+vMgzZHWoRYnsFQjRbUxbIR5eR7skxSgQP0yD3gI1g0WkQECnHKRRN", - "7jWVaFe/Md6mMg+u5VqjX63M12nvq8NMyXX9eb8i16owpUbY+qPe6q8+6luxAHRaaRKu5mGmUTMOtlYm", - "jhU+lR57RBnsFSmosJkPDTj6myIEFxU8qLeEAi0OpIkrAKhxahGTpKfsH74eenMEHJOIKOIbBJmKnNbt", - "g8qci65mqVt7+36TGb4eBhCQuLCQXwlJtJwec2krZMaYzb0LqybBRu2+KwApEQzfNYm4LLTKi3u8Ugqp", - "Parm+cQrFl0T8VVM355l6Pi6ycRtz5WlHm5PUHlrVaW6jDJVX8+i3fGw+w9jZ0TD3sHmz3/7f7sf/voX", - "f2mVkiIlieiGZAwPYJdk3jU1Y7XS1iunI4VsNy2psC1IogiOwYoQXBJjtYjxdXG9j/oZJc1f4XhhC/By", - "GFOW/Xvlhv72l/p3twIY3wHzWHmOt5HYVXHHYtsxEROXPd15e0ExbRbNNaQlKmSPs9e0o8qfZNalWLb1", - "wog2PSg2PKKQhFMOmNZScBCQRJGwZ7NoUViL4EBR1eLFNoud887WtIYhdaeNlqkkqfrkLRF80GLkqmtm", - "CLsadXYf7dl67UVIbi2ckO/MTMB1XX1DDWWPFeAllRBN4JxmC41Rm8SJmrscrc6tcWO9APDDbEDvS+ZX", - "zn7Vf/I1cnW+W5qc8zusrlmMz3cLWhmZv3D+tRnx/H5Rx9VEO4YmbcWwcmKYisYjVbfebSrWV/QQvP8W", - "XZz0N+NZaLNRTtJqWu7NmKlNm/vWF80QQkHgpb6kOZW5YPUudFrtIrlUSCzsrLCS+rM5dbJQtUhzPYDO", - "NGiupkSQwkFAhzyB55ogs35+DWJkTIbKhIhutaScqXogKDgOZmqsA0HmC7po11qeoOYUX2czgE0Uy4WX", - "A9hHnqpt68VTqGTyxpUWo2M3BCyjIqn6s82UsahJcfTFwyhi1eK+TXsv4VletYT71dFWBTnzOUqo6cPH", - "3zBVz7kA2bY+IuXWk9aA3BwSASG51ZQ0jfK50JiEQ56q5fRvU7zbcJQQjciYC1LIfuvkeAxIbCsOr+AF", - "LmYiX8MHn9wgSZAKquZa8bMS5YhgQcRhaggeAAkTwc/5xJBM9vNnsICNPQ5oLwgjggbo8OwE6BGq5mvi", - "eH+KIjomwTzQ+jPkAl1InwFC3uujE6s7uYRt8B5IFaCeKwJ8eHYCNUWF0R9a/d52rw/EnBCGE9o6aO30", - "tqDCqkY42OIm5J6HP61vuQkro5ydhFYOemqa6F4Cx0QRIVsHv3t8tBURJpe9BKkTTwpif4KpsHJ/EoHn", - "uEEVqvtC9iJ3lR6Y+7hjAN7Y9CPV3PrRkeS1PdYPGhMM1cAWt/t9o2YxZS9enNea3PzDxtvl8zaS5wA8", - "nlQ+C3K9kyktyD93Wrv9rbXWs7I8pG/adwynasoF/UhgmY/WBMKNJj1hxrkXmTQR1n2hSGeAQkUK+/2D", - "Pi+ZxjEWcweuHFYJl3XCMJEIQ4E6U0jhDz7qIWvYhuylcsrTSHMTZDyXSagvLKx5Sm/yEWERTOmMDJi9", - "p02pTywgu3OM9P1s1JYyaZipzelnMWVPeTivQDcbblMP13XmzBzA1SyJkgwh2dOwrkpKbsikjEG1RUls", - "SsmsXMCijwSUx5UB99YFJgwzlVdbNXVxL8nc2kq9AzbKyqIZHhwLgTLsWbbw7Q1/OAIkv/RH8hxn35AF", - "b1mcYPDAEERpmMtczkMWixGOIm/Y/iTiIxzZ8sGXxCOivoAWFijFPKFOuGE8JCbnYzJXU87M3+koZSo1", - "f48Ev5JEaBHI5n62sLa1My3qQh13GkP+ZVNZQs+5aZa4+emSzD/3BuwwjF3VEGlLv0eS27rKJvsNlci5", - "VBrc9WcnrXmtP0ql4rFFKVYsA2mWyVOVpMq+VEqibMJqaA5VQuWUhAOmOPokTFH4+efNT/mMn0F3ITjU", - "eFJoYra0+YmGn+tWLYdY734ITT3aHwEADFr6dhm09N8TgbXuksopmDIkmC8mxSNtZ6HUWi7cqEI4wAwl", - "PDFh6IBUplx0aQxI/o+jCCkgJddXS5twkjX7sZElvkp2NqzExAFUyAhq2hWIqb+776cnSQJBfAaO/zl/", - "/QrBVaXPwDTLzUYAI8r0LYrCFCR5mL03YM9wMEVGboJUY4MWDQetTLsIN2CtqbR+r90uiLg/66X9bKbp", - "0PDnXk8PZaTnA/T7JzPKgaalJB4qfknYoPW5gwofJlRN01H27YMfoHXe+eclRoDahvdvuNItkCUgvwbN", - "vYFZiLjltdEcYZRzoKIdZUQZFkvrznhAbyGoVXk8kUVgfBqAAXTQOhg4E+ig1Rm0CJvBb9ZOOmh99kPA", - "CtH1ea1M6R0na2dItNfvb6wOm7Pw9YjQpYaa/D4vSF/bX03wsELXouBhNueS8ukTNEWUjLh1B5LPUxy6", - "tPw/RLwVIp61XBSEN+hfvAcM+kbEKLgVCUzrs5GTwJZqJwYtICslaBwuyNUoHNRJcDnyFtWPqjq/qFbs", - "1lFZAEuMHP7t3gH+wbx5IXKY98ldzYsjSDGZleV9WOgIh+UQsePXiF8Q9S1gXP+uWKnNh3mf+PtQ8OcF", - "sXJfDrQKN9skM/fe5A/lBxd+aUcxjbWueg5r6p4TptAz+LVn/+s0HkhMexHxycUBMiCM+ARFlNnXuMJr", - "kb4ULSyhk/Hiz/pZp36XR6lt7s9///NfsCjKJv/+57+0NG3+AnLfNMktIO/qxZRgoUYEq4sD9CshSRdH", - "dEbcZiAzIpkRMUc7fRAzEwGfPKUe5YAN2BuiUsEKr5YmpZG0A4LqwWA/lKVE2igI3ZCObb4FY2D2qPCO", - "lg0o75SiO4suo2YHhQ3oW9HhAATQUpN81upfLb/1zOy5ZD+r2soXLKar+Ysi18pgb9cscE0GAyD20R18", - "sJtG7fPzZxs9BDqGwQrIqQEScz6MFZ57P3jSap5kOEqZoQCUDW8qVPmutf8e2zbNDMB2xO/JAlxXtrze", - "BGxMHkSQ0MHrh67QxBzsh5szDfvss8cu9q3eQHvz/RancM5AjRThr3fODvcWYW6+FEB2Hyowajs/a1dx", - "7+zoxJV22bg3pL+TW0Pv1BZEyK4OxE2dvztTy444G0c0UKjr1gI5+WOSqWplBHko7OCNXTXCbl/V7HXF", - "+22zlIyl9qbL8rLkV97t3x6VSde5RvIMezmu/bhJVqHOMZUB130L2NINcGLrDRrxJaPTIhatMkgZt+3s", - "ylkqLln2fHLsCPLuTFN26pRV74Y7YIrHFYZ4j4ywUkOtkJPyIWHzu+wUXZz/EsvVt4Wa/buTgu7aiuVD", - "84dkxgorYNNc0GTmrb1AXxD1i2lxiwdtZ/Bs/JwIR9UuhTDsOtuW6YqCKQkuzYbgQXq57ntimjRTfc14", - "35PmC+BZR2KxIP8hojRQdnNYLVNwT2xduNvTb2GGtdTbr/fOaxHMA2RwNhk5i7UpuYblnAUb39VT753c", - "ZgbYD/IyO0ujyL14zIhQKCu/XLwDNj+BW9Jq2d5R29Lr4N2bl13CAg5+aJkPlV+Isl++soRvDsxs5Qea", - "NNEJTcAtdfdZnYTzBedv3AVRVt77v7af2wLf/7X93JT4/q+dQ1Pke+PWkKV/V6z5riXuB4x8WuCmZaAB", - "a2JQ6nOVhJq1aiikuvbflZxqNr2WpJrB9Yew2kRYLYJrqbxqj+JWJVYzxz09yWTI5oM2fHL+id+ZpHq3", - "Vj6LkS79LpXlZw9bX4ULsPPCJ8pQKskDdKCkGcYVr42G5uqcIJdeHw51T447AMiOBh3kA7IBIndkvHbr", - "uHPh1s5795brw3hEJylPZTH2JMYqmBJpg5UiUmbAD03szq/nWsH7G8bS/l1eHXcuV//A+1uS+KsHapi3", - "eYFaJfO7Vk1lfttey/y29r2JXXvjaurbNEcbNU6FLoi6KRqXYs0XnR196/LpIuidVlRydQGBBnEwYP+t", - "9Y/fFcHxh59dkEza72/vwe+EzT787OJk2KlDFcKUoDb35uGrY3j2m0D0OaTnzEPyqusw+fwB9VwCm/84", - "BSl/+WyuITks/KEhNdKQCuBariHZs7hdFamcw+rOdSSHbz6A2yQmP7Sku9CSZDoe04ASpvJyVwtOYrZa", - "3gOMLWP2fajg3FG6aBtrSRlRrhBA82Trd+7Yk01+98qRy+v+MH3kuYmKCZ06kl+G9frIt4YP/btlznev", - "hzxkFDMC/yLoEi1T+qooQqbHOFXglJhnCAGvTySM1J6N2EN58UKZJgkXSppskSAAQzUrNdUCsC+zZDlZ", - "pC87JOS1pUR2Bgzyv+vPJpZ/85LMTS5IyvOi/NlObf5HX+xVOZXmvZLR15ex/HlCG8lYd0zGNh3y/clY", - "98Y67kTSOillmW9nhAEK5YhklMyz4D76kbLJxoPyQDXMKttbIZ+RR9TaHNt6gn4V6DkXl02Zgqe+zQPg", - "DcUdfoPal14e5E+6fyUM1BNDPxpp7pxvLBQtuk+ndVrlJEGUhpp1OBbiLt+x4PHQ/mgyfGqqsPkTQakL", - "7Kj3zWT07HegYr/iCtE4iYiWe0iIugab9GlaYcmlyaayUOJrPSaoyaYYQmDSd0lXJsQWQoPnCHdgbXiZ", - "XDwuL9eM+GR12oBschcj78kbMGAmjTdxOb8vUMZkkeJIkogECl1NaTCFHAL6N1NZEML7cZJcZEmDNg7Q", - "C6DUYu4kmLwtidCiY8CZ5BExqQFmcXxxsJjj8v3pKXQy6QNMNsuLA+TyWmYXhNStijkBsvojr2ymg7bG", - "JMGjyJzohZazC/vbsNkC8qROA+bLHMDIlR2QjtFFIYnARU0WAcdQX/KJvC9RtlOfis/sRXEkAHAGNwkL", - "W3Wmaxr58wds9b21GRrmMjDLuOVUBguLecknWRrAEirjJGmKvnaZgMWzOF6Cw6hdKBIoVchT9TepQiIE", - "dLbYXYfcqI0D8w+FLzWi2iIfWZlFQD/vA43Jy+UFlWaqhYoW5l+zOG51WnY9hXxea1gYVuSEqA64+JCg", - "T6aQ+OGHLWGdlA5lZl/I6VC5OWwZ6nqR21bX/u4tWhZQ4fegl5ZfAPJVUOZEFThbnlfEeVCx4abwelUW", - "M+V7fDTidtmVhUJ+zR4EFkoAfgNK66p3gqyeW1Zs7q4fDBZX8JDDBuTCbqBQPFvrJeGbR6SvdyQLW22C", - "IT9wc/0nh0aImaRL6vpBWUKJsKt1B5lwgynnsoD2IzLFM8qFzVltayZlmAkmC6M9Wn+jC42qF7Ys2oUV", - "zw+srQnh4ic7Rw+6Wy8lfw/3Ke/xvKBtZxy/40RqyJsnEUYjQckYJTiVREtLaUyQqclgUx8THExd1dre", - "gL2dEmRr1RUMCFlpUyrRxVZ80UGjVKEIiwloO+aj8T0SJOBxTFho6k8O2JTgGdWqmkARVoQF864kUI90", - "RvKSD1p1t286CJ50soqHHeQKZYKB4aJQBvMCJYIAEhl1mZVqTg6YSNnfTa4/PeyFW+gFIlLhUUTlNMuu", - "H+CQsMCbSO/822ZjX9+Ie07UYqXIe3nluREvvc9nn6ItM6vV+028CD0w1xYuXEXABmx+idAr61XDsq/Y", - "eV4d8z+QpM1e3R7v6WUmA/EyKv42nmRK5bF/PMsoS5JhaqYj5RLS3+1bS8ZQUMpKzy3WJnvTB5csd3wG", - "5rV43uYn9+fJDWxk3wgn7Cwp1O6fIN/0t8ByLVRvxHPvyThobUkFq9g9smC7qPsTn7gocLlvgg0bgsu4", - "cZHnKIFBpzJF7X8w49Lbt3EPuCkzdhbXhQfwAnumrJtEuI4vW+NsLQOulFD/D/MXrCkQ/9lywvtkfPmL", - "wJ0xu5OMvRmGl+B5xPH3/i4TcCFMCJwt4PpwUjAVbIGFB6Y2WNw6GYfoOP/796enG3VcQqilPEKoB8wh", - "yoUgg9hT3+71jAhBQ1ds7+j02JbmoxKJlPXQ65hCBbxLQhIorUF5Kk1Z/l6xQv1i2bBKCXrClJgnnDK1", - "chV509tZzOcbFRu7Yz5pk9B994/HYIV/eEwKeIcWV+wGlmuRCqtaZzznnEaZqRCopS084qkefaGCPhrT", - "iMi5VCQ2nnnjNAIigjSltoqN7Wdi8DqIKok0PXQgZikhIqZSUs7kgNmC2QkRem7dHcql5k5GXuO9whnX", - "PDOs79twYIOi+uCzhVUd1Mr19HGSuHr6Picpu7wvWNJz8EhDch6PeEQDFFF2KVE7opdG6UAziSL9x8ZS", - "l7Yh9PvaNXpuTlka0idszL1lDAzOZsj8fYRtlNmae0R8cGztBSkSi+M/cNB+tiZX8jVBcNRVNCZZuDBK", - "FY3oR8Pq9CBUKhqY2s95sBqUrbXxagN2SpTQbbAgKOBRRALljCubieDB5iDt93eChEJehx0CiwOGV/85", - "hhmPzt5BO1NatzNg+h8w8NvDM/MSO8bWRlBYKCPqiotLdLL5eoWT7zmA6T/YS85scGnUmPfAfzzfrR8L", - "WktDsoZEebJMAeLJd+/GaSW4H9aCh2ktgGD8bDfticABCMVymqqQXzG/ZWDGozTW/zB/nKxK6aBwMH0P", - "Tb8ZadcsZ+U0boMPgijtnkJiyqzcywOFAdhD9S/VgHNbACGm5LnnvQUO1feI3V/fKF+E4zf4NGkh6koY", - "fTO0ddc3n12Dy1RUhMdDIXODaW4nii+3Pl1hWm99ehrx4FKilCkagcGkIGliyJyof8wz3dmHPxATIDrS", - "FV9G5DqhAnJ+gE23MBLRO5YII0VETBmONmHPZhDI2eesWHjGKQQpBxGFMDEaEpTwKIK8JFdTwpDeDRiq", - "3ACFd1ppc+YX2xSfGBVHIxLwmLg8hhs+1e03TNVzLspJCb8Vvvi2AH+9H71Vvc8VeRjrZ/yivIyn+Brc", - "msPUPhO7FbVf8PxHYwrqIDibQWunLwetDhq0tuNBS5/AEQYTKlboEYopSxWRPXRs7FsQhrrXR5IEnIXS", - "pVN0FrydvqwLSjVoWRPhuAf97lLssVgFoHxjJ/GxB90O6f4QYIPaRYKzNBl2gOhCxFMFDtyOrmyrkCgw", - "j2zc+QtsgUZ+6PZNOPlvlnxLPApOWbPLwtEbzp4l3FtpdXNBFVMu8zx9KMAJDqiadxCOIh7k1oNUZq8D", - "3WwpI0HwpdahegP2Jkv1ZwMh0NHZu44zmqGQykszgrWL9dDrGREyHWWLQ8ANjAUPDoOEA6Y4CnAUpJHG", - "WzIekwBiGCIaUyVr7GrZUm6zcFw+iefg3UcLuodmTPLjBJxejhaygnGb5qg3BQkiTOOiUakKHBB94UkX", - "zL4jPSjX1/A4ss9bgeBSIjtUl0R0QkeRfayRPfRWixw4JgOWRJgxIlAqjd+RXno3EUTK1ATG6AGgMqfB", - "qA7KE50kgitrJo44F9JYdjWGvz9FUpFkCZq9MSOfwp5vKbGqGdzOdE8KQ2UN9deSbYL0gRhMMQDXeKSv", - "6Xtw9jELuu8ErA+F8N8KOpkQoakCGyZrnkYNWTtwGqIvRXrUZhU/z1o1yyqejVrw5i54Oi9NVDF0DYcg", - "QK/zAuuZ/JLW5jKxn9aLvvhVd2o4d9nL378I++kLd/m9FGs6LzhXN81FnmP4Q0sLXlh5iVRLAQqr0xE0", - "jki4zQiBxnkH7i3dwEPOMoBLYQd16QS+PUTo32103F0nJn7YuFXKElAqRVITKrU6fec3gYG3k7fznqND", - "b5C385uKV4K8i/cXN/pNRSqV7ICu3MJ3n5nztgKUTHpOSGNRF6BkuJ51JFiqKL23bZqpSXbE70mCt2/P", - "a8jvDuw/tP4GKkMBWH6TnYmNdnlbSJyouXtc5OPKA6CkHyEYw5f4IfMhuL18Czd4Xv966OHwtPZx/UcF", - "ojt7v8/LtJ4cP/yyQ0WaK10sm/rW6WIRTOmM1BvdyxRsQZQI0k14Ao8roQGYhYe7yxQWvclHZIe3uars", - "vxB1KY5JiEIqSKCiOaJMceAIZo6fJBJcawLwnYu5z5hepNzngseHdjcr7kNLU9YYlr/5xvNuiBXuzhy3", - "WWJC+4KXdve2rRkeogy9eIra5FoJk3EXjbXmg+g4Aym5DggJJeDkRnHBW/0ayyb9SIaTUZNVLsmd/Nrm", - "pkZBKhWP3dmfHKM2ThXvTgjTZ6FF/TFIsongMxqa0o05UGc8MlDdqgHounZXLVTY8L5cuTCLuxcZpsmF", - "NPlIkzJbMK4LrYPWiDIMi1uZpbhMUyagSs+HKYQ15LTjMKf14wqzml/bKTsaE7WS44CoODep8TZ+XHMP", - "+ZorOqa6O6102zUrrtfMV7WhC+ltJMzN/Jjv1mz9/ttxr6TyQXpWWtP5LFNI68zm3xYK9u/ufrhrc/n7", - "B+yO/4I45btgKocB9Ig+hHnJAxyhkMxIxBOou2fatjqtVEStg9ZUqeRgczPS7aZcqoP9/n6/9fnD5/8b", - "AAD//4ZlynVGYgEA", + "H4sIAAAAAAAC/+y9+3IbOZI3+ioIntloaoakqItlWRMde2TJdmvbsnUs2312mv4osAok0aoCqgEUJdrh", + "f+cB5hHnSb5AAqgbUWRRtiRr7I2NaZlVhUsikchMZP7yUyvgccIZYUq2Dj61ZDAlMYY/D5XCwfQ9j9KY", + "vCF/pkQq/XMieEKEogReinnK1DDBaqr/FRIZCJooylnroHWG1RRdTYkgaAatIDnlaRSiEUHwHQlbnRa5", + "xnESkdZBazNmajPECrc6LTVP9E9SCcomrc+dliA45Cyam27GOI1U62CMI0k6lW5PddMIS6Q/6cI3WXsj", + "ziOCWesztPhnSgUJWwe/F6fxIXuZj/4ggdKdH84wjfAoIsdkRgOySIYgFYIwNQwFnRGxSIoj8zyaoxFP", + "WYjMe6jN0ihCdIwYZ2SjRAw2oyHVlNCv6K5bB0qkxEOZEMY0pKFnBY5OkHmMTo5Re0quy51sPx7tt+qb", + "ZDgmi43+ksaYdTVx9bBc+/Buse2Xu76WKY/jdDgRPE0WWz55fXr6DsFDxNJ4RESxxf3trD3KFJkQoRtM", + "AjrEYSiIlP75u4fFsfX7/f4B3j7o93t93yhnhIVc1JLUPPaTdKsfkiVNNiKpbX+BpK/enxyfHKIjLhIu", + "MHy70FOFsYvkKc6ryDblVfHx/9OURuEi14/0z0QMKZMKsxoePLEPNbn4GKkpQfY79P4UtcdcoJCM0smE", + "sslGE37XAisiioRDrBa7g6Ei+w7lDCkaE6lwnLQ6rTEXsf6oFWJFuvpJow4FwSu602806mxxq6VmJYex", + "rGvdvYIoQzGNIipJwFkoi31QpvZ26ydT2DBECO6RUM/0zygmUuIJQW0tNrXsZkgqrFKJqERjTCMSNloj", + "HyOYyfzBR4iGhCk6puX9bdipi0fB1vaOV3bEeEKGIZ3Yk6jc/DH8rllMt6MQvO2fiN5o82bzgC4FGS/2", + "9xxEN3QiyJgIonn8C7tLBJ8RpneL7u8v0G/r/9nMj+hNez5vAjHP8tc/d1p/piQlw4RLaka4ILnsE81G", + "QGoEX/jHDI+WrXWBo6TCYvn+gDe+wk4042tEm3Pz6udOS+HJyk/e6neqshNEo+2yJAVqReSzGWEeJSng", + "TNkHZeq85BMUUUaQfcOuhZaJuoOfIw4i8SvRISP/4ubX476B8DI/1LSmn3VahKWxJmbEJ0VqTgkWakRK", + "xKw5wmxD+ehqyX9W2j6VswpLMlwuQc4oYyRE+k27sc2bKJWgqS5MH3bRJVXDGRHSu+dgWL9ShewbtU1F", + "PLgc04gMp1hOzYhxGMJ+xdFZaSYeba2k/uJEC0HXIGgREimOzn853H60h2wHHhpKnorAjGBxJoWvdfPm", + "XaSwGOEo8vJGPbutf0YvcoifA86zjVF39mQc6BjTSLqWXU3dfKeVpHJq/gLZrUcFZ58WA5q9Iv33B8+k", + "j0BIGCuh1mby64CvE7PYaBJxTdM5Shn9My0p2D10om0FhfRBQUMSdhCGB1pk41Tx7oQwIrScQmPBY9C2", + "CkowapPepNdBA60XdrUW3MXb3X6/2x+0ympstNudJKkmBVaKCD3A//M77n487P6j333yIf9z2Ot++Ntf", + "fAzQVDN3WqGdZ9vt/Q5ygy2q69WBrlLlbyz9i8P3SRyz1CdaTqy70kcni4qDmWvIg0siepRvRnQksJhv", + "sgll1wcRVkSq8syXv/tVaQHzWEIENtFkWpMMFaMH2Lgd8SsiAi2BI6IZT3a0EKZKdhDWdjMIL6RPyb+j", + "ADO9F4xywQUiLERXVE0RhvfK1IrnXZzQLjVDbXVaMb5+SdhETVsHezsLfK6ZvG3/6H74q/tp47+9rC7S", + "iHiY/A1PFWUTBI/NqT6lEuVjoIrEK1fEUTeNQM2LKTsxn21lI8FC4PmXr7CbyLKVNsZc7VIHsUfzfz0j", + "QtDQnapHp8eoHdFLYtkdiZShQdrv7wTwAvxJ7C8Bj2PMQvPbRg+9jqnSp1maH9LGG9QrLvfvLRJMOegZ", + "UcT1hDJS1ygxOQ0DQcA+wdHSY3gZib3EOsraXTy0f+FSdWPM8ISANWlfRCPBL4keKEp4RANKJLokc62k", + "zNFEN9qdUUn19iFshmbYOA16A/Z2yiUxr7hH2hAJCJ0RFPPgEiURDsiUgyE+w1FKZAddTbXGoIWxIDiy", + "PyNBYkzZgE31IGXAExJqG8K8BlNDF4TNLlCME9ilWBDYoijGigiKI/qRhIibT2ISUn1ADRgBvkYJ1ls2", + "CLjQp69eW4KDaYEKP0l0YfSNC2j+gjLNlRdmX/UGrLjyn1qv3719+vrdq+Ph67Nnrw5Phr8++1/9s/mo", + "dfD7p5bxb2aKxlOCBRHoL59gvp+NdhoS0TpoHaZqygX9aJwtnzstTQOp+QsntMcTwjDtBTxudVp/Lf7z", + "w+cPTp/SXRE209vAM7DPXl3GHIUeiXLsnHkSWQcRqHYYXLUgYV6cvdvUh2uCpVRTwdPJtLwx7Mm+1pYI", + "qbwcUj4cJb4xUXmJTjZfI613oIjqDZrpGVv9/unTTTlo6X88cv/Y6KFjs2th+FqEcGHVHznV7KOVcGCZ", + "o7N3CEcRD6wLZKxtpTGdpIKEvYrnDVr3yWfClJgnnPpssIpwyl9dlFHdbv50DVG0OaJsU+pl6Abr0R34", + "5saWwDM2o4KzWFtjMyyoPmZlea+8en38bPjs1fvWgZbjYRpYp+LZ6zdvWwetnX6/3/IxqOagFTLwxdm7", + "I1gps21UEqWToaQfPZrAYTY/FJOYC2MB229Qe1pWFMy+RbA4g9bOi6eGubZeAF+5RQmphLddK6bhMsds", + "v3jq45bpPCFiRqXPTfZL9sytfOFYN+K+zNuSiBkRGdMCF/cK5kcQ8TTsFrrstMZUkEBgzXatTutPEms9", + "fPZRs04+ds93fu9VI/1zhWKJo4QyskSz/EY0vCsuLiOOw+7WV1bwGFG67cUpvjIPyutreYJkLNHqLHgj", + "WHhFQzUdhvyK6SF75Kp9grKXM+F6rWeCo3//81/vT3MzaevFKLGSdmv70RdK2ops1U17XSDZRNLEP413", + "iX8S70///c9/uZnc7ySMInIjpc6u/zPTAohszeth6ZrSeDPLZPltStSUiMLp7ZhF/2TsYfgcOd4rTKXk", + "Hi3eaS4Iaj4jIsLzguC1Y2pt9UH6VUYlqIK9ar/TYvQS6Y9XiGHdmjvkX1Rt9O2+X9B6BuUZ01MtK+y5", + "0GQk2UC2tk/tn9uLQ6oZ0SVNhqA1D/Ekc9kuu20+v6SJVcXhC7OMUWQEQZiC8j7iXPUG7LcpYQjWDhaY", + "XJMAZJ5UWKHDsxOJrmgUgYMHhMri0aIV+1ysmNel0v8rUtZBo1RpbZ0rgqzdBJ2kMBZ4eURQyrC7zq7o", + "znaCVb6yZLkkgpFoaHRj2ZAy5iNkP6olDkx1jKUiwkj7NCnT6/jX03PUPp4zHNMA/WpaPeVhGhF0niZa", + "HmyUqdcZsESQmTYh2AScjdT2y8eIp6rLx10lCHFDjKGxzEVm71pnL87e2dt6udEbsDdEE5awkIQwZnfi", + "SKSmWKGQs5/0jiVhudli/xWi+/dypyUZTuSUqyFYfvNV0uncvn5m3l7LF9BpzYIkLS/pdnU5X8GFvCbe", + "jAqV4kjL2pI66b2fN5EfHrPBBJYUzRcr9zLuxqp8sdrUYWJahjCQRaXa7/cwmlJjv0fBlF/wgDg781Oz", + "wa5o/4S5gSz1++Sm5hf0dW4aqZLItt1xM7sBlU4ympRphb8OeQ5lwTSv9auHRCrKDDvpd5HVCCVqX2hr", + "3vKxtt8vOujir6Uf9N53poXWL66QoQbIE6Z/KrZfdUqsdBc0Nwori4PlzdfjUNYGKqHZFlICM6nPVq1j", + "JaSHfgEhjhSJEy3J2ARRiaQRviREjF/9HXGj1LhPB0wPTZowD0uOzGkk6YRRNtnQar4+mHAYGs/SOFWp", + "0O/NqMypWWYd572pTuCtGR0x8jhOpT6RgygNCbpwHp6Lsl646P9ZNAmtQ2jBwjEkAcsGjD21GadKd68n", + "HGMVTDWdeKpM3JeduiwPoOxlWnUfaseS3ZTdYP3PM3FRJqr1N1QEv56cvaMBt2DBP1nnBrSKit9FeUnm", + "sOTOHYkXHJJFT6TfXyiI5NGM2GO36Msc4eDSHCUm9MK6MY1D0vog9favbFGvd27VUmh6NSZ/2VRYZCVw", + "AdvJ5hxjtX/j/51nUkhPzvTX0YaxJEB8MD0OEKhjFx1jKxHwQCCmmSVCIRUkUAvNUzYZMAgBubC/9Gxr", + "F3qTax3Ftwl9xo5XFyxYO+ab0tKiwso6tQ+a0VPjMVWKhJ2ybnBJSCJXT0qr19Zx7fGuC3IlqBNk1mEU", + "NlTPCBtzEZDYGglfZjg+KzTmNePWa2IxIsPQtzBmy08IJ0lESWjCf8x6gJtV2nUCH2s15DesWG0mAqDc", + "5QWOogvUti9tIEH0XKRbK8ZZzuxvj84cC2S31u9PO5ojtRS4mCqVDPX/yKHexRfVxuy3bofr5vSZJNF+", + "H+yr3d0du6rW6WYGXGm27F/zRjXUL41Tv+svxnis96ILE2miyh/ln+Se1EvKwqYN/KrfrfXOZYqRszRu", + "20GXCNJNk4nAECH7Nd1zN772BGrWS/AVwe++KMeMqkEqFY8LsY6oXYnQoOVYjjKxZjzqhlhhcGU29Lea", + "4S7GDcdz05Sxxeo8McPJyBP2Qz9qqYsmdIJHc1W+P9jq+yy+L72DdmPxLUtd/L2xIEk4VHx5BDIdI/du", + "k4BDOE+Gig9nY+ppOTvW8vAVKlFQCfa3dq1uopsE1LoTQMcJpiZA1BABlMb3p8W7u96AdeH4PUDHWQdZ", + "s1mTGHRLHJqbkzYXhUFQiDpDo/kGwuj9aQ+9zUb7k0TaYJkRl5AwxRKNCGEoBdcznIZdcxYXB5BKODRV", + "9XPrOzG5CxtwRcntsx76ZZ6QGFs/lN4KMVY0gEinEa3MB44js1D2ThizoheskddqWdz2GzKhUolK1DZq", + "v3l+tLOz86Tqv9x+1O1vdbcevd3qH/T1//+jeYD310/P8LV1WJYtNnasKH2O3p0cb1tnabkf9XEXP9m/", + "vsbqyR69kk8+xiMx+WMH30kCh1+UHedBb6idSiK6TkxqrvKFuhUiympC2W4coXZLAWd5/Oyydw0l3uo3", + "byMzxRfzbCNu188dqQrMlVHThcktWvLzBOzOfJcUNDgbnBhQbxjmMZWXTwXBlyG/Yp5zO8YTIofmPPPH", + "M6TSBNmQa+vdEJyrsTT3pmWv59bu4939nb3d/X7fk5CxyPA8oMNAn0CNBvD66ARFeE4Egm9QGy68QjSK", + "+KjM6I929vYf959sbTcdh7niaUaHzPByX6G2pcjfXHKfe1Ia1Pb2472dnZ3+3t72bqNRWX9xo0E533JJ", + "JXm883h3a397txEVfAr9M5cgU1XgQ1/ografzGVjVyYkoGMaIEixQfoD1I7hCCPZbVV5T45wOLTOE//Z", + "oTCN5NKICdOZfdM42uI0UjSJiHkGC9LIFw0zP4aWfNEolDEihln+0Bot2bSilRECbi7ZK6iUHlYi3SmV", + "oIXkyhMlUXhgduhKOQermQ/sQx0f2Dk05IaX2nTqRmRGoiITmKNLDzbmgqCMT8yilWZF2QxHNBxSlqRe", + "lqgl5fNUgC5qGkV4xFNlrhlhwYqdQNAy2B5jLa6b2bnPubhcGf6pT+KhSBnTzaz0Ch2CI31sXTVwimNk", + "v3YZBgWlL7sONJem9rlEb8wXxkOU/5ykClGmuLZOWTiad6An60liSBCpOEhS6zC0zTTVLv16CzhLXfiH", + "6S+XnXcU+9Idm3CBr2thiwlRQ6mwWqmxaE55C++fw+uNo8n1hysdKQ3ozsjVXRAdwu27mm27kuHkdii+", + "LBgt8zXkL8EpLGhIegh2F0TFuPS+yk47VzxJSJj5f3oDdm62SvaTNDco+kNDBzUlVCAu6ISWOy472G4z", + "qm0dVnTcdGN2LH64qKHCQwjfqN/0eKyIMBR0mcvF9CO7CK1Oy9K+1WlZSVQmjfvRQ5E81HJhiC/O3q0b", + "m5YIPqaRZ7oQC2GfWsvMRW293O2fd7f+PxOBqfkNVDTKTPxEzEPSq4ADwPvNTp4XZ+/O6saUITOg4ugW", + "5pRFvHgkRxbX4ChiL5XsraS1YBz764Ml6yTXvZ/4dNmxwDEZpeMxEcPY41x7rp8j84IJbaIMnT4t67Na", + "b25qNZ+VFgfM5jEObGJ9M+p7HHKVaXQK1PzgX643xBzDdel4eqmEfcdm5PXQqwwLA704eydRHqXk8dSV", + "l7c2Xv5sOpc0wJFp0WTXUlZ0sAFzNtaQz/IPrSvSoyfHXt3QbQTUnk2SFLbh+Zvuyev3m3FIZp3SmCCy", + "aMojose9UZAWM5eUlwf3l4TErM7TYRhDNt1ABVplO7gxkQr71UMdxRWOhjLivmCNt/ohgoeo/f65SZrS", + "I+igpLSU+vcCFUr8vefdMVoi1XV7Dh1WXaalDe61HcsQMsa9UpheqVPfVvmF4Mgg55T5Oc/vdgvPL8sL", + "zS9X7l7biK/fExcY3iB56+j02CgMAWcKU0YEionCFqenEOIC6lCr0+rqMyrEJIZQu/Hfl0e31Ljgi9lY", + "tU7cowXYjVtx4Naki78xIQghijGjYyKVTRcv9SynePvR3oEBtQjJePfRXq/XWzdH5VmelNJoKTZNCH8h", + "XaUnp1+2DreQitJkLp9aZ4dvf2kdtDZTKTYjHuBoU44oOyj8O/tn/gD+MP8cUeZNYWmEg0LHC/gn5StN", + "fWaZ3w/0TJgNCdO8xMGAX3nFVGPPQGQD5M1504UVnmj7xHDcl+YF3xg5JIevUgXEkGJAaAP0EPpxuSfU", + "KUbwju0zZYpGObDKog/0RtA4cil6wAJyQEJYhhcQReavgLOZ3hU+8ICSAHfPvuj+wEa5DEPq4eTfrLVn", + "giQgq2r1fmtt4iRZzbZ+RTGTf01BU2xqs+ckunepf5M7tnLvryf/8+f/L88e/7H158v37/939uJ/jl/R", + "/30fnb3+ogyq5Vnt95qa/tWy0eFiqZSS3pSVTrEKPArVlEtVQ2H7BClu4jV76AgMv4MB66KXVBGBowM0", + "aFVChAct1CbXOFDmK8QZ0k3ZTIcN/fGZcf/ojz852/JztY3QpjQIuyBZJpNMRyGPMWUbAzZgti3kJiLh", + "Tl//FaIAJyoVRK+e1mGjORoJHOSpDHnnHfQJJ8nnjQEDC5dcK6FnkGChMhgO1wMwhR2ViRmwr5PQJYYb", + "C3nAsnMpyws3Pppe5gQB33w14tJPFK/5wkU5FWe/78ugh6gvvZARlYpAYHbG2ZqNsnA0tN8viYr9/n5/", + "pYKf8dAS9oOdsAiS6ZiywV4yDAxdG8ENEWoNfOlaNpk9gn55+/ZMk0H/9xy5hnJaZEtsjDwTAyiNj1BF", + "shD9t9Hyub7N6jackHGSwWdRg6yhZyY89O3Lc6SIiF3AfjvQ5BzTQM8Prv+plKlmRYrR4dHps41eA5RP", + "oG02/iXr+DabYTW5wzrN6nyBGcdr+nbQyTGE59odmitwEFbznAsUGQGT7+sD9E6ScqwrLJW51TcrGc1z", + "z5s5AQatDddiUpUUB+hNpjfibChZoGXODK7JfF9Cs/bixcT8LLReicuFaCZrF1nRBhE+WGVB4vrErRcF", + "y7e/h+Kw521cd8Gnud7eLjpDdWd+1sjX/ta1lZ11bdR1ARrKOZSF/NsMo6E5uMJtgBQs2mvXVA1rL+GR", + "fmyv3J1V8v4UTbFkPyl4WLFNtnYeN0LL1L02vb4uXlzzsRlStqtcQmZ27WpSUy9pFJloBkknDEfoCWqf", + "n7z49eTlyw3URa9fn1aXYtkXvvVpgNXgWPvF2TvIdsFy6G6A6oMecR44TK6pVHIxX7XRRepybIhfSvgN", + "3gTgja8I6uBunxemcRdwDfcZ1vftQUUsBXf4UoQGq+zeEkBDrXD1gRuU5az5+etCLdzKcEq5Pz75UNQJ", + "XMz1jbENOi3qiTc9lFoEkhCdnOUQh7lTyjVfmdOT7d7W3n5vq9/vbfWbuOhiHCzp+/TwqHnn/W3jiDjA", + "o4MgPCDjL3ARWsY2yhuOrvBcooFTrwcto88XFPnCtrUqeKPr10UIiZshRlQVilWYEOtgQDQDd/jSjPpl", + "QMfnZYjjxkreo398ERoyaXq029gH+9VwHe83QQFPo1ArUiO9dY1dRkJrPkqicvRo2O3v2CXjV6w8deME", + "1QLgz5SIOXp/elpymQsytuC4DSYOMRM168CTtZZhe4WuvXI0N8RZuAtsharYLRx3Xx1JoeizczGYhkMb", + "+O5y9dN7b06ZWRrNJ0vmVPG6hGQ2TFOfVqUfucyLd+9OjkvMgfHe1n5//0l3f7S1190N+1tdvLWz191+", + "hPvjneDxTg08ffO4mZuHwpR3c32mExAePJgmkS080Psti2UZpQplcW56Ix9p9RQV9GCT1wNOhRNGFWA4", + "UjbRzYCNb9Vkk6BpYCYpowoQAQCPhjI9ZXCm6EZs9NIBegHvwiMcQ76RG4Q2jsp+BBzOjR9VCwbXdQL/", + "Wj7k82mqtN4G38hpqpD+F0xbk8GaK8ubMDLmAL3i8I1wQaaMV+0e8zoEby2+XrWR2jasyIWfQmdWYB6g", + "55mQzMSsFattSeyfRnbbyGiI+t4oxd7ZFW9pbslXrhBW1mkZirY6LUcoCD9bDESz4/LmWBRZ0XfBQHAE", + "IjQP9EkVjSzIAcyESkUDYzViWNy6nWwBvUg4NCpA3XWhiR6xakL2kRMU709RG9IZ/4asUan/tZFdLRZ3", + "5e72k90ne4+3n+w1SlrIB7hawB9BbNPi4FZK+yBJh67yR83Uj87ewdmnz1WZxsZLYOdeiBFNBA+0tkoZ", + "ykuJ5J0/6T0p5mqEPB1FBa+TTeyChIAmdV9q7sf+pNGMjsfsz4/B5fYfgsZb13tye+Q17rKO/JrwSdFT", + "umA2klHXgDD6w+mBoYSszTh5QyTMAJ0ThYB/uggHcEhnIUmW5VxeiqW4l7F2d3Z29h8/2m7EV3Z0hY0z", + "BPt1cZSndgSFLQZvovab83O0WWA406aL0wR8CGYVOP8+QxaPuV8K4dS2046PS2r0pZxrbNuzuJbk760S", + "ZCdliQ6RVZmCtLDLvdTe2ek/3n20/6jZNrYW21BcL5cwDpPDkMfCmBRXvg3e9beHZ0i3LsY4KFsoW9s7", + "u4/2Hu+vNSq11qgAgsdAZ6wxsP3He492d7a3mqVO+TzoNimwtGHLssuz6TxM4VkNDykWRW+n7rTwKZ6G", + "wd6QIMI0Pgxc9Evl9DEQGUNhXssXocnBYJ0ECwdXg28bmWiVsj1GNeACpSwDZuqtdofezLtZL6bNebBa", + "jC/q0BFmmlw2xt8gMd6AdokgM8pT+RUa4ooEmpnGEedirW/rwoneEJlGyrggqUTvT38CIaKZC0lFknKo", + "vGW/JZkQN5zcWhu4xBN+rq4jVqPVaLL0yybcqdmmnWVhsKXtX5twFGpRlbLVV9dHOApSwB7D2XrqWUHq", + "AE8VXLTPTZBHFHHOUDDFbEIAy90gHbIJwmjKo7DX8l+VROFw7L3C4Fco4gYq4ZKQxMJymUHoz7TOQmcE", + "tV/wvKCcYaUKvO6j2EgVC7xU5sZHcU1xTukLHMwSlDQ9seKFLH7zScmaj/hEghWoIHylVwWPSbAwUSmY", + "GZi5WWyMx3Lm1bY+7T1DrEhv3xFqjk4+that1TEUzyiJA8GlRCSiE4A0e39aHuay+MOYMhprObv6Oro8", + "2AasKxPOpA8XBc402RiN0ncgegK7vuRIBB6G+E3v1YG5yLeh+CjGLAWgrgIjk+uECsMezS7Hp1yqYZZN", + "suZgpRoCCFMqSJ5y5s7LKcTvz42Ig3e856ITbTchl42auNHXC1zlb6pugPUy1UtRP7U6GQ/62Hgxn2Zp", + "Ck+eE1RNAFkn4ytH7aESWqWFZCPUZlyVxFIBeWajyUWV30bV/dQVdH252z9vmoy1PPfqDKvpCRtzD7Tj", + "Gg5/G9HuYhcSImIKMGQoJIyS0BmPmeff+rYgRj6SBIUpsZQzCqnAluDYbG+AcGTOKUbZpCLrqx02ccOb", + "MSzHaIJ+7YtNrhylP7L6rUiBViZIQCKcx1g3inigcuj3FC82LMgkjbBA1YTDJUOW8zii7LJJ63Iej3hE", + "A6Q/qF7njHkU8auhfiR/hrlsNJqd/mCYhwhWrmfM4GyAqFmQSr/5FH7Ws9yohKeD62XTfL8JFbub3OB6", + "44ae04jYnLx3jF4XGL0MYrK73a/LXKhptJSzsJjPua7ktizr2/Eu1fIwK3rgiU8zEUCVW4myI7I0X99s", + "IcRsWZ7GoisGtd2lsAOJKdO1ANbSyBPSLMqtGv7gRrMpSVDufXf/0eO9hmg5X+TrXFLT+As8m7N4iUez", + "ZqVOm7jN9h/tP3mys/voyfZaDioXKVOzPnXRMsX1qdQ2qTjNHvXh/9YalImV8Q+pJl6mPKBSnZIbD+jz", + "kq2bZ0nXXHvUwnRHxZV09yxlD2gzH+MSbemwpHIVSnG1yXhMwKgcGrp188FUousbjSHACQ6omnscJvjK", + "ILZnr1SyfZt408qD9ZDUtm0BG7TkkukoD+hsu87RX41rvcIL+41Bt2Q6qnPjv672apz4uQ+oeEXU4IYm", + "rwuw6C7I5nOFZSmqQ/8dAORyXmqtGj9k3mheE9rxelYWOo+M9GWs+0tAF5e/spwFt29JSa5SfNkRWr8F", + "17KhPSeyr8rk6qjcinywB+DNvhqOinB4S/EGS9h5+am7fr/NisQtfmdOsPX7K4SArvNhFRkM+NGOwZI8", + "b7tTYokablJcrAaEvgV8HxNTcCOEHxuOcCcgP/bnWwH2WViOc6Lcu+faok+jJXjOTBExwx7HlGsCuVfK", + "flQjiTvIuvjQVrxRqTS4O/XrajYDt2H4mNYCh4kgY3q9hFvMC+a4LoePS0uBsIz5LVE7xtdo9zEKpljI", + "ytgZnUxVNC87WXc92RNfVkKZKK06N0dHz1fTfbh4o2GXs9i6b8ueF3Id/KjtJBwuy1M/yl5zPuMEz0G3", + "rDUEH+/s9vs72/0bJap/LTD5Qjt1EaGF76wzp3T1WGwhi/9cRBy8EtTUJHNkkkoQHB9ANFWCA4IiMoY0", + "rgzpdaVNv9D18sHbS1Ib+J/xv1soV3jU+lmsiGOcgeLh2rE5/m4aLXdLW871KD5fHPaSZLFMzAQLWWPV", + "+NW9bn+n2997u7Vz8GjvYGvrNjLbMyLVhfA8/rh19TjaxuPdaH/++M+t6ePJdrzjTRe4hboFlTKAlTIG", + "dg4JEVUoySoEqyQRZaQrs7C31QHIS2SBuUlauf/X8z6YGSxVFs7LkyzqDFjlxKlWVLuLxCY7+qUulOrw", + "T46XD/tGcWTVgfgZrDoU4KdmgwGkla0vhfVIWcNz513hxcYnz9LYxlVnjy/oG7a2d5VrKO7j55JgLO2w", + "ZSf24qnmMeEmXFA1jZcfD9lrGUgAXIZ/lCosJ9L00MmEAXBs8efs7qNY21l/3Oq0oo+75T1jf2+eUmWT", + "4jMGtEtdVAMa3A0ALvFyKsAruWkhTHgCFgQI8fNWd+sJ3NBHH3d/7nef9NBvhUiBjqFWkXxb7u3Sr/0m", + "NMwEJeid5uZ868la1+iOnss46Fd7LtUdxDZd3vJ4jtrpzgoXNl1a4PzxwhpXsopurU6QPc2GRS0pJBGe", + "+2otRHiORmQM4NQV+7DIZGhEJpTJDiLXTuhgiTAqmkKuLHtfDlqICzRoPYoHrR46tCATYK3moMyl5gGO", + "t8AnNLYloiwgb31YynbcLH+iajysl8vvvvKoZz2/fvbkbX9rbeSh9Y7J3hdEVX+RudvMxI2wVHW2xUss", + "VWaTIpFaC6MDVXpYpVi/RUa36RAQ3gp4AwdIVrwcDhJe5oqf84R00IQrlCdCrNT0YPgiZV5+KI8/r7QM", + "CBe1DLG9giGajSlLZqTLxNfJMUoED9MgjwKOYNBpEBApxykUju411epX37PepkMDwuvHXKDVDo06D8bq", + "VFtyXb/er8i1KnSpGbZ+qbf6q5f6VrwgnVaahKtlmHmpmQRbC41kRVypxydTJntFEyxM5kMDif6mSMFF", + "IxdqTqFAq0Rp4oogap5a5CTpKX2Ir4denIRjEhF9TC02gkxVUhv6QmUuRVeL1K29fb/bEF8PA0jKXBjI", + "r4Qk2laJubRVQmPM5t6BVYHAUbvvimBKBM13DRiZpVZ5cI9XamK1S9UcU73i1TZZb0UI+wyl5OsCqtsv", + "V5a7uA0/3H0qaa/t5YIsM4ZDa8jwXVz/JqzX1MCOKuf17iPZTCd7ay3jOgChamhv0c182P2HcSujYe9g", + "8+e//b/dD3/9i7+STslulkR0QzKG+85LMu+aEsHaRu+V0WcB3Egr07b+jCI4BqdRcEmMkyrG18XxPupn", + "QmP+CscLU4CL4piy7N8rJ/S3v9RfsxbI+A7k5EqWvQ0cX8XdadKOiZg4sHwX3Ae107Xaf0nmEhXAAq1G", + "4vjsJ5l9UqzSe2G0uB7Ulh5RwFyVA6aNUhwEJNHGgAVNozAWwUF4VGtVW9BCF4yvxQoGpFabHFXBJPvk", + "rQh90GLkqmt6CLuadXYf7dny/EVKbi2skG/NTH59XTlLTWWP0+cllZA84mKkCy+jNokTNXeQvC6KdWO9", + "fP/DrEHvxfVXBjvrP/ka0KzvlmKxfofFVItwDG5AK4EYFta/FgDRHwZ3XMVVMnvSFogr4wBVjDupuvVR", + "crHWRoYQ7LkY0aafmUBSCz46Saso7JsxU5sW6tiXvBJC/eelocP5LnPYBF34aHVE7FJ9uDCzwkjq1+bU", + "qX3Vmtz1BDrTpLmaEkEKCwEf5Hita5LMhnU2SIkygKQJEd1qBUFT5EJQiBPNLHZHgiz0d9GNuRyP6BRf", + "Zz2ACxzLhYsimEeOzLf14ikUrnnjKsnRsWsChlFRyv3gQmUualILf3Exily1OG/zvnfjWVm1RPrV7a0K", + "c+Z9lFjTx4+/YaqecwFqfH0C0q1jFIGJEBIBGdhVBKJG8D00JuGQp2r5/reI/jb7KHSaeA527EwWDExs", + "C0yvkAUuRSYfwwef3iBJkAqq5trGtRrliGBBxGFqNjwQEjqCn/OOATv482dw9o098YYvCCOCBujw7AT2", + "Y4wZaLro/SmK6JgE8yAiFvp1AS0FlLzXRyfWTHT4fKD2UwWs52o+H56dQAlZYUylVr+33evDZk4Iwwlt", + "HbR2eltQUFczHExxE0oNwJ82lSAzN05Cqwc9Na/orwSOiSJCtg5+94TkKyJM6QIJWieeFNT+BFNh9f4k", + "gkQBwypUfwtgVe4oPTDncccQvLGXS6q5DZskyWu7rB80J5hdA1Pc7veNRcmUPXhxXlp08w+bXpn320if", + "A/J4kJsW9HqnU1qSf+60dvtba41nZTVQX7fvGE7VlAv6kcAwH61JhBt1esJMLDcyqCA2WqW4z4CFijvs", + "9w96vWQax1jMHblyWiVc1inDRJvejFzZuhl/8FEPWR8+gNXKKU8jLU2QCVR31rrCojf5iLAIpnRGBsye", + "06ayKxZgi8dIn8/GbClvDdO1Wf0shfApD+cV6mbNbermus5zmxO4CoopyRCwvYZ1RXFyny1lDIprSmIR", + "RLPqEIshMVANWQbcWwaaMMxUXlzXlEG+JHPrFvY22AiERws8WBYCVfczcPjtDX/2CWCd+hO3jrNnyJK3", + "rE4wuEsJojTMdS4XEI3FCEeRF6VhEvERjmy16EviUVFfwBuWKEVYWKfcMB4SA/GZzNWUM/N3OkqZSs3f", + "I8GvJBFaBbJQ35bWtlSqZV0o209jgNs2hUR0n5tmiJufLsn8c2/ADsPYFYmRttJ/JLkto23AjqhELoLW", + "8K4fjLYmOOMolYrHlqVYseqnGSZPVZIqezEtibL45PA6FIWVUxIOmOLokyATKpWYf978lPf4GWwXgkPN", + "J4VXzJQ2P9Hwc92o5RDr2Q/hVY/1R4AAg5Y+XQYt/fdEYG27pHIKrgwJ7otJcUnbWea81gs3qhQOMEMJ", + "TwzqADCVqQ5eagNqPeAoQgq2kvtWa5uwkjXzsYlEvsKFNovIpH1UthGUMCxspv7uvn8/SRII4nNw/M/5", + "61cIjiq9Bua13G1kboaZPkVRmIImD733BuwZDqbI6E2ALDdo0XDQyqyLcAPGmkob5tztgor7sx7az6ab", + "Dg1/7vV0U0Z7PkC/fzKtHOi9lMRDxS8JG7Q+d1DhwYSqaTrKnn3wE7QuGeO8JAhQ28j+DVepB0Ah8mPQ", + "nBuYhYhbWRvNEUa5BCr6UUaUYbG0zJCH9JaC2pTHE1kkxqcBOEAHrYOBc4EOWp1Bi7AZ/Gb9pIPWZz8F", + "rBJdD2NmKi05XTtjor1+f2N1lqSlr0eFLr2ot9/nBe1r+6spHlbpWlQ8zOQcBqNeQVMzy6hbd6D5PMWh", + "q8LwQ8VboeJZz0VBeYPvi+eAYd+IGAO3ooFpezZyGthS68SwBYCQgsXhcpqNwUGdBpczb9H8qJrzi2bF", + "bt0uC2CIkeO/3TvgP+g3rzsP/T65q35xBIiiWRXmh8WOsFiOETt+i/gFUd8Cx/XvSpRa+NP75N+Hwj8v", + "iNX7cqJVpNkmmbn7Jj9yA2RsSNuKeVnbqucwpu45YQo9g1979r/O4gEc4ouITy4OkCFhxCcooszexhVu", + "i/ShaGkJH5mkjew7m8PhYLPa5vz89z//BYOibPLvf/5La9PmL9jumwbLBGB2L6YECzUiWF0coF8JSbo4", + "ojPiJgNAmGRGxBzt9EHNTAQ88lT2lAM2YG+ISgUr3FoaBCtpGwTTg8F8KEuJtEkv+kU6tvAaxsHsMeHd", + "XjakvNMd3VmMEDYzKExAn4qOByBfmhqsYWt/tfzeMzPnkv+s6itf8Jiuli+KXCvDvV0zwDUFDJDYt+/g", + "gZ00ap+fP9voIbAxDFcAhApozHkzVnnu/ZBJq2WSkShlgQJUNrKpUNS91v97bN9p5gC2LX5PHuC6KvX1", + "LmDj8iCChI5eP2yFJu5gP92ca9jnnz12qY71Dtqbz7fYhQsGamQIf711dry3SHPzpECy+zCBUduFlLsC", + "i2dHJ66Sz8a9Mf2dnBp6prb+RXZ0IG7KOt6ZWXbE2TiigUJdNxYowRCTzFQrM8hDEQdv7KgRdvOqghUW", + "z7fNEvZO7UmXwfDkR97tnx6VTtc5RnJAxZzXfpwkq1jnmMqA628L3NINcGLLSxr1JdunRS5a5ZAyEerZ", + "kbNUXbLi+eTYbci7c03ZrlNWPRvuQCgeVwTiPQrCSsm8AgTpQ+Lmd9kqOliHJZ6rb4s1+3enBd21F8vH", + "5g/JjRVWyKaloAFirj1AXxD1i3njFhfa9uCZ+DkRblc7xGiYdTYt8ykKpiS4NBOCC+nltu+JeaWZ6Wva", + "+54sXyDPOhqLJfkPFaWBsZvTapmBe2LLAN6efQs9rGXefr17XstgHiJDsMnIeaxNhT0s5yzY+K6ueu/k", + "NDPEfpCH2VkaRe7GY0aEQlm17eIZsPkJwpJW6/Zuty09Dt69edklLOAQh5bFUPmVKPvkK2v4ZsHMVH6w", + "SROb0OQWU3ee1Wk4X7D+JlwQZdXc/2v7ua3n/l/bz01F9//aOTQ13TdujVn6dyWa71rjfsDMpxVuWiYa", + "iCYGlV1XaajZWw2VVPf+d6WnmkmvpalmdP2hrDZRVovkWqqv2qW4VY3V9HFPVzIZs/moDY9cfOJ3pqne", + "rZfPcqRDW6ayfO1hy+lwAX5eeEQZSiV5gAGUNOO44rHR0F2db8ilx4dj3ZPjDhCyo0kH0Ec2QeSOnNdu", + "HHeu3Np+795zfRiP6CTlqSzmnsRYBVMibbJSRMoC+KGp3fnxXKt4f8Nc2r/Lo+PO9eoffH9LGn91QY3w", + "NjdQq3R+91ZTnd++r3V+k0Jtc9csPlPHYfdt1AQVuiTqpmxcyjVfDHb0jctni6B32lDJzQUEFsTBgP23", + "tj9+VwTHH352STJpv7+9B78TNvvws8uTYaeOVQhTglqo1cNXx3DtN4Hsc0BjzVPyquMw5RuA9RyAzX+c", + "gZTffDa3kBwX/rCQGllIBXItt5DsWtyuiVTGsLpzG8nxm4/gFsTkh5V0F1aSTMdjGlDCVF7dbCFIzBZH", + "fIC5ZczeDxWCO0oHbWMrKduUKxTQHFv/zgN7TnIwwbs2jhyM/8OMkeeJxcW25kh+GNbbI98aP/TvVjjf", + "vR3ykFnMKPyLpEu0TukrmglIj3GqICgxRwiBqE8kjNaetdhDea1KmSYJF0oatEhQgA0c/FQrwD5kyTJY", + "pA8dEiB8KZGdAQO4f/3Y5PJvXpK5wYKknGWwj9lMLf6jL/eqDKV5r9vo6+tYfpzQRjrWHW9ji/x8fzrW", + "vYmOO9G0TkqA+u1sY4BBOSLZTuZZch/9SNlk40FFoBphlc2tgGfkUbU2x7Z8pN8Ees7FZVOh4Cln9ABk", + "Q3GG36D1pYcH+En3b4SBeWL2j2aaO5cbCzWq7jNonVYlSRCloRYdToS4w3cseDy0PxqET70rLH4iGHWB", + "bfW+hYzu/Q5M7FdcIRonEdF6DwlR13CTXk2rLDmYbCoLFd3WE4J62xRTCAx8l3QVUSyYO1xHuAVrw83k", + "4nJ5pWbEJ6thA7LOXY68BzdgwAyMN3GY3xcoE7JQM4hEJFDoakqDKWAIQB0hKCQJ6f04SS4y0KCNA/QC", + "dmoROwk6b0sitOoYcCZ5RAw0wCyOLw4WMS7fn57CRwY+wKBZXhwgh2uZHRBSv1XEBMhKrbyySAdtzUmC", + "R5FZ0QutZxfmt2HRAnJQpwHzIQcwcmUbpGN0UQARuKhBEXAC9SWfyPtSZTv1UHxmLoojAYQzvElY2Kpz", + "XdPIjx+w1feWoWiIZWCGcctQBguDecknGQxgiZVxkjRlXztM4OJZHC/hYdQu1ISUKuSp+ptUIRECPrbc", + "XcfcqI0D8w+FLzWj2nomWVVNYD/vBY3B5fKSSgvVQvEO869ZHLc6LTueAp7XGh6GFZgQ1QYXLxL0yhSA", + "H374EtaBdCgL+wKmQ+XksFXH61VuW0z9u/doWUKF34NdWr4ByEdBWV53RkDhaFf850Hlhps6+1VdzFQq", + "8u0RN8uuLNQsbHYhsFDt8BswWlfdE2Sl67K6end9YbA4goecNiAXZjPmoppQvOom4ZtnpK+3JAtTbcIh", + "P3hz/SuHRoyZpEtKGEIFRomwK+sHSLjBlHNZYPsRmeIZ5cJiVtuaSRlngsvCWI823uhCs+qFrQB3YdXz", + "A+trQrj4yPbRg89tlJL/C/co/+J5wdrOJH7HqdSAmycRRiNByRglOJVEa0tpTJCpyWChjwkOpq5IcW/A", + "3k4JsmX5Cg6ErIorlehiK77ooFGqUITFBKwd89DEHgkS8DgmLDSlNgdsSvCMalNNoAgrwoJ5VxIovToj", + "eckHbbrbOx1T4Tcr7thBriYoOBguChU/L1AiCDCRMZdZqbzmgImU/d1g/elmL9xALxCRCo8iKqcZun6A", + "Q8ICL5De+bctxr6+E/ecqMWimPdyy3MjWXqf1z5FX2ZWlvibuBF6YKEtXLiKgA3E/BKlV9abhuVYsfO8", + "EOh/4JY2c3VzvKebmYzEy3bxt3ElU6oE/uNaRtktGaamO1Kulv3d3rXk5WtTVrpusT7Zm164ZNjxGZnX", + "knmbn9yfJzfwkX0jkrCzpCa9v4N80t+CyLVUvZHMvSfnoPUlFbxi9yiC7aDuT33ioiDlvgkxbDZcJo2L", + "MkcJDDaVqd//QxiX7r5NeMBNhbHzuC5cgBfEM2XdJMJ1ctk6Z2sFcKVa/H9YvGBNLfzPVhLep+DLbwTu", + "TNidZOLNCLwEzyOOv/d7mYALYVLgbAHXhwPBVPAFFi6Y2uBx62QSouPi79+fnm7USQmhlsoIoR6whCgX", + "ggxiT3271zMiBA1dsb2j02Nbmo9KJFLWQ69jChXwLglJoLQG5ak0Zfl7xQr1i2XDKiXoCVNinnDK1MpR", + "5K/ezmA+36jY2B3LSQtC991fHoMX/uEJKZAdWl2xE1huRSqsaoPxXHAaZaZCoNa28IinuvWFCvpoTCMi", + "51KR2ETmjdMINhHAlNoqNvY7k4PXQVRJpPdDB3KWEiJiKiXlTA6YLZidEKH71p9DudQ8yMjrvFc4k5pn", + "RvR9GwFsUFQfYrawqqNauZ4+ThJXT98XJGWH9wVDeg4RaUjO4xGPaIAiyi4lakf00hgdaCZRpP/YWBrS", + "NoTvvnaNnpvvLE3pEzbm3jIGhmczZv4+0jbKYs1dIj44sfaCFDeLkz+w0H6xJlfKNUFw1FU0Jlm6MEoV", + "jehHI+p0I1QqGpjaz3myGpSttflqA3ZKlNDvYEFQwKOIBMo5VzYTwYPNQdrv7wQJBVyHHQKDA4FX/ziG", + "Ho/O3sF7prRuZ8D0P6Dht4dn5iZ2jK2PoDBQRtQVF5foZPP1iiDfcyDTf3CUnJng0qwx74L/uL5bPxe0", + "dg/Jmi3Kk2UGEE+++zBOq8H98BY8TG8BJONns2lPBA5AKZbTVIX8ivk9AzMepbH+h/njZBWkg8LB9D28", + "+s1ou2Y4K7txE3wQm9LOKSSmzMq9XFAYgj3U+FJNODcFUGJKkXveU+BQfY/c/fWd8kU6foNXk5airoTR", + "N7O37vrks2NwSEVFejyUbW44zc1E8eXepytM671PTyMeXEqUMkUjcJgUNE0MyIn6xxzpzl78gZoA2ZGu", + "+DIi1wkVgPkBPt1CS0TPWCKMFBExZTjahDmbRgCzz3mx8IxTSFIOIgppYjQkKOFRBLgkV1PCkJ4NOKpc", + "A4V7Wmkx84vvFK8YFUcjEvCYOBzDDZ/p9hum6jkXZVDCb0Uuvi3QX89HT1XPcwUOY32PX4TLeIqvIaw5", + "TO01sRtR+wXPfzSuoA6CtRm0dvpy0OqgQWs7HrT0ChxhcKFihR6hmLJUEdlDx8a/BWmoe30kScBZKB2c", + "ovPg7fRlXVKqYcuaDMc9+O4u1R7LVUDKN7YTn3jQ7yH9PSTYoHZxw9k9GXZg04WIpwoCuN2+sm+FRIF7", + "ZOPOb2ALe+SHbd9Ekv9mt29JRsEqa3FZWHoj2TPAvZVeN5dUMeUyx+lDAU5wQNW8g3AU8SD3HqQyux3o", + "ZkMZCYIvtQ3VG7A3GdSfTYRAR2fvOs5phkIqL00L1i/WQ69nRMh0lA0OgTQwHjxYDBIOmOIowFGQRppv", + "yXhMAshhiGhMlazxq2VDuc3CcXknnoV3Dy3pHpozyc8TsHo5W8gKx22apd4UJIgwjYtOpSpxQPWFK11w", + "+450o1wfw+PIXm8FgkuJbFNdEtEJHUX2skb20FutcuCYDFgSYcaIQKk0cUd66N1EEClTkxijG4DKnIaj", + "OigHOkkEV9ZNHHEupPHsag5/f4qkIskSNntjWj6FOd8SsKpp3PZ0TwZDZQz1x5J9BekFMZxiCK75SB/T", + "9xDsYwZ03wCsD2XjvxV0MiFC7wpshKy5GjXb2pHTbPpSpkctqvh59lYzVPGs1UI0dyHSeSlQxdC9OAQF", + "ep0bWE/nl7QWy8Q+Wi/74lf9UcO+y1H+/kHYR184y++lWNN5Ibi6KRZ5zuEPDRa8MPLSVi0lKKyGI2ic", + "kXCbGQKNcQfuDW7gIaMM4FLaQR2cwLfHCP27zY67a2Dih81bJZSAUimSmlSp1fCd3wQH3g5u5z1nh94A", + "t/ObylcC3MX7yxv9pjKVSn5AV27hu0fmvK0EJQPPCTAWdQlKRurZQIKlhtJ7+04zM8m2+D1p8PbueQ39", + "3ZH9h9XfwGQoEMvvsjO50Q63hcSJmrvLRT6uXABK+hGSMXzAD1kMwe3hLdzgev3rsYfj09rL9R8ViO7s", + "/j4v03py/PDLDhX3XOlg2dSnTheLYEpnpN7pXt7BlkSJIN2EJ3C5EhqCWXq4s0xh0Zt8RLZ5i1Vl/4Wo", + "gzgmIQqpIIGK5ogyxUEimD5+kkhwbQnAcy7mPmd6cec+Fzw+tLNZcR7aPWWdYfmdbzzvhljh7sxJmyUu", + "tC+4aXd321rgIcrQi6eoTa6VMIi7aKwtH0THGUnJdUBIKIEnN4oD3urXeDbpRzKcjJqMcgl28muLTY2C", + "VCoeu7U/OUZtnCrenRCm10Kr+mPQZBPBZzQ0pRtzos54ZKi6VUPQdf2uWqmw6X25cWEGdy86TJMDafKR", + "JmWxYEIXWgetEWUYBrcSpbi8p0xCle4PU0hryPeO45zWjyPMWn5tZ+xoTtRGjiOi4txA4238OOYe8jFX", + "DEx1Z1rptGtWXK9ZrGrDENLbAMzN4pjv1m39/tsJr6TyQUZWWtf5LDNI69zm3xYL9u/ufLhrd/n7BxyO", + "/4I447vgKocGdIs+hnnJAxyhkMxIxBOou2febXVaqYhaB62pUsnB5mak35tyqQ72+/v91ucPn/9vAAAA", + "///CkTc9NWQBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/otel/README.md b/lib/otel/README.md index 04f9113c..ed4613b6 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -70,6 +70,17 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_instances_restore_duration_seconds` | histogram | status, hypervisor, algorithm, level | Restore time | | `hypeman_instances_standby_duration_seconds` | histogram | status, hypervisor, algorithm, level | Standby time | | `hypeman_instances_state_transitions_total` | counter | from, to | State transitions | +| `hypeman_snapshot_compression_jobs_total` | counter | hypervisor, algorithm, source, result | Snapshot compression job outcomes (`success`, `skipped`, `canceled`, `failed`) | +| `hypeman_snapshot_compression_duration_seconds` | histogram | hypervisor, algorithm, source, result | Active compression execution time only | +| `hypeman_snapshot_compression_wait_duration_seconds` | histogram | hypervisor, algorithm, source, outcome | Standby compression delay wait time before start or skip | +| `hypeman_snapshot_compression_active_total` | gauge | hypervisor, algorithm, source | Currently running compression jobs | +| `hypeman_snapshot_compression_pending_total` | gauge | hypervisor, algorithm, source | Delayed standby compression jobs waiting to start | +| `hypeman_snapshot_compression_saved_bytes` | histogram | hypervisor, algorithm, source | Bytes saved by compression | +| `hypeman_snapshot_compression_ratio` | histogram | hypervisor, algorithm, source | Compressed-to-raw size ratio | +| `hypeman_snapshot_codec_fallbacks_total` | counter | algorithm, operation, reason | Native codec fallback count | +| `hypeman_snapshot_restore_memory_prepare_total` | counter | restore_source, result, hypervisor | Restore memory preparation outcomes | +| `hypeman_snapshot_restore_memory_prepare_duration_seconds` | histogram | restore_source, result, hypervisor | Restore memory preparation time | +| `hypeman_snapshot_compression_preemptions_total` | counter | hypervisor, algorithm, source, operation | Foreground operations that interrupted active compression | ### Network | Metric | Type | Labels | Description | diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index 82cd4300..8b3d6223 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -42,6 +42,7 @@ Snapshot memory compression is optional and is **off by default**. ### When compression runs - A standby operation can request compression explicitly. +- A standby operation can also set a standby-only `compression_delay` so raw memory remains on disk for a grace period before background compression starts. - A snapshot create request can request compression explicitly. - If the request does not specify compression, Hypeman falls back to: - the instance's `snapshot_policy` @@ -56,6 +57,8 @@ Compression runs **asynchronously after the snapshot is already durable on disk* - This keeps the standby path fast. - Standby can return successfully while compression is still running in the background. +- For standby only, Hypeman can wait for a configured grace period before starting that background compression. +- During that delay window, the raw memory snapshot remains in place so resume can skip decompression entirely. - That means a later restore can arrive before compression has finished. - While compression is running, the snapshot remains valid and reports `compression_state=compressing`. - Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. diff --git a/openapi.yaml b/openapi.yaml index dae2115b..adcb8ecf 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -321,7 +321,7 @@ components: description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor snapshot_policy: - description: Snapshot compression policy for this instance. Controls compression settings applied when creating snapshots or entering standby. + description: Snapshot policy for this instance. Controls compression settings applied when creating snapshots or entering standby, plus any default standby-only compression delay. $ref: "#/components/schemas/SnapshotPolicy" skip_kernel_headers: type: boolean @@ -486,6 +486,10 @@ components: properties: compression: $ref: "#/components/schemas/SnapshotCompressionConfig" + standby_compression_delay: + type: string + description: Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Applies only to standby compression and defaults to immediate start when omitted. + example: "2m" CreateSnapshotRequest: type: object @@ -509,7 +513,12 @@ components: type: object properties: compression: + description: Compression settings for standby snapshot memory. Overrides instance defaults. $ref: "#/components/schemas/SnapshotCompressionConfig" + compression_delay: + type: string + description: Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Overrides the instance default for this standby operation only. + example: "45s" RestoreSnapshotRequest: type: object From 889812cdf95dc71aa4da22a45e608bc3bec27bd1 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Sat, 4 Apr 2026 16:16:58 -0400 Subject: [PATCH 2/6] Update Stainless config for standby delay --- stainless.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stainless.yaml b/stainless.yaml index c88f36cc..291e7b87 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -73,6 +73,7 @@ resources: instances: models: snapshot_policy: "#/components/schemas/SnapshotPolicy" + standby_instance_request: "#/components/schemas/StandbyInstanceRequest" snapshot_schedule: "#/components/schemas/SnapshotSchedule" snapshot_schedule_retention: "#/components/schemas/SnapshotScheduleRetention" set_snapshot_schedule_request: "#/components/schemas/SetSnapshotScheduleRequest" From ab68241e01f5510a5e32b4aa2716962cc9feca11 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 11:13:43 -0400 Subject: [PATCH 3/6] Recover pending standby compression after restart --- lib/instances/fork.go | 1 + lib/instances/manager.go | 3 + lib/instances/restore.go | 1 + lib/instances/snapshot.go | 3 + lib/instances/snapshot_compression.go | 130 ++++++++++++++ lib/instances/snapshot_compression_test.go | 197 +++++++++++++++++++++ lib/instances/standby.go | 7 + lib/instances/types.go | 11 ++ 8 files changed, 353 insertions(+) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 9e846ab9..8c80418a 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -472,6 +472,7 @@ func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { dst := src + dst.PendingStandbyCompression = nil if src.Env != nil { dst.Env = make(map[string]string, len(src.Env)) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index b2b7a042..68f2cf4e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -181,6 +181,9 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste m.metrics = metrics } } + if err := m.recoverPendingStandbyCompressionJobs(context.Background()); err != nil { + logger.FromContext(context.Background()).WarnContext(context.Background(), "failed to recover pending standby compression jobs", "error", err) + } return m } diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 68bcb7f1..53eecce1 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -81,6 +81,7 @@ func (m *manager) restoreInstance( if err != nil { return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) } + stored.PendingStandbyCompression = nil starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { return nil, fmt.Errorf("get vm starter: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 3c7f94bf..72874e31 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -110,6 +110,9 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if target != nil && target.State == compressionJobStateRunning { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, target.Target) } + if err := m.clearPendingStandbyCompression(ctx, id); err != nil && !errors.Is(err, ErrNotFound) { + return nil, fmt.Errorf("clear source standby compression plan: %w", err) + } if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), "", stored.HypervisorType); err != nil { return nil, fmt.Errorf("prepare source snapshot memory for copy: %w", err) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 78d35312..4c71e35d 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -129,6 +129,16 @@ func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { } } +func clonePendingStandbyCompression(plan *PendingStandbyCompression) *PendingStandbyCompression { + if plan == nil { + return nil + } + return &PendingStandbyCompression{ + Policy: *cloneCompressionConfig(&plan.Policy), + NotBefore: plan.NotBefore, + } +} + func normalizeStandbyCompressionDelay(delay *time.Duration) (*time.Duration, error) { if delay == nil { return nil, nil @@ -250,6 +260,115 @@ func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { return "snapshot:" + snapshotID } +func instanceIDFromCompressionJobKey(key string) (string, bool) { + const prefix = "instance:" + if !strings.HasPrefix(key, prefix) { + return "", false + } + instanceID := strings.TrimPrefix(key, prefix) + if instanceID == "" { + return "", false + } + return instanceID, true +} + +func (m *manager) clearPendingStandbyCompression(ctx context.Context, instanceID string) error { + if instanceID == "" { + return nil + } + + meta, err := m.loadMetadata(instanceID) + if err != nil { + return err + } + if meta.StoredMetadata.PendingStandbyCompression == nil { + return nil + } + + meta.StoredMetadata.PendingStandbyCompression = nil + if err := m.saveMetadata(meta); err != nil { + return fmt.Errorf("save metadata: %w", err) + } + + logger.FromContext(ctx).DebugContext(ctx, "cleared pending standby compression plan", "instance_id", instanceID) + return nil +} + +func (m *manager) recoverPendingStandbyCompressionJobs(ctx context.Context) error { + metaFiles, err := m.listMetadataFiles() + if err != nil { + return fmt.Errorf("list metadata files: %w", err) + } + + log := logger.FromContext(ctx) + for _, metaPath := range metaFiles { + instanceID := filepath.Base(filepath.Dir(metaPath)) + meta, err := m.loadMetadata(instanceID) + if err != nil { + log.WarnContext(ctx, "failed to load instance metadata during standby compression recovery", "instance_id", instanceID, "metadata_path", metaPath, "error", err) + continue + } + + plan := clonePendingStandbyCompression(meta.StoredMetadata.PendingStandbyCompression) + if plan == nil { + continue + } + if !plan.Policy.Enabled { + log.WarnContext(ctx, "clearing invalid pending standby compression plan with disabled policy", "instance_id", instanceID) + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear invalid pending standby compression plan", "instance_id", instanceID, "error", err) + } + continue + } + + state := m.deriveStateWithoutHydration(ctx, &meta.StoredMetadata) + if state.State != StateStandby { + log.InfoContext(ctx, "clearing stale pending standby compression plan for non-standby instance", "instance_id", instanceID, "state", state.State) + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear stale pending standby compression plan", "instance_id", instanceID, "error", err) + } + continue + } + + snapshotDir := m.paths.InstanceSnapshotLatest(instanceID) + if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + delay := time.Duration(0) + now := m.nowUTC() + if plan.NotBefore.After(now) { + delay = plan.NotBefore.Sub(now) + } + log.InfoContext(ctx, "recovering pending standby snapshot compression", + "instance_id", instanceID, + "operation", "recover_snapshot_compression", + "source", string(snapshotCompressionSourceStandby), + "algorithm", string(plan.Policy.Algorithm), + "compression_delay", delay.String(), + ) + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForInstance(instanceID), + OwnerID: instanceID, + SnapshotDir: snapshotDir, + HypervisorType: meta.StoredMetadata.HypervisorType, + Source: snapshotCompressionSourceStandby, + Policy: plan.Policy, + Delay: delay, + }) + continue + } + + if _, _, ok := findCompressedSnapshotMemoryFile(snapshotDir); ok { + log.InfoContext(ctx, "clearing pending standby compression plan after finding compressed snapshot artifact", "instance_id", instanceID) + } else { + log.InfoContext(ctx, "clearing pending standby compression plan after standby snapshot disappeared", "instance_id", instanceID) + } + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear completed pending standby compression plan", "instance_id", instanceID, "error", err) + } + } + + return nil +} + func nativeCodecBinaryName(algorithm snapshotstore.SnapshotCompressionAlgorithm) string { switch algorithm { case snapshotstore.SnapshotCompressionAlgorithmLz4: @@ -426,6 +545,11 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar defer func() { m.recordSnapshotCompressionJob(metricsCtx, target, result, compressionStart, uncompressedSize, compressedSize) + if target.Source == snapshotCompressionSourceStandby && target.SnapshotID == "" { + if err := m.clearPendingStandbyCompression(context.Background(), target.OwnerID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(context.Background(), "failed to clear pending standby compression plan after job completion", "instance_id", target.OwnerID, "error", err) + } + } m.compressionMu.Lock() delete(m.compressionJobs, target.Key) m.compressionMu.Unlock() @@ -618,6 +742,7 @@ func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) ( func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, hvType hypervisor.Type) error { start := time.Now() + log := logger.FromContext(ctx) if jobKey != "" { target, err := m.cancelAndWaitCompressionJob(ctx, jobKey) @@ -627,6 +752,11 @@ func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jo if target != nil && target.State == compressionJobStateRunning { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, target.Target) } + if instanceID, ok := instanceIDFromCompressionJobKey(jobKey); ok { + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear pending standby compression plan while preparing snapshot memory", "instance_id", instanceID, "error", err) + } + } } if rawPath, ok := findRawSnapshotMemoryFile(snapshotDir); ok { diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index ea4fdff6..8610eb20 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -454,6 +454,203 @@ func TestStartCompressionJobDelayedCancellationRecordsSkipped(t *testing.T) { assert.True(t, os.IsNotExist(err)) } +func TestRecoverPendingStandbyCompressionJobsRequeuesDelayedJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + now := time.Date(2026, time.April, 6, 12, 0, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + const instanceID = "recover-delayed" + delay := 30 * time.Second + snapshotDir := mgr.paths.InstanceSnapshotLatest(instanceID) + rawPath := filepath.Join(snapshotDir, "memory") + + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, os.WriteFile(rawPath, []byte("pending standby snapshot"), 0o644)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(delay), + }, + }})) + + timer := newFakeCompressionTimer() + delayCh := make(chan time.Duration, 1) + mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { + delayCh <- got + return timer + } + + require.NoError(t, mgr.recoverPendingStandbyCompressionJobs(context.Background())) + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + job, ok := mgr.compressionJobs[mgr.snapshotJobKeyForInstance(instanceID)] + return ok && job.state == compressionJobStatePendingDelay + }, time.Second, 10*time.Millisecond) + select { + case gotDelay := <-delayCh: + assert.Equal(t, delay, gotDelay) + case <-time.After(time.Second): + t.Fatal("timed out waiting for recovered compression delay") + } + + canceled, err := mgr.cancelAndWaitCompressionJob(context.Background(), mgr.snapshotJobKeyForInstance(instanceID)) + require.NoError(t, err) + require.NotNil(t, canceled) + assert.Equal(t, compressionJobStatePendingDelay, canceled.State) + + meta, err := mgr.loadMetadata(instanceID) + require.NoError(t, err) + assert.Nil(t, meta.StoredMetadata.PendingStandbyCompression) + _, err = os.Stat(rawPath) + require.NoError(t, err) +} + +func TestRecoverPendingStandbyCompressionJobsStartsImmediateCompression(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + now := time.Date(2026, time.April, 6, 12, 5, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + const instanceID = "recover-immediate" + snapshotDir := mgr.paths.InstanceSnapshotLatest(instanceID) + rawPath := filepath.Join(snapshotDir, "memory") + + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, os.WriteFile(rawPath, []byte("standby snapshot that should compress now"), 0o644)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(-time.Second), + }, + }})) + mgr.compressionTimerFactory = func(time.Duration) compressionTimer { + t.Fatal("unexpected delay timer for immediate recovery") + return newFakeCompressionTimer() + } + + require.NoError(t, mgr.recoverPendingStandbyCompressionJobs(context.Background())) + + require.Eventually(t, func() bool { + meta, err := mgr.loadMetadata(instanceID) + if err != nil { + return false + } + _, rawExistsErr := os.Stat(rawPath) + _, _, compressed := findCompressedSnapshotMemoryFile(snapshotDir) + return meta.StoredMetadata.PendingStandbyCompression == nil && os.IsNotExist(rawExistsErr) && compressed + }, 5*time.Second, 20*time.Millisecond) +} + +func TestRecoverPendingStandbyCompressionJobsClearsStalePlans(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + prepare func(t *testing.T, mgr *manager, instanceID string, now time.Time) + }{ + { + name: "stopped_instance_without_snapshot", + prepare: func(t *testing.T, mgr *manager, instanceID string, now time.Time) { + t.Helper() + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(time.Minute), + }, + }})) + }, + }, + { + name: "already_compressed_snapshot", + prepare: func(t *testing.T, mgr *manager, instanceID string, now time.Time) { + t.Helper() + snapshotDir := mgr.paths.InstanceSnapshotLatest(instanceID) + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, "memory.zst"), []byte("compressed"), 0o644)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(time.Minute), + }, + }})) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + now := time.Date(2026, time.April, 6, 12, 10, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + instanceID := "recover-stale-" + tt.name + + tt.prepare(t, mgr, instanceID, now) + + require.NoError(t, mgr.recoverPendingStandbyCompressionJobs(context.Background())) + + meta, err := mgr.loadMetadata(instanceID) + require.NoError(t, err) + assert.Nil(t, meta.StoredMetadata.PendingStandbyCompression) + + mgr.compressionMu.Lock() + _, ok := mgr.compressionJobs[mgr.snapshotJobKeyForInstance(instanceID)] + mgr.compressionMu.Unlock() + assert.False(t, ok) + }) + } +} + type fakeCompressionTimer struct { ch chan time.Time mu sync.Mutex diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 00c9532a..d9de8053 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -214,6 +214,13 @@ func (m *manager) standbyInstance( now := time.Now() stored.StoppedAt = &now stored.HypervisorPID = nil + stored.PendingStandbyCompression = nil + if compressionPolicy != nil { + stored.PendingStandbyCompression = &PendingStandbyCompression{ + Policy: *cloneCompressionConfig(compressionPolicy), + NotBefore: m.nowUTC().Add(compressionDelay), + } + } meta = &metadata{StoredMetadata: *stored} if err := m.saveMetadata(meta); err != nil { diff --git a/lib/instances/types.go b/lib/instances/types.go index 0df715fe..f429da93 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -139,6 +139,10 @@ type StoredMetadata struct { // Snapshot policy defaults for this instance. SnapshotPolicy *SnapshotPolicy + // Pending standby compression plan for the latest standby snapshot. + // Persisted so server restarts can recover delayed or interrupted jobs. + PendingStandbyCompression *PendingStandbyCompression + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -291,6 +295,13 @@ type SnapshotPolicy struct { StandbyCompressionDelay *time.Duration } +// PendingStandbyCompression stores the effective standby compression plan that +// should be recovered after a server restart. +type PendingStandbyCompression struct { + Policy snapshot.SnapshotCompressionConfig + NotBefore time.Time +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string From 9f3a5ceae2d384f2a8882b644367654becdd9f75 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 11:23:18 -0400 Subject: [PATCH 4/6] Serialize standby compression recovery tests --- lib/instances/snapshot_compression_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 8610eb20..f654a5f3 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -455,8 +455,6 @@ func TestStartCompressionJobDelayedCancellationRecordsSkipped(t *testing.T) { } func TestRecoverPendingStandbyCompressionJobsRequeuesDelayedJob(t *testing.T) { - t.Parallel() - mgr, _ := setupTestManager(t) now := time.Date(2026, time.April, 6, 12, 0, 0, 0, time.UTC) mgr.now = func() time.Time { return now } @@ -521,8 +519,6 @@ func TestRecoverPendingStandbyCompressionJobsRequeuesDelayedJob(t *testing.T) { } func TestRecoverPendingStandbyCompressionJobsStartsImmediateCompression(t *testing.T) { - t.Parallel() - mgr, _ := setupTestManager(t) now := time.Date(2026, time.April, 6, 12, 5, 0, 0, time.UTC) mgr.now = func() time.Time { return now } @@ -569,8 +565,6 @@ func TestRecoverPendingStandbyCompressionJobsStartsImmediateCompression(t *testi } func TestRecoverPendingStandbyCompressionJobsClearsStalePlans(t *testing.T) { - t.Parallel() - tests := []struct { name string prepare func(t *testing.T, mgr *manager, instanceID string, now time.Time) @@ -628,8 +622,6 @@ func TestRecoverPendingStandbyCompressionJobsClearsStalePlans(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Parallel() - mgr, _ := setupTestManager(t) now := time.Date(2026, time.April, 6, 12, 10, 0, 0, time.UTC) mgr.now = func() time.Time { return now } From c4595667586ea345951dadd46690127c203eaaa9 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 6 Apr 2026 11:53:40 -0400 Subject: [PATCH 5/6] Use lightweight fixtures for standby compression tests --- lib/instances/snapshot_compression_test.go | 29 ++++++++++++-- skills/test-agent/agents/test-agent/NOTES.md | 41 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index f654a5f3..9aee5931 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -405,10 +406,22 @@ func assertCompressionJobCanceled(t *testing.T, mgr *manager, target *compressio }, time.Second, 10*time.Millisecond) } +func newSnapshotCompressionTestManager(t *testing.T) *manager { + t.Helper() + + p := paths.New(t.TempDir()) + return &manager{ + paths: p, + now: time.Now, + compressionJobs: make(map[string]*compressionJob), + stateSubscribers: newSubscribers(), + } +} + func TestStartCompressionJobDelayedCancellationRecordsSkipped(t *testing.T) { t.Parallel() - mgr, _ := setupTestManager(t) + mgr := newSnapshotCompressionTestManager(t) delay := 45 * time.Second timer := newFakeCompressionTimer() mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { @@ -455,7 +468,9 @@ func TestStartCompressionJobDelayedCancellationRecordsSkipped(t *testing.T) { } func TestRecoverPendingStandbyCompressionJobsRequeuesDelayedJob(t *testing.T) { - mgr, _ := setupTestManager(t) + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) now := time.Date(2026, time.April, 6, 12, 0, 0, 0, time.UTC) mgr.now = func() time.Time { return now } @@ -519,7 +534,9 @@ func TestRecoverPendingStandbyCompressionJobsRequeuesDelayedJob(t *testing.T) { } func TestRecoverPendingStandbyCompressionJobsStartsImmediateCompression(t *testing.T) { - mgr, _ := setupTestManager(t) + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) now := time.Date(2026, time.April, 6, 12, 5, 0, 0, time.UTC) mgr.now = func() time.Time { return now } @@ -565,6 +582,8 @@ func TestRecoverPendingStandbyCompressionJobsStartsImmediateCompression(t *testi } func TestRecoverPendingStandbyCompressionJobsClearsStalePlans(t *testing.T) { + t.Parallel() + tests := []struct { name string prepare func(t *testing.T, mgr *manager, instanceID string, now time.Time) @@ -622,7 +641,9 @@ func TestRecoverPendingStandbyCompressionJobsClearsStalePlans(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mgr, _ := setupTestManager(t) + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) now := time.Date(2026, time.April, 6, 12, 10, 0, 0, time.UTC) mgr.now = func() time.Time { return now } instanceID := "recover-stale-" + tt.name diff --git a/skills/test-agent/agents/test-agent/NOTES.md b/skills/test-agent/agents/test-agent/NOTES.md index 200c6905..d8ac20e1 100644 --- a/skills/test-agent/agents/test-agent/NOTES.md +++ b/skills/test-agent/agents/test-agent/NOTES.md @@ -218,6 +218,47 @@ - `TestVolumeMultiAttachReadOnly` - `exec-agent not ready for instance ... within 15s (last state: Initializing)` - `TestVolumeFromArchive` + +## 2026-04-06 - PR #184 standby compression delay branch (`codex/standby-compression-delay`) + +### CI red signature +- Linux `test` job on PR [#184](https://github.com/kernel/hypeman/pull/184) failed while the other checks passed. +- Observed failures from the GitHub Actions log: + - `TestQEMUStandbyRestoreCompressionScenarios` + - `TestQEMUStandbyAndRestore` + - `TestBasicEndToEnd` + - `TestForkCloudHypervisorFromRunningNetwork` +- Failure shapes were integration stalls, not deterministic assertion failures: + - `instance ... did not reach Running within 20s (last state: Initializing)` + - `rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL` + +### Investigation +- Initial stopgap of removing `t.Parallel()` from the new restart-recovery tests was rejected; that was the wrong direction and was not kept. +- Reproduced the branch on `deft-kernel-dev` using the CI-like Linux/root flow with correct prewarm env: + - `go mod download` + - `make oapi-generate` + - `make build` + - `go run ./cmd/test-prewarm` + - `sudo env ... go test -count=1 -tags containers_image_openpgp -timeout=20m ./...` +- Tight loop on the exact CI-failing tests did not reproduce a flake once the command shape matched CI and prewarm settings were correct. + +### Root cause and fix +- The new standby compression recovery tests were unit-style tests but used `setupTestManager`, which pulls in much heavier integration-style manager setup than needed. +- That extra setup was unnecessary for these tests and added avoidable load to an already heavy `lib/instances` package. +- Fix: + - Added a lightweight `newSnapshotCompressionTestManager` helper in `lib/instances/snapshot_compression_test.go` + - Moved the new delayed-job and restart-recovery tests to that lightweight fixture + - Restored `t.Parallel()` on the new recovery tests and subtests +- This keeps coverage and parallelism intact while removing needless setup cost. + +### Validation +- Targeted stress loop after the fixture change: + - `go test -count=20 -run '^(TestRecoverPendingStandbyCompressionJobs|TestStartCompressionJobDelayedCancellationRecordsSkipped)$' ./lib/instances` + - Result: pass +- Deft full fresh-cache CI-like runs after the fix: + - Run 1: pass (`lib/instances` 193.279s) + - Run 2: pass (`lib/instances` 261.633s) + - Run 3: pass (`lib/instances` 173.573s) - `exec-agent not ready for instance ... within 15s (last state: Initializing)` ### Additional flakes reproduced during Deft full-suite verification From 6ef8a8aeeefaf2dd75edbd001251ed2aaa7d3868 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 7 Apr 2026 10:55:20 -0400 Subject: [PATCH 6/6] Harden network and restore test retries --- .../compression_integration_linux_test.go | 50 ++++++++++--- lib/instances/network_test.go | 14 ++-- lib/network/bridge_linux.go | 72 +++++++------------ skills/test-agent/agents/test-agent/NOTES.md | 35 +++++++++ 4 files changed, 108 insertions(+), 63 deletions(-) diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index f58b5b1f..9c5b0644 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -258,30 +258,64 @@ func waitForRunningAndExecReady(t *testing.T, ctx context.Context, mgr *manager, require.NoError(t, waitHypervisorUp(ctx, inst)) } require.NoError(t, waitForExecAgent(ctx, mgr, instanceID, 30*time.Second)) + waitForGuestExecReady(t, ctx, inst) return inst } -func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { +func waitForGuestExecReady(t *testing.T, ctx context.Context, inst *Instance) { t.Helper() - execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(compressionGuestExecTimeout)) - defer cancel() - output, exitCode, err := execCommand(execCtx, inst, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) + require.Eventually(t, func() bool { + execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(5*time.Second)) + defer cancel() + + output, exitCode, err := execCommand(execCtx, inst, "true") + return err == nil && exitCode == 0 && output == "" + }, integrationTestTimeout(15*time.Second), 500*time.Millisecond, "guest exec should succeed after restore") +} + +func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { + t.Helper() + output, exitCode, err := execCommandWithRetry(ctx, inst, compressionGuestExecTimeout, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) require.NoError(t, err) require.Equal(t, 0, exitCode, output) } func assertGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, expected string) { t.Helper() - execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(compressionGuestExecTimeout)) - defer cancel() - - output, exitCode, err := execCommand(execCtx, inst, "cat", path) + output, exitCode, err := execCommandWithRetry(ctx, inst, compressionGuestExecTimeout, "cat", path) require.NoError(t, err) require.Equal(t, 0, exitCode, output) assert.Equal(t, expected, output) } +func execCommandWithRetry(ctx context.Context, inst *Instance, timeout time.Duration, command ...string) (string, int, error) { + deadline := time.Now().Add(integrationTestTimeout(timeout)) + var lastOutput string + var lastExitCode int + var lastErr error + + for { + execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(5*time.Second)) + output, exitCode, err := execCommand(execCtx, inst, command...) + cancel() + + if err == nil { + return output, exitCode, nil + } + + lastOutput = output + lastExitCode = exitCode + lastErr = err + + if time.Now().After(deadline) { + return lastOutput, lastExitCode, lastErr + } + + time.Sleep(500 * time.Millisecond) + } +} + func waitForCompressionJobStart(t *testing.T, mgr *manager, key string, timeout time.Duration) { t.Helper() deadline := time.Now().Add(timeout) diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index 60eaf4e3..9122a8f7 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -245,42 +245,42 @@ func TestDockerForwardChainRestored(t *testing.T) { require.NoError(t, manager.networkManager.Initialize(ctx, nil)) // Check if DOCKER-FORWARD chain exists (Docker must be running on host). - checkChain := exec.Command("iptables", "-L", "DOCKER-FORWARD", "-n") + checkChain := exec.Command("iptables", "-w", "5", "-L", "DOCKER-FORWARD", "-n") if checkChain.Run() != nil { t.Skip("DOCKER-FORWARD chain not present (Docker not running), skipping") } // Verify jump currently exists. - checkJump := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + checkJump := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") require.NoError(t, checkJump.Run(), "DOCKER-FORWARD jump should exist before test") // Safety net: restore the jump if the test fails or aborts after we delete it, // so we don't leave the host's Docker networking broken. t.Cleanup(func() { - check := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + check := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") if check.Run() != nil { - restore := exec.Command("iptables", "-A", "FORWARD", "-j", "DOCKER-FORWARD") + restore := exec.Command("iptables", "-w", "5", "-A", "FORWARD", "-j", "DOCKER-FORWARD") _ = restore.Run() } }) // Simulate the hypervisor flush: remove every jump. for { - delJump := exec.Command("iptables", "-D", "FORWARD", "-j", "DOCKER-FORWARD") + delJump := exec.Command("iptables", "-w", "5", "-D", "FORWARD", "-j", "DOCKER-FORWARD") if err := delJump.Run(); err != nil { break } } // Confirm it's gone. - checkGone := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + checkGone := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") require.Error(t, checkGone.Run(), "DOCKER-FORWARD jump should be gone after delete") // Re-initialize network — this should restore the jump. require.NoError(t, manager.networkManager.Initialize(ctx, nil)) // Verify jump is restored. - checkRestored := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + checkRestored := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") require.NoError(t, checkRestored.Run(), "ensureDockerForwardJump should have restored the DOCKER-FORWARD jump") } diff --git a/lib/network/bridge_linux.go b/lib/network/bridge_linux.go index 62831b97..06b31b7a 100644 --- a/lib/network/bridge_linux.go +++ b/lib/network/bridge_linux.go @@ -20,6 +20,18 @@ import ( ) const netlinkDumpRetryCount = 3 +const iptablesWaitSeconds = "5" + +func newIPTablesCommand(args ...string) *exec.Cmd { + fullArgs := make([]string, 0, len(args)+2) + fullArgs = append(fullArgs, "-w", iptablesWaitSeconds) + fullArgs = append(fullArgs, args...) + cmd := exec.Command("iptables", fullArgs...) + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + return cmd +} func listBridgeAddrsWithRetry(link netlink.Link) ([]netlink.Addr, error) { var err error @@ -288,13 +300,10 @@ func (m *manager) setupIPTablesRules(ctx context.Context, subnet, bridgeName str // ensureNATRule ensures the MASQUERADE rule exists with correct uplink func (m *manager) ensureNATRule(subnet, uplink, comment string) (string, error) { // Check if rule exists with correct subnet and uplink - checkCmd := exec.Command("iptables", "-t", "nat", "-C", "POSTROUTING", + checkCmd := newIPTablesCommand("-t", "nat", "-C", "POSTROUTING", "-s", subnet, "-o", uplink, "-m", "comment", "--comment", comment, "-j", "MASQUERADE") - checkCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } if checkCmd.Run() == nil { return "existing", nil } @@ -303,13 +312,10 @@ func (m *manager) ensureNATRule(subnet, uplink, comment string) (string, error) m.deleteNATRuleByComment(comment) // Add rule with comment - addCmd := exec.Command("iptables", "-t", "nat", "-A", "POSTROUTING", + addCmd := newIPTablesCommand("-t", "nat", "-A", "POSTROUTING", "-s", subnet, "-o", uplink, "-m", "comment", "--comment", comment, "-j", "MASQUERADE") - addCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } if err := addCmd.Run(); err != nil { return "", fmt.Errorf("add masquerade rule: %w", err) } @@ -327,10 +333,7 @@ func (m *manager) ruleComment(base string) string { // deleteNATRuleByComment deletes any NAT POSTROUTING rule containing our comment func (m *manager) deleteNATRuleByComment(comment string) { // List NAT POSTROUTING rules - cmd := exec.Command("iptables", "-t", "nat", "-L", "POSTROUTING", "--line-numbers", "-n") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-t", "nat", "-L", "POSTROUTING", "--line-numbers", "-n") output, err := cmd.Output() if err != nil { return @@ -350,10 +353,7 @@ func (m *manager) deleteNATRuleByComment(comment string) { // Delete in reverse order for i := len(ruleNums) - 1; i >= 0; i-- { - delCmd := exec.Command("iptables", "-t", "nat", "-D", "POSTROUTING", ruleNums[i]) - delCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + delCmd := newIPTablesCommand("-t", "nat", "-D", "POSTROUTING", ruleNums[i]) delCmd.Run() // ignore error } } @@ -369,14 +369,11 @@ func (m *manager) ensureForwardRule(inIface, outIface, ctstate, comment string, m.deleteForwardRuleByComment(comment) // Insert at specified position with comment - addCmd := exec.Command("iptables", "-I", "FORWARD", fmt.Sprintf("%d", position), + addCmd := newIPTablesCommand("-I", "FORWARD", fmt.Sprintf("%d", position), "-i", inIface, "-o", outIface, "-m", "conntrack", "--ctstate", ctstate, "-m", "comment", "--comment", comment, "-j", "ACCEPT") - addCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } if err := addCmd.Run(); err != nil { return "", fmt.Errorf("insert forward rule: %w", err) } @@ -386,10 +383,7 @@ func (m *manager) ensureForwardRule(inIface, outIface, ctstate, comment string, // isForwardRuleCorrect checks if our rule exists at the expected position with correct interfaces func (m *manager) isForwardRuleCorrect(inIface, outIface, comment string, position int) bool { // List FORWARD chain with line numbers - cmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n", "-v") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-L", "FORWARD", "--line-numbers", "-n", "-v") output, err := cmd.Output() if err != nil { return false @@ -417,10 +411,7 @@ func (m *manager) isForwardRuleCorrect(inIface, outIface, comment string, positi // deleteForwardRuleByComment deletes any FORWARD rule containing our comment func (m *manager) deleteForwardRuleByComment(comment string) { // List FORWARD rules - cmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-L", "FORWARD", "--line-numbers", "-n") output, err := cmd.Output() if err != nil { return @@ -440,10 +431,7 @@ func (m *manager) deleteForwardRuleByComment(comment string) { // Delete in reverse order for i := len(ruleNums) - 1; i >= 0; i-- { - delCmd := exec.Command("iptables", "-D", "FORWARD", ruleNums[i]) - delCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + delCmd := newIPTablesCommand("-D", "FORWARD", ruleNums[i]) delCmd.Run() // ignore error } } @@ -460,19 +448,13 @@ func (m *manager) ensureDockerForwardJump(ctx context.Context) { log := logger.FromContext(ctx) // Check if DOCKER-FORWARD chain exists (Docker is installed and configured) - checkChain := exec.Command("iptables", "-L", "DOCKER-FORWARD", "-n") - checkChain.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + checkChain := newIPTablesCommand("-L", "DOCKER-FORWARD", "-n") if checkChain.Run() != nil { return // Chain doesn't exist — Docker not installed or not configured } // Check if jump already exists in FORWARD - checkJump := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") - checkJump.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + checkJump := newIPTablesCommand("-C", "FORWARD", "-j", "DOCKER-FORWARD") if checkJump.Run() == nil { return // Jump already present } @@ -481,10 +463,7 @@ func (m *manager) ensureDockerForwardJump(ctx context.Context) { // Insert right after hypeman's last rule so the jump is evaluated before any // explicit DROP/REJECT rules that an external firewall tool may have added. insertPos := m.lastHypemanForwardRulePosition() + 1 - addJump := exec.Command("iptables", "-I", "FORWARD", fmt.Sprintf("%d", insertPos), "-j", "DOCKER-FORWARD") - addJump.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + addJump := newIPTablesCommand("-I", "FORWARD", fmt.Sprintf("%d", insertPos), "-j", "DOCKER-FORWARD") if err := addJump.Run(); err != nil { log.WarnContext(ctx, "failed to restore Docker FORWARD chain jump", "error", err) return @@ -496,10 +475,7 @@ func (m *manager) ensureDockerForwardJump(ctx context.Context) { // lastHypemanForwardRulePosition returns the line number of the last hypeman-managed // rule in the FORWARD chain, or 0 if none are found. func (m *manager) lastHypemanForwardRulePosition() int { - cmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n", "-v") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-L", "FORWARD", "--line-numbers", "-n", "-v") output, err := cmd.Output() if err != nil { return 0 diff --git a/skills/test-agent/agents/test-agent/NOTES.md b/skills/test-agent/agents/test-agent/NOTES.md index d8ac20e1..92429048 100644 --- a/skills/test-agent/agents/test-agent/NOTES.md +++ b/skills/test-agent/agents/test-agent/NOTES.md @@ -259,6 +259,41 @@ - Run 1: pass (`lib/instances` 193.279s) - Run 2: pass (`lib/instances` 261.633s) - Run 3: pass (`lib/instances` 173.573s) + +## 2026-04-07 - PR #184 follow-up CI round on `codex/standby-compression-delay` + +### Initial CI red signature +- Linux `test` job failed on `TestDockerForwardChainRestored`. +- Failure: + - `ensureDockerForwardJump should have restored the DOCKER-FORWARD jump` + - raw `iptables -C FORWARD -j DOCKER-FORWARD` exited non-zero in the test after re-initialization. + +### Root cause and fix +- The Docker-forward recovery path and the test both used plain `iptables` invocations with no wait for the xtables lock. +- Under parallel CI activity, a transient lock holder can cause checks/deletes/inserts to fail immediately and make the test observe a missing rule even though the recovery logic is otherwise correct. +- Fix: + - Added a small `newIPTablesCommand` helper in `lib/network/bridge_linux.go` that uses `iptables -w 5 ...` with the existing `CAP_NET_ADMIN` setup. + - Switched the bridge NAT/FORWARD rule management and `ensureDockerForwardJump` commands to that helper. + - Updated `TestDockerForwardChainRestored` in `lib/instances/network_test.go` to use `iptables -w 5` for its direct host-global mutations/checks. + +### Secondary flake surfaced during Deft reruns +- A subsequent Deft full-suite rerun exposed a post-restore guest exec race in `TestCloudHypervisorStandbyRestoreCompressionScenarios`: + - `receive response (stdout=0, stderr=0): rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL` +- The compression integration harness was only waiting for the exec agent socket and then issuing marker reads/writes immediately after restore. +- Fix: + - Added a no-op post-restore guest exec readiness probe in `waitForRunningAndExecReady`. + - Added a small retry wrapper for the compression integration test’s guest marker read/write commands so transient post-restore transport resets do not fail the scenario immediately. + +### Validation +- Deft targeted loop: + - `go test -count=20 -run '^TestDockerForwardChainRestored$' -v ./lib/instances` + - Result: pass +- Deft targeted loop: + - `go test -count=10 -run '^TestCloudHypervisorStandbyRestoreCompressionScenarios$' -tags containers_image_openpgp -timeout=30m ./lib/instances` + - Result: pass +- Local sanity: + - `go test ./lib/instances -count=1` + - Result: pass (`117.538s`) - `exec-agent not ready for instance ... within 15s (last state: Initializing)` ### Additional flakes reproduced during Deft full-suite verification