diff --git a/src/control/cmd/dmg/storage_test.go b/src/control/cmd/dmg/storage_test.go index 124b46e984a..2f0fdb50588 100644 --- a/src/control/cmd/dmg/storage_test.go +++ b/src/control/cmd/dmg/storage_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -154,6 +155,24 @@ func TestStorageCommands(t *testing.T) { printRequest(t, nvmeAddDeviceReq().WithStorageTierIndex(0)), nil, }, + { + "Format with replace; no hosts in hostlist", + "storage format --replace", + "", + errors.New("expects a single host"), + }, + { + "Format with replace; multiple hosts in hostlist", + "storage format --replace -l foo[1,2].com", + "", + errors.New("expects a single host"), + }, + { + "Format with replace and force", + "storage format --replace --force", + "", + errors.New("may not be mixed with --force"), + }, { "Nonexistent subcommand", "storage quack", diff --git a/src/control/server/ctl_firmware_test.go b/src/control/server/ctl_firmware_test.go index cab959063ec..7cda1fcfe28 100644 --- a/src/control/server/ctl_firmware_test.go +++ b/src/control/server/ctl_firmware_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -15,10 +16,8 @@ import ( "github.com/daos-stack/daos/src/control/common/proto/convert" ctlpb "github.com/daos-stack/daos/src/control/common/proto/ctl" "github.com/daos-stack/daos/src/control/common/test" - "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" "github.com/daos-stack/daos/src/control/server/config" - "github.com/daos-stack/daos/src/control/server/engine" "github.com/daos-stack/daos/src/control/server/storage" "github.com/daos-stack/daos/src/control/server/storage/bdev" "github.com/daos-stack/daos/src/control/server/storage/scm" @@ -803,14 +802,10 @@ func TestCtlSvc_FirmwareUpdate(t *testing.T) { cfg := config.DefaultServer() cs := mockControlService(t, log, cfg, tc.bmbc, tc.smbc, nil) for i := 0; i < 2; i++ { - rCfg := new(engine.TestRunnerConfig) - rCfg.Running.Store(tc.enginesRunning) - runner := engine.NewTestRunner(rCfg, engine.MockConfig()) - instance := NewEngineInstance(log, nil, nil, runner, nil) - if !tc.noRankEngines { - instance._superblock = &Superblock{} - instance._superblock.ValidRank = true - instance._superblock.Rank = ranklist.NewRankPtr(uint32(i)) + instance := NewEngineInstance(log, nil, nil, nil, nil) + setupTestEngine(t, instance, uint32(i), !tc.enginesRunning) + if tc.noRankEngines { + instance._superblock = nil } if err := cs.harness.AddInstance(instance); err != nil { t.Fatal(err) diff --git a/src/control/server/ctl_ranks_rpc.go b/src/control/server/ctl_ranks_rpc.go index 5fcb092f3ea..3907effb794 100644 --- a/src/control/server/ctl_ranks_rpc.go +++ b/src/control/server/ctl_ranks_rpc.go @@ -29,12 +29,14 @@ const ( instanceUpdateDelay = 500 * time.Millisecond ) +type pollValidateFn func(Engine) bool + // pollInstanceState waits for either context to be cancelled/timeout or for the // provided validate function to return true for each of the provided instances. // // Returns true if all instances return true from the validate function, false // if context is cancelled before. -func pollInstanceState(ctx context.Context, instances []Engine, validate func(Engine) bool) error { +func pollInstanceState(ctx context.Context, instances []Engine, validate pollValidateFn) error { ready := make(chan struct{}) go func() { for { diff --git a/src/control/server/ctl_ranks_rpc_test.go b/src/control/server/ctl_ranks_rpc_test.go index 8057325710f..8be48353fb7 100644 --- a/src/control/server/ctl_ranks_rpc_test.go +++ b/src/control/server/ctl_ranks_rpc_test.go @@ -76,17 +76,6 @@ func checkUnorderedRankResults(t *testing.T, expResults, gotResults []*sharedpb. } } -func setupTestEngine(t *testing.T, ei *EngineInstance, rank uint32, stopped ...bool) { - ei._superblock.Rank = ranklist.NewRankPtr(rank) - - trc := &engine.TestRunnerConfig{} - if len(stopped) == 0 || !stopped[0] { - trc.Running.SetTrue() - ei.ready.SetTrue() - } - ei.runner = engine.NewTestRunner(trc, engine.MockConfig()) -} - func TestServer_CtlSvc_PrepShutdownRanks(t *testing.T) { for name, tc := range map[string]struct { missingSB bool diff --git a/src/control/server/ctl_storage_rpc.go b/src/control/server/ctl_storage_rpc.go index 4efefd6c32f..14eda4d4fd4 100644 --- a/src/control/server/ctl_storage_rpc.go +++ b/src/control/server/ctl_storage_rpc.go @@ -1,6 +1,6 @@ // // (C) Copyright 2019-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent @@ -856,10 +856,10 @@ type formatScmReq struct { } func formatScm(ctx context.Context, req formatScmReq, resp *ctlpb.StorageFormatResp) (map[int]string, map[int]bool, error) { - needFormat := make(map[int]bool) + needScmFormat := make(map[int]bool) emptyTmpfs := make(map[int]bool) scmCfgs := make(map[int]*storage.TierConfig) - allNeedFormat := true + allNeedScmFormat := true for idx, ei := range req.instances { needs, err := ei.GetStorage().ScmNeedsFormat() @@ -867,9 +867,9 @@ func formatScm(ctx context.Context, req formatScmReq, resp *ctlpb.StorageFormatR return nil, nil, errors.Wrap(err, "detecting if SCM format is needed") } if needs { - needFormat[idx] = true + needScmFormat[idx] = true } else { - allNeedFormat = false + allNeedScmFormat = false } scmCfg, err := ei.GetStorage().GetScmConfig() @@ -882,19 +882,20 @@ func formatScm(ctx context.Context, req formatScmReq, resp *ctlpb.StorageFormatR if scmCfg.Class == storage.ClassRam && !needs { info, err := ei.GetStorage().GetScmUsage() if err != nil { - return nil, nil, errors.Wrapf(err, "failed to check SCM usage for instance %d", idx) + return nil, nil, errors.Wrapf(err, + "failed to check SCM usage for instance %d", idx) } emptyTmpfs[idx] = info.TotalBytes-info.AvailBytes == 0 } } - if req.replace && len(needFormat) == 0 { + if req.replace && len(needScmFormat) == 0 { // Only valid if at least one engine requires format. - return nil, nil, errors.New("format replace option only valid if at " + - "least one engine requires format but no engines need format") + return nil, nil, errors.New("format replace option only valid if at least one " + + "engine requires scm-format but currently no engines need scm-format") } - if allNeedFormat { + if allNeedScmFormat { // Check available RAM is sufficient before formatting SCM on engines. if err := checkTmpfsMem(req.log, scmCfgs, req.getSysMemInfo); err != nil { return nil, nil, err @@ -907,7 +908,7 @@ func formatScm(ctx context.Context, req formatScmReq, resp *ctlpb.StorageFormatR formatting := 0 for idx, ei := range req.instances { - if needFormat[idx] || req.reformat { + if needScmFormat[idx] || req.reformat { formatting++ go func(e Engine) { scmChan <- e.StorageFormatSCM(ctx, req.reformat) diff --git a/src/control/server/ctl_storage_rpc_test.go b/src/control/server/ctl_storage_rpc_test.go index cde9c4387f8..e9f6ca70432 100644 --- a/src/control/server/ctl_storage_rpc_test.go +++ b/src/control/server/ctl_storage_rpc_test.go @@ -2360,7 +2360,7 @@ func TestServer_CtlSvc_StorageFormat(t *testing.T) { }, }, }, - expErr: errors.New("only valid if at least one engine requires format"), + expErr: errors.New("only valid if at least one engine requires scm-format"), expResp: &ctlpb.StorageFormatResp{ Crets: []*ctlpb.NvmeControllerResult{ { diff --git a/src/control/server/ctl_svc_test.go b/src/control/server/ctl_svc_test.go index 9c6bd03f043..09d7b96378b 100644 --- a/src/control/server/ctl_svc_test.go +++ b/src/control/server/ctl_svc_test.go @@ -12,7 +12,6 @@ import ( "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/events" - "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" "github.com/daos-stack/daos/src/control/provider/system" "github.com/daos-stack/daos/src/control/server/config" @@ -72,19 +71,10 @@ func newMockControlServiceFromBackends(t *testing.T, log logging.Logger, cfg *co } for idx, ec := range cfg.Engines { - trc := new(engine.TestRunnerConfig) - if started[idx] { - trc.Running.SetTrue() - } - runner := engine.NewTestRunner(trc, ec) storProv := storage.MockProvider(log, 0, &ec.Storage, syp, sp, bp, nil) - - ei := NewEngineInstance(log, storProv, nil, runner, nil) - ei.setSuperblock(&Superblock{ - Rank: ranklist.NewRankPtr(uint32(idx)), - }) + ei := NewEngineInstance(log, storProv, nil, nil, nil) + setupTestEngineWithConfig(t, ei, uint32(idx), ec, !started[idx]) if started[idx] { - ei.ready.SetTrue() ei.setDrpcSocket("/dontcare") } if err := cs.harness.AddInstance(ei); err != nil { diff --git a/src/control/server/instance.go b/src/control/server/instance.go index c763ae1b7a9..02be3b4151a 100644 --- a/src/control/server/instance.go +++ b/src/control/server/instance.go @@ -1,6 +1,6 @@ // // (C) Copyright 2019-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -216,9 +216,24 @@ func (ei *EngineInstance) determineRank(ctx context.Context, ready *srvpb.Notify Replace: ei.replaceRank.Load(), } + // Reset replaceRank state for instance after joinSystem() has been attempted. + defer ei.replaceRank.SetFalse() + resp, err := ei.joinSystem(ctx, joinReq) if err != nil { ei.log.Errorf("join failed: %s", err) + + // If this is a replace operation and join failed, clean up the formatted storage to + // prevent leaving the rank in a formatted state. This prevents the engine + // inadvertently being joined later with a new rank. + if ei.replaceRank.Load() { + ei.log.Infof("cleaning up after join failure during replace") + if cleanupErr := ei.cleanupFailedJoinReplace(ctx); cleanupErr != nil { + ei.log.Errorf("failed to cleanup after join failure: %v", cleanupErr) + // Don't override the original join error + } + } + return ranklist.NilRank, false, 0, err } switch resp.State { @@ -237,9 +252,6 @@ func (ei *EngineInstance) determineRank(ctx context.Context, ready *srvpb.Notify } r = ranklist.Rank(resp.Rank) - // Reset replaceRank state for instance after joinSystem() has returned. - ei.replaceRank.SetFalse() - if !superblock.ValidRank || ready.Uri != superblock.URI { ei.log.Noticef("updating rank %d URI to %s", resp.Rank, ready.Uri) superblock.Rank = new(ranklist.Rank) diff --git a/src/control/server/instance_drpc_test.go b/src/control/server/instance_drpc_test.go index ecfd395ed68..8fec113f0a6 100644 --- a/src/control/server/instance_drpc_test.go +++ b/src/control/server/instance_drpc_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -26,7 +26,6 @@ import ( "github.com/daos-stack/daos/src/control/lib/daos" . "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" - "github.com/daos-stack/daos/src/control/server/engine" . "github.com/daos-stack/daos/src/control/system" ) @@ -90,10 +89,8 @@ func TestEngineInstance_CallDrpc(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) defer test.ShowBufferOnFailure(t, buf) - trc := engine.TestRunnerConfig{} - trc.Running.Store(!tc.notStarted) - runner := engine.NewTestRunner(&trc, engine.MockConfig()) - instance := NewEngineInstance(log, nil, nil, runner, nil) + instance := NewEngineInstance(log, nil, nil, nil, nil) + setupTestEngine(t, instance, 0, tc.notStarted) instance.ready.Store(!tc.notReady) if !tc.noSocket { @@ -188,11 +185,8 @@ func TestEngineInstance_CallDrpc_Parallel(t *testing.T) { }(t) t.Log("setting up engine...") - trc := engine.TestRunnerConfig{} - trc.Running.Store(true) - runner := engine.NewTestRunner(&trc, engine.MockConfig()) - instance := NewEngineInstance(log, nil, nil, runner, nil) - instance.ready.Store(true) + instance := NewEngineInstance(log, nil, nil, nil, nil) + setupTestEngine(t, instance, 0) instance.getDrpcClientFn = func(s string) drpc.DomainSocketClient { t.Log("fetching drpc client") diff --git a/src/control/server/instance_storage.go b/src/control/server/instance_storage.go index ab8f8adcb98..206b0648392 100644 --- a/src/control/server/instance_storage.go +++ b/src/control/server/instance_storage.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "os" + "syscall" "github.com/dustin/go-humanize" "github.com/pkg/errors" @@ -76,6 +77,63 @@ func (ei *EngineInstance) NotifyStorageReady(replaceRank bool) { }() } +func (ei *EngineInstance) clearFormat(ctx context.Context, stopEngineFn func(context.Context, *EngineInstance) error) error { + idx := ei.Index() + ei.log.Infof("instance %d: cleaning up after join failure during replace", idx) + + storageProv := ei.GetStorage() + + // Get SCM config to access mount point and class + scmCfg, err := storageProv.GetScmConfig() + if err != nil { + return errors.Wrap(err, "failed to get SCM config") + } + + if scmCfg == nil { + ei.log.Debugf("instance %d: no SCM config, nothing to clean", idx) + return nil + } + + if ei.IsStarted() { + ei.log.Infof("instance %d: stopping engine before cleanup", idx) + if err := stopEngineFn(ctx, ei); err != nil { + return err + } + ei.log.Debugf("instance %d: engine stopped successfully", idx) + } + + // On RAM-based SCM (tmpfs) unmount here unnecessary as will be done on engine exit + + // Removing superblock prevents subsequent join without reformat. + if err := ei.RemoveSuperblock(); err != nil { + return err + } + + ei.log.Infof("instance %d: cleanup after join failure complete", idx) + return nil +} + +// Production implementation of stopEngineFn. +func stopEngine(ctx context.Context, ei *EngineInstance) error { + if err := ei.Stop(syscall.SIGKILL); err != nil { + return errors.Wrap(err, "failed to stop engine") + } + + pollFn := func(e Engine) bool { return !e.IsStarted() } + if err := pollInstanceState(ctx, []Engine{ei}, pollFn); err != nil { + return errors.Wrap(err, "waiting for engine to stop") + } + + return nil +} + +// cleanupFailedJoinReplace cleans up storage after a join failure during replace operation. +// This is called when format succeeded but the join to the system failed, leaving +// the storage in a partially initialized state. +func (ei *EngineInstance) cleanupFailedJoinReplace(ctx context.Context) error { + return ei.clearFormat(ctx, stopEngine) +} + func (ei *EngineInstance) checkScmNeedFormat() (bool, error) { msgIdx := fmt.Sprintf("instance %d", ei.Index()) diff --git a/src/control/server/instance_storage_test.go b/src/control/server/instance_storage_test.go index 3b4ebf82ae0..99f4d611709 100644 --- a/src/control/server/instance_storage_test.go +++ b/src/control/server/instance_storage_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -11,6 +12,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -23,6 +25,7 @@ import ( "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" "github.com/daos-stack/daos/src/control/provider/system" + sysprov "github.com/daos-stack/daos/src/control/provider/system" "github.com/daos-stack/daos/src/control/server/engine" "github.com/daos-stack/daos/src/control/server/storage" "github.com/daos-stack/daos/src/control/server/storage/scm" @@ -558,3 +561,134 @@ func TestIOEngineInstance_awaitStorageReady(t *testing.T) { }) } } + +func TestEngineInstance_clearFormat(t *testing.T) { + defStopEngine := func(_ context.Context, _ *EngineInstance) error { + return errors.New("stop failed") + } + + for name, tc := range map[string]struct { + scmClass storage.Class + getSCMErr error + unmountErr error + engineStarted bool + stopEngine func(context.Context, *EngineInstance) error + expErr error + expSBCreated bool + expSBRemoved bool + }{ + "no SCM config": { + scmClass: "", + expErr: errors.New("expected exactly 1 SCM tier"), + }, + "get SCM config fails": { + getSCMErr: errors.New("mock config error"), + expErr: errors.New("failed to get SCM config"), + }, + "RAM class not started": { + scmClass: storage.ClassRam, + expErr: nil, + expSBRemoved: true, + }, + "DCPM class not started": { + scmClass: storage.ClassDcpm, + expErr: nil, + }, + "RAM class engine started": { + scmClass: storage.ClassRam, + engineStarted: true, + stopEngine: func(_ context.Context, _ *EngineInstance) error { + return nil + }, + expErr: nil, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + testDir, cleanup := test.CreateTestDir(t) + defer cleanup() + + mnt := "/mnt/daos0" + testMountPoint := filepath.Join(testDir, mnt) + if err := os.MkdirAll(testMountPoint, 0777); err != nil { + t.Fatal(err) + } + + storageCfg := storage.Config{ + Tiers: storage.TierConfigs{}, + } + + if tc.scmClass != "" { + storageCfg.Tiers = append(storageCfg.Tiers, &storage.TierConfig{ + Class: tc.scmClass, + Scm: storage.ScmConfig{ + MountPoint: mnt, //testMountPoint, + }, + }) + if tc.scmClass == storage.ClassRam { + storageCfg.ControlMetadata = storage.ControlMetadata{ + Path: "control_meta", + } + } + } + + sysCfg := &sysprov.MockSysConfig{} + sysProv := sysprov.NewMockSysProvider(log, sysCfg) + scmProv := scm.DefaultMockProvider(log) + metaProv := &storage.MockMetadataProvider{ + UnmountErr: tc.unmountErr, + } + + ec := engine.MockConfig().WithStorage(storageCfg.Tiers...) + storProv := storage.MockProvider(log, 0, &storageCfg, sysProv, scmProv, nil, + metaProv) + + ei := NewEngineInstance(log, storProv, nil, nil, nil) + ei.fsRoot = testDir + setupTestEngineWithConfig(t, ei, uint32(0), ec, !tc.engineStarted) + + testSBPath := ei.superblockPath() + t.Logf("SB path: %s", testSBPath) + if err := os.MkdirAll(filepath.Dir(testSBPath), 0777); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(testSBPath); err == nil { + t.Fatalf("unexpected superblock exists (%s)", err) + } else if !os.IsNotExist(err) { + t.Fatalf("unexpected error %s", err) + } + if len(storageCfg.Tiers) != 0 { + if err := ei.WriteSuperblock(); err != nil { + //storage.FormatControlMetadata([]uint{0}); err != nil { + t.Fatal(err) + } + } + if _, err := os.Stat(testSBPath); tc.expSBCreated && err != nil { + t.Fatalf("superblock missing (%s)", err) + } + + if tc.stopEngine == nil { + tc.stopEngine = defStopEngine + } + + test.CmpErr(t, tc.expErr, ei.clearFormat(test.Context(t), tc.stopEngine)) + + // Check log output for unexpected unmount call + logOutput := buf.String() + unmountCalled := strings.Contains(logOutput, "unmounting tmpfs") + if unmountCalled { + t.Error("unexpected unmount call") + } + + if !tc.expSBCreated { + return + } + + _, err := os.Stat(testSBPath) + test.AssertEqual(t, tc.expSBRemoved, os.IsNotExist(err), "is superblock removed") + }) + } +} diff --git a/src/control/server/server_utils_test.go b/src/control/server/server_utils_test.go index e9de4583ba0..7954d9df862 100644 --- a/src/control/server/server_utils_test.go +++ b/src/control/server/server_utils_test.go @@ -2104,22 +2104,6 @@ const ( testProcessingDelay = 100 * time.Millisecond ) -func setupAddTestEngine(t *testing.T, log logging.Logger, h *EngineHarness, isRunning bool, ranks ...uint32) { - t.Helper() - - rank := uint32(1) - if len(ranks) != 0 { - rank = ranks[0] - } - - ei := newTestEngine(log, false, storage.MockProvider(log, 0, nil, nil, nil, nil, nil)) - setupTestEngine(t, ei, rank, !isRunning) - - if err := h.AddInstance(ei); err != nil { - t.Fatal(err) - } -} - func TestServer_handleEngineSelfTerminated(t *testing.T) { testRank := ranklist.Rank(1) testIncarnation := uint64(42) diff --git a/src/control/server/util_test.go b/src/control/server/util_test.go index 44f06c4375b..75dcdb8438b 100644 --- a/src/control/server/util_test.go +++ b/src/control/server/util_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2019-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -290,3 +290,48 @@ func newTestMgmtSvcNonReplica(t *testing.T, log logging.Logger) *mgmtSvc { svc.sysdb = raft.MockDatabaseWithAddr(t, log, nil) return svc } + +// setupTestEngineWithConfig configures an EngineInstance with a custom engine config. +// If cfg is nil, uses engine.MockConfig(). If rank is 0, no rank is assigned. +func setupTestEngineWithConfig(t *testing.T, ei *EngineInstance, rank uint32, cfg *engine.Config, stopped ...bool) { + if ei._superblock == nil { + ei._superblock = &Superblock{} + } + ei._superblock.Rank = ranklist.NewRankPtr(rank) + ei._superblock.ValidRank = true + + trc := &engine.TestRunnerConfig{} + if len(stopped) == 0 || !stopped[0] { + trc.Running.SetTrue() + ei.ready.SetTrue() + } + + if cfg == nil { + cfg = engine.MockConfig() + } + ei.runner = engine.NewTestRunner(trc, cfg) +} + +// setupTestEngine configures an EngineInstance for testing with the specified rank and state. +// If stopped is true (or provided), the engine is marked as stopped, otherwise as running. +// If rank is 0, no rank is assigned to the superblock. +func setupTestEngine(t *testing.T, ei *EngineInstance, rank uint32, stopped ...bool) { + setupTestEngineWithConfig(t, ei, rank, nil, stopped...) +} + +// setupAddTestEngine adds test engine to harness with specified ranks and running state. +func setupAddTestEngine(t *testing.T, log logging.Logger, h *EngineHarness, isRunning bool, ranks ...uint32) { + t.Helper() + + rank := uint32(1) + if len(ranks) != 0 { + rank = ranks[0] + } + + ei := newTestEngine(log, false, storage.MockProvider(log, 0, nil, nil, nil, nil, nil)) + setupTestEngine(t, ei, rank, !isRunning) + + if err := h.AddInstance(ei); err != nil { + t.Fatal(err) + } +} diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 691243c0682..d7773870ea0 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -198,9 +198,6 @@ func (m *Membership) joinReplace(req *JoinRequest) (*JoinResponse, error) { return nil, err } - if cm.State == MemberStateAdminExcluded { - return nil, ErrJoinAdminExcluded(cm.UUID, cm.Rank) - } memberToReplace := &Member{} *memberToReplace = *cm diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 7d3e2db8473..e6a599d4c82 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1029,7 +1029,20 @@ func TestSystem_Membership_Join(t *testing.T) { FabricContexts: curMember.PrimaryFabricContexts, FaultDomain: curMember.FaultDomain, }, - expErr: ErrJoinAdminExcluded(adminExcludedMember.UUID, 0), + expResp: &JoinResponse{ + Created: false, + Member: func() *Member { + cm := *adminExcludedMember + cm.Rank = 0 + cm.UUID = curMember.UUID + cm.State = MemberStateJoined + cm.Info = "" + return &cm + }(), + PrevState: MemberStateAdminExcluded, + // Extra map increment because of remove and add operations. + MapVersion: 3, + }, }, "successful replace; different UUID but otherwise identical member": { req: &JoinRequest{