From 9a64c82efcce9aa849d390c57adaff09d50c1503 Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Fri, 6 Feb 2026 14:52:15 +0800 Subject: [PATCH 1/5] feat: use shared cluster to improve test speed --- .../TestSnapshotBackgroundServices.java | 183 +++++++++++------- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 34 ++++ 2 files changed, 143 insertions(+), 74 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java index e5da202421a3..9869ab9647c9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -62,7 +63,6 @@ import org.apache.hadoop.ozone.conf.OMClientConfig; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmSnapshot; -import org.apache.hadoop.ozone.om.OmTestUtil; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SstFilteringService; import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; @@ -80,18 +80,24 @@ import org.apache.ozone.test.tag.Flaky; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; -import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; /** * Tests snapshot background services. */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class TestSnapshotBackgroundServices { - private MiniOzoneHAClusterImpl cluster; - private ObjectStore objectStore; + private static MiniOzoneHAClusterImpl cluster; + private static OzoneClient client; + private static ObjectStore objectStore; + private OzoneBucket ozoneBucket; private String volumeName; private String bucketName; @@ -102,11 +108,10 @@ public class TestSnapshotBackgroundServices { private static final BucketLayout TEST_BUCKET_LAYOUT = BucketLayout.OBJECT_STORE; private static final String SNAPSHOT_NAME_PREFIX = "snapshot-"; private static final String KEY_NAME_PREFIX = "key-"; - private OzoneClient client; - private final AtomicInteger counter = new AtomicInteger(); + private static final AtomicInteger COUNTER = new AtomicInteger(); - @BeforeEach - public void init(TestInfo testInfo) throws Exception { + @BeforeAll + public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String omServiceId = "om-service-test1"; OzoneManagerRatisServerConfig omRatisConf = conf.getObject(OzoneManagerRatisServerConfig.class); @@ -120,66 +125,48 @@ public void init(TestInfo testInfo) throws Exception { conf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, LOG_PURGE_GAP); conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY, 16, StorageUnit.KB); conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY, 16, StorageUnit.KB); - if ("testSSTFilteringBackgroundService".equals(testInfo.getDisplayName())) { - conf.setTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, 1, - TimeUnit.SECONDS); - } - if ("testCompactionLogBackgroundService" - .equals(testInfo.getDisplayName())) { - conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED, 1, - TimeUnit.MILLISECONDS); - } - if ("testBackupCompactionFilesPruningBackgroundService" - .equals(testInfo.getDisplayName())) { - conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED, 1, - TimeUnit.MILLISECONDS); - conf.setTimeDuration( - OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 1, - TimeUnit.SECONDS); - } - if ("testSnapshotAndKeyDeletionBackgroundServices" - .equals(testInfo.getDisplayName())) { - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 1, - TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, - TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED, 1, - TimeUnit.MILLISECONDS); - conf.setTimeDuration( - OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 3, - TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, 3, - TimeUnit.SECONDS); - } + + // Used by: testSSTFilteringBackgroundService + conf.setTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); + + // Used by: testCompactionLogBackgroundService, testBackupCompactionFilesPruningBackgroundService, + // testSnapshotAndKeyDeletionBackgroundServices + conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_MAX_TIME_ALLOWED, 1, TimeUnit.MILLISECONDS); + + // Used by: testCompactionLogBackgroundService, testBackupCompactionFilesPruningBackgroundService + conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 3, TimeUnit.SECONDS); + + // Used by: testSnapshotAndKeyDeletionBackgroundServices + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); + // Used by: testSnapshotAndKeyDeletionBackgroundServices + conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); + conf.setLong( OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, SNAPSHOT_THRESHOLD); int numOfOMs = 3; cluster = MiniOzoneCluster.newHABuilder(conf) - .setOMServiceId("om-service-test1") + .setOMServiceId(omServiceId) .setNumOfOzoneManagers(numOfOMs) - .setNumOfActiveOMs(2) + .setNumOfActiveOMs(numOfOMs) .build(); - if ("testBackupCompactionFilesPruningBackgroundService" - .equals(testInfo.getDisplayName())) { - cluster. - getOzoneManagersList() - .forEach( - TestSnapshotBackgroundServices - ::suspendBackupCompactionFilesPruning); - } + cluster.waitForClusterToBeReady(); client = OzoneClientFactory.getRpcClient(omServiceId, conf); objectStore = client.getObjectStore(); + } - volumeName = "volume" + counter.incrementAndGet(); - bucketName = "bucket" + counter.incrementAndGet(); + @BeforeEach + public void setupTest() throws IOException, InterruptedException, TimeoutException { + recoverCluster(); + stopFollowerOM(cluster.getOMLeader()); + volumeName = "volume" + COUNTER.incrementAndGet(); + bucketName = "bucket" + COUNTER.incrementAndGet(); VolumeArgs createVolumeArgs = VolumeArgs.newBuilder() - .setOwner("user" + counter.incrementAndGet()) - .setAdmin("admin" + counter.incrementAndGet()) + .setOwner("user" + COUNTER.incrementAndGet()) + .setAdmin("admin" + COUNTER.incrementAndGet()) .build(); - objectStore.createVolume(volumeName, createVolumeArgs); OzoneVolume retVolumeinfo = objectStore.getVolume(volumeName); @@ -188,8 +175,33 @@ public void init(TestInfo testInfo) throws Exception { ozoneBucket = retVolumeinfo.getBucket(bucketName); } - @AfterEach - public void shutdown() { + private void recoverCluster() throws InterruptedException, TimeoutException, IOException { + for (OzoneManager ozoneManager : cluster.getOzoneManagersList()) { + if (!ozoneManager.isRunning()) { + cluster.restartOzoneManager(ozoneManager, false); + } + + if (!ozoneManager.getMetadataManager().getStore().getRocksDBCheckpointDiffer().shouldRun()) { + resumeBackupCompactionFilesPruning(ozoneManager); + } + } + cluster.waitForClusterToBeReady(); + cluster.waitForLeaderOM(); + } + + private void stopFollowerOM(OzoneManager leaderOM) throws TimeoutException, InterruptedException { + for (OzoneManager om : cluster.getOzoneManagersList()) { + if (om != leaderOM && om.isRunning()) { + String omNodeId = om.getOMNodeId(); + cluster.stopOzoneManager(omNodeId); + GenericTestUtils.waitFor(() -> !om.isRunning() && !om.isOmRpcServerRunning(), 100, 15000); + break; + } + } + } + + @AfterAll + public static void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { cluster.shutdown(); @@ -219,7 +231,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() cluster.getOzoneManager(leaderOM.getOMNodeId()); assertEquals(leaderOM, newFollowerOM); - SnapshotInfo newSnapshot = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); + SnapshotInfo newSnapshot = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); /* Check whether newly created key data is reclaimed @@ -244,7 +256,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() assertNotNull(keyInfoA); // create snapshot b - SnapshotInfo snapshotInfoB = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); + SnapshotInfo snapshotInfoB = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); assertNotNull(snapshotInfoB); // delete key a @@ -254,7 +266,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() () -> !isKeyInTable(keyA, omKeyInfoTable)); // create snapshot c - SnapshotInfo snapshotInfoC = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); + SnapshotInfo snapshotInfoC = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); // get snapshot c OmSnapshot snapC; @@ -270,7 +282,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() () -> isKeyInTable(keyA, snapC.getMetadataManager().getDeletedTable())); // create snapshot d - SnapshotInfo snapshotInfoD = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); + SnapshotInfo snapshotInfoD = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); // delete snapshot c client.getObjectStore() @@ -332,7 +344,7 @@ private void startInactiveFollower(OzoneManager leaderOM, long leaderOMSnapshotIndex = leaderOMTermIndex.getIndex(); // Start the inactive OM. Checkpoint installation will happen spontaneously. - cluster.startInactiveOM(followerOM.getOMNodeId()); + cluster.restartOzoneManager(followerOM, true); actionAfterStarting.run(); // The recently started OM should be lagging behind the leader OM. @@ -357,24 +369,39 @@ private void createSnapshotsEachWithNewKeys(OzoneManager ozoneManager) } } - private OzoneManager getInactiveFollowerOM(OzoneManager leaderOM) { - String followerNodeId = leaderOM.getPeerNodes().get(0).getNodeId(); - if (cluster.isOMActive(followerNodeId)) { - followerNodeId = leaderOM.getPeerNodes().get(1).getNodeId(); - } - return cluster.getOzoneManager(followerNodeId); + private OzoneManager getInactiveFollowerOM(OzoneManager leaderOM) + throws TimeoutException, InterruptedException { + // Wait for an inactive OM to be available + AtomicReference inactiveOM = new AtomicReference<>(); + GenericTestUtils.waitFor(() -> { + Iterator iterator = cluster.getInactiveOM(); + if (iterator.hasNext()) { + inactiveOM.set(iterator.next()); + return true; + } + return false; + }, 100, 10000); + OzoneManager result = inactiveOM.get(); + assertNotNull(result, "No inactive OM available"); + return result; } private OzoneManager getLeaderOM() { - final String leaderOMNodeId = OmTestUtil.getCurrentOmProxyNodeId(objectStore); - return cluster.getOzoneManager(leaderOMNodeId); + return cluster.getOMLeader(); } @Test @DisplayName("testCompactionLogBackgroundService") @Flaky("HDDS-11672") + @Order(Integer.MAX_VALUE) public void testCompactionLogBackgroundService() throws IOException, InterruptedException, TimeoutException { + + // reset to the default value to avoid side effects + cluster.restartOzoneManagersWithConfigCustomizer(config -> { + config.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 10, TimeUnit.MINUTES); + }); + OzoneManager leaderOM = getLeaderOM(); OzoneManager followerOM = getInactiveFollowerOM(leaderOM); @@ -407,6 +434,7 @@ public void testCompactionLogBackgroundService() newLeaderOM.getMetadataManager().getStore() .getRocksDBCheckpointDiffer().getForwardCompactionDAG().nodes() .stream().map(CompactionNode::getFileName).collect(toSet())); + assertEquals(leaderOM.getMetadataManager().getStore() .getRocksDBCheckpointDiffer().getForwardCompactionDAG().edges() .stream().map(edge -> @@ -440,9 +468,15 @@ private List getCompactionLogEntries(OzoneManager om) @DisplayName("testBackupCompactionFilesPruningBackgroundService") public void testBackupCompactionFilesPruningBackgroundService() throws IOException, InterruptedException, TimeoutException { + + cluster.getOzoneManagersList().stream() + .filter(OzoneManager::isRunning) + .forEach(TestSnapshotBackgroundServices::suspendBackupCompactionFilesPruning); + OzoneManager leaderOM = getLeaderOM(); OzoneManager followerOM = getInactiveFollowerOM(leaderOM); + startInactiveFollower(leaderOM, followerOM, () -> suspendBackupCompactionFilesPruning(followerOM)); @@ -463,6 +497,7 @@ public void testBackupCompactionFilesPruningBackgroundService() assertNotNull(files); int numberOfSstFiles = files.length; + assertEquals(cluster.getOMLeader(), newLeaderOM); resumeBackupCompactionFilesPruning(newLeaderOM); checkIfCompactionBackupFilesWerePruned(sstBackupDir, @@ -520,10 +555,10 @@ public void testSSTFilteringBackgroundService() private void confirmSnapDiffForTwoSnapshotsDifferingBySingleKey( OzoneManager ozoneManager) throws IOException, InterruptedException, TimeoutException { - String firstSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()) + String firstSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()) .getName(); String diffKey = writeKeys(1).get(0); - String secondSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()) + String secondSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()) .getName(); SnapshotDiffReportOzone diff = getSnapDiffReport(volumeName, bucketName, firstSnapshot, secondSnapshot); assertEquals(Collections.singletonList( @@ -557,7 +592,7 @@ private static File getSstBackupDir(OzoneManager ozoneManager) { private void checkIfSnapshotGetsProcessedBySFS(OzoneManager ozoneManager) throws IOException, TimeoutException, InterruptedException { writeKeys(1); - SnapshotInfo newSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); + SnapshotInfo newSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); assertNotNull(newSnapshot); Table snapshotInfoTable = ozoneManager.getMetadataManager().getSnapshotInfoTable(); @@ -641,7 +676,7 @@ private List writeKeys(long keyCount) throws IOException { List keys = new ArrayList<>(); long index = 0; while (index < keyCount) { - String key = KEY_NAME_PREFIX + counter.incrementAndGet(); + String key = KEY_NAME_PREFIX + COUNTER.incrementAndGet(); createKey(ozoneBucket, key); keys.add(key); index++; diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index d33695346175..f49c21826364 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import java.util.function.Function; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.ExitManager; @@ -125,6 +126,10 @@ public Iterator getInactiveSCM() { return scmhaService.inactiveServices(); } + public Iterator getInactiveOM() { + return omhaService.inactiveServices(); + } + public StorageContainerManager getSCM(String scmNodeId) { return this.scmhaService.getServiceById(scmNodeId); } @@ -141,6 +146,30 @@ public List getOzoneManagersList() { return omhaService.getServices(); } + public void restartOzoneManagersWithConfigCustomizer(Consumer configCustomizer) + throws IOException, TimeoutException, InterruptedException { + List toRestart = new ArrayList<>(); + for (OzoneManager om : getOzoneManagersList()) { + OzoneConfiguration configuration = new OzoneConfiguration(om.getConfiguration()); + if (configCustomizer != null) { + configCustomizer.accept(configuration); + } + om.setConfiguration(configuration); + if (om.isRunning()) { + toRestart.add(om); + } + } + for (OzoneManager om : toRestart) { + if (!om.stop()) { + continue; + } + om.join(); + om.restart(); + GenericTestUtils.waitFor(om::isRunning, 1000, 30000); + } + waitForLeaderOM(); + } + public List getStorageContainerManagersList() { return scmhaService.getServices(); } @@ -238,6 +267,11 @@ public void shutdownOzoneManager(OzoneManager ozoneManager) { public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM) throws IOException, TimeoutException, InterruptedException { LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId()); + omhaService.inactiveServices().forEachRemaining(om -> { + if (om.equals(ozoneManager)) { + this.omhaService.activate(om); + } + }); ozoneManager.restart(); if (waitForOM) { From 66c82f4913b758290415c0ec4524ae03c9be092e Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Tue, 10 Feb 2026 09:51:17 +0800 Subject: [PATCH 2/5] fix: address comments --- .../TestSnapshotBackgroundServices.java | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java index 9869ab9647c9..0efe9e1cc9d2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java @@ -87,16 +87,18 @@ import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestMethodOrder; /** * Tests snapshot background services. */ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TestSnapshotBackgroundServices { - private static MiniOzoneHAClusterImpl cluster; - private static OzoneClient client; - private static ObjectStore objectStore; + private MiniOzoneHAClusterImpl cluster; + private OzoneClient client; + private ObjectStore objectStore; private OzoneBucket ozoneBucket; private String volumeName; @@ -108,10 +110,10 @@ public class TestSnapshotBackgroundServices { private static final BucketLayout TEST_BUCKET_LAYOUT = BucketLayout.OBJECT_STORE; private static final String SNAPSHOT_NAME_PREFIX = "snapshot-"; private static final String KEY_NAME_PREFIX = "key-"; - private static final AtomicInteger COUNTER = new AtomicInteger(); + private final AtomicInteger counter = new AtomicInteger(); @BeforeAll - public static void init() throws Exception { + public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String omServiceId = "om-service-test1"; OzoneManagerRatisServerConfig omRatisConf = conf.getObject(OzoneManagerRatisServerConfig.class); @@ -161,11 +163,11 @@ public void setupTest() throws IOException, InterruptedException, TimeoutExcepti recoverCluster(); stopFollowerOM(cluster.getOMLeader()); - volumeName = "volume" + COUNTER.incrementAndGet(); - bucketName = "bucket" + COUNTER.incrementAndGet(); + volumeName = "volume" + counter.incrementAndGet(); + bucketName = "bucket" + counter.incrementAndGet(); VolumeArgs createVolumeArgs = VolumeArgs.newBuilder() - .setOwner("user" + COUNTER.incrementAndGet()) - .setAdmin("admin" + COUNTER.incrementAndGet()) + .setOwner("user" + counter.incrementAndGet()) + .setAdmin("admin" + counter.incrementAndGet()) .build(); objectStore.createVolume(volumeName, createVolumeArgs); OzoneVolume retVolumeinfo = objectStore.getVolume(volumeName); @@ -201,7 +203,7 @@ private void stopFollowerOM(OzoneManager leaderOM) throws TimeoutException, Inte } @AfterAll - public static void shutdown() { + public void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { cluster.shutdown(); @@ -231,7 +233,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() cluster.getOzoneManager(leaderOM.getOMNodeId()); assertEquals(leaderOM, newFollowerOM); - SnapshotInfo newSnapshot = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); + SnapshotInfo newSnapshot = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); /* Check whether newly created key data is reclaimed @@ -256,7 +258,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() assertNotNull(keyInfoA); // create snapshot b - SnapshotInfo snapshotInfoB = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); + SnapshotInfo snapshotInfoB = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); assertNotNull(snapshotInfoB); // delete key a @@ -266,7 +268,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() () -> !isKeyInTable(keyA, omKeyInfoTable)); // create snapshot c - SnapshotInfo snapshotInfoC = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); + SnapshotInfo snapshotInfoC = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); // get snapshot c OmSnapshot snapC; @@ -282,7 +284,7 @@ public void testSnapshotAndKeyDeletionBackgroundServices() () -> isKeyInTable(keyA, snapC.getMetadataManager().getDeletedTable())); // create snapshot d - SnapshotInfo snapshotInfoD = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); + SnapshotInfo snapshotInfoD = createOzoneSnapshot(newLeaderOM, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); // delete snapshot c client.getObjectStore() @@ -555,10 +557,10 @@ public void testSSTFilteringBackgroundService() private void confirmSnapDiffForTwoSnapshotsDifferingBySingleKey( OzoneManager ozoneManager) throws IOException, InterruptedException, TimeoutException { - String firstSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()) + String firstSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()) .getName(); String diffKey = writeKeys(1).get(0); - String secondSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()) + String secondSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()) .getName(); SnapshotDiffReportOzone diff = getSnapDiffReport(volumeName, bucketName, firstSnapshot, secondSnapshot); assertEquals(Collections.singletonList( @@ -592,7 +594,7 @@ private static File getSstBackupDir(OzoneManager ozoneManager) { private void checkIfSnapshotGetsProcessedBySFS(OzoneManager ozoneManager) throws IOException, TimeoutException, InterruptedException { writeKeys(1); - SnapshotInfo newSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + COUNTER.incrementAndGet()); + SnapshotInfo newSnapshot = createOzoneSnapshot(ozoneManager, SNAPSHOT_NAME_PREFIX + counter.incrementAndGet()); assertNotNull(newSnapshot); Table snapshotInfoTable = ozoneManager.getMetadataManager().getSnapshotInfoTable(); @@ -676,7 +678,7 @@ private List writeKeys(long keyCount) throws IOException { List keys = new ArrayList<>(); long index = 0; while (index < keyCount) { - String key = KEY_NAME_PREFIX + COUNTER.incrementAndGet(); + String key = KEY_NAME_PREFIX + counter.incrementAndGet(); createKey(ozoneBucket, key); keys.add(key); index++; From 2b49fd0232497e6ff669c54614a8739e56db4f62 Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Tue, 10 Feb 2026 09:57:08 +0800 Subject: [PATCH 3/5] chore: revert unrelated changes --- .../ozone/om/snapshot/TestSnapshotBackgroundServices.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java index 0efe9e1cc9d2..cc348dd4463a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java @@ -97,9 +97,7 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TestSnapshotBackgroundServices { private MiniOzoneHAClusterImpl cluster; - private OzoneClient client; private ObjectStore objectStore; - private OzoneBucket ozoneBucket; private String volumeName; private String bucketName; @@ -110,6 +108,7 @@ public class TestSnapshotBackgroundServices { private static final BucketLayout TEST_BUCKET_LAYOUT = BucketLayout.OBJECT_STORE; private static final String SNAPSHOT_NAME_PREFIX = "snapshot-"; private static final String KEY_NAME_PREFIX = "key-"; + private OzoneClient client; private final AtomicInteger counter = new AtomicInteger(); @BeforeAll From 593b07db5938c133e00be9361d8b0d60ab2796da Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Wed, 11 Feb 2026 16:46:50 +0800 Subject: [PATCH 4/5] fix: move active API call after OM restart to avoid inconsistent state --- .../apache/hadoop/ozone/MiniOzoneHAClusterImpl.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index f49c21826364..06a9c9f4ee2c 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java @@ -267,17 +267,18 @@ public void shutdownOzoneManager(OzoneManager ozoneManager) { public void restartOzoneManager(OzoneManager ozoneManager, boolean waitForOM) throws IOException, TimeoutException, InterruptedException { LOG.info("Restarting OzoneManager " + ozoneManager.getOMNodeId()); - omhaService.inactiveServices().forEachRemaining(om -> { - if (om.equals(ozoneManager)) { - this.omhaService.activate(om); - } - }); ozoneManager.restart(); if (waitForOM) { GenericTestUtils.waitFor(ozoneManager::isRunning, 1000, waitForClusterToBeReadyTimeout); } + + omhaService.inactiveServices().forEachRemaining(om -> { + if (om.equals(ozoneManager)) { + this.omhaService.activate(om); + } + }); } public void shutdownStorageContainerManager(StorageContainerManager scm) { From 7176929ee0d37fdd14b0ab7c034efc527e8a840e Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Wed, 11 Feb 2026 16:53:23 +0800 Subject: [PATCH 5/5] fix: remove unused parameter --- .../om/snapshot/TestSnapshotBackgroundServices.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java index cc348dd4463a..5e3f49e4f39b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotBackgroundServices.java @@ -215,7 +215,7 @@ public void shutdown() { public void testSnapshotAndKeyDeletionBackgroundServices() throws Exception { OzoneManager leaderOM = getLeaderOM(); - OzoneManager followerOM = getInactiveFollowerOM(leaderOM); + OzoneManager followerOM = getInactiveFollowerOM(); createSnapshotsEachWithNewKeys(leaderOM); @@ -370,7 +370,7 @@ private void createSnapshotsEachWithNewKeys(OzoneManager ozoneManager) } } - private OzoneManager getInactiveFollowerOM(OzoneManager leaderOM) + private OzoneManager getInactiveFollowerOM() throws TimeoutException, InterruptedException { // Wait for an inactive OM to be available AtomicReference inactiveOM = new AtomicReference<>(); @@ -404,7 +404,7 @@ public void testCompactionLogBackgroundService() }); OzoneManager leaderOM = getLeaderOM(); - OzoneManager followerOM = getInactiveFollowerOM(leaderOM); + OzoneManager followerOM = getInactiveFollowerOM(); createSnapshotsEachWithNewKeys(leaderOM); @@ -475,7 +475,7 @@ public void testBackupCompactionFilesPruningBackgroundService() .forEach(TestSnapshotBackgroundServices::suspendBackupCompactionFilesPruning); OzoneManager leaderOM = getLeaderOM(); - OzoneManager followerOM = getInactiveFollowerOM(leaderOM); + OzoneManager followerOM = getInactiveFollowerOM(); startInactiveFollower(leaderOM, followerOM, @@ -530,7 +530,7 @@ private static void suspendBackupCompactionFilesPruning( public void testSSTFilteringBackgroundService() throws IOException, InterruptedException, TimeoutException { OzoneManager leaderOM = getLeaderOM(); - OzoneManager followerOM = getInactiveFollowerOM(leaderOM); + OzoneManager followerOM = getInactiveFollowerOM(); createSnapshotsEachWithNewKeys(leaderOM);