From 4760e3e333847434add73651dc0c194c57354c49 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Tue, 31 Mar 2026 19:11:27 -0700 Subject: [PATCH 01/10] Setting the writable flag of the child directory before deleting. --- .../assay/AssayTransformMissingParentDirTest.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index d0979ba985..32d6c96fb6 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -19,6 +19,7 @@ import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.stream.Stream; /** * Issue 54156: Regression test to ensure a reasonable error message is shown when an assay design references @@ -48,11 +49,19 @@ public void testMissingParentDirectoryRegression() throws Exception assayDesignerPage.addTransformScript(transformFile); assayDesignerPage.clickSave(); - // Now delete the parent dir to ensure we handle it reasonably. On Windows something locks the directory, maybe - // an external process. If that happens sleep for a second and try again. + // Now delete the parent dir to ensure we handle it reasonably. + // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is + // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. for (int attempt = 1; attempt <= 10; attempt++) { try { + parentDir.toFile().setWritable(true, false); + + // Wrap in a try to close the stream. + try (Stream files = Files.walk(parentDir)) { + files.forEach(p -> p.toFile().setWritable(true, false)); + } + FileUtils.deleteDirectory(parentDir.toFile()); log(String.format("Deletion of directory %s was successful.", parentDir)); break; From a6716e09ccf49a32baf18bed2107c74ed237ff17 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Mon, 6 Apr 2026 19:36:14 -0700 Subject: [PATCH 02/10] Adding a call to run "tasklist" if running on Windows. --- .../AssayTransformMissingParentDirTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index 32d6c96fb6..660bc32079 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -1,9 +1,11 @@ package org.labkey.test.tests.assay; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; import org.junit.Test; import org.junit.experimental.categories.Category; import org.labkey.api.util.FileUtil; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; import org.labkey.test.categories.Assays; @@ -76,7 +78,22 @@ public void testMissingParentDirectoryRegression() throws Exception catch (IOException ioException) { log(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", parentDir, ioException.getMessage(), attempt)); - if (attempt == 10) throw ioException; + if (attempt == 10) + { + if (SystemUtils.IS_OS_WINDOWS) { + try { + log("Lock diagnostic..."); + ProcessBuilder pb = new ProcessBuilder("tasklist"); + pb.redirectErrorStream(true); + Process p = pb.start(); + String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); + log("Running processes:\n" + output); + } catch (IOException diagnosticException) { + log("Failed to run lock diagnostic: " + diagnosticException.getMessage()); + } + } + throw ioException; + } sleep(10_000); } } From 195b11234ec01d97aab066b7cb2c87a9116a8eae Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Tue, 7 Apr 2026 18:11:11 -0700 Subject: [PATCH 03/10] Let the search indexer run. --- .../test/tests/assay/AssayTransformMissingParentDirTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index 660bc32079..4ea482faa0 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -15,6 +15,7 @@ import org.labkey.test.pages.assay.AssayImportPage; import org.labkey.test.pages.assay.AssayRunsPage; import org.labkey.test.params.assay.GeneralAssayDesign; +import org.labkey.test.util.search.SearchAdminAPIHelper; import java.io.File; import java.io.IOException; @@ -51,6 +52,9 @@ public void testMissingParentDirectoryRegression() throws Exception assayDesignerPage.addTransformScript(transformFile); assayDesignerPage.clickSave(); + // Let the index run if needed. + SearchAdminAPIHelper.waitForIndexer(); + // Now delete the parent dir to ensure we handle it reasonably. // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. From 7850ea49bbbab99b0a72e8df4b4da4497d113a90 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Wed, 8 Apr 2026 18:44:33 -0700 Subject: [PATCH 04/10] Remove wait for pipeline jobs and add heapDump. --- .../test/tests/assay/AssayTransformMissingParentDirTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index 4ea482faa0..2d1733f8e9 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -84,6 +84,9 @@ public void testMissingParentDirectoryRegression() throws Exception parentDir, ioException.getMessage(), attempt)); if (attempt == 10) { + log("Dump the heap."); + dumpHeap(); + if (SystemUtils.IS_OS_WINDOWS) { try { log("Lock diagnostic..."); From 8bbb7eec0b3002e9063531c4acbdf40100457510 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Fri, 10 Apr 2026 16:10:14 -0700 Subject: [PATCH 05/10] Revert changes to AssayReimportIndexTest. No need to wait for the search indexer. --- .../test/tests/assay/AssayTransformMissingParentDirTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index 2d1733f8e9..5cd0267904 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -52,9 +52,6 @@ public void testMissingParentDirectoryRegression() throws Exception assayDesignerPage.addTransformScript(transformFile); assayDesignerPage.clickSave(); - // Let the index run if needed. - SearchAdminAPIHelper.waitForIndexer(); - // Now delete the parent dir to ensure we handle it reasonably. // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. From 2c7ee8f76c4e04edeca98ce36209ced23979d8e8 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Tue, 14 Apr 2026 18:13:19 -0700 Subject: [PATCH 06/10] Revert AssayTransformMissingParentDirTest.testMissingParentDirectoryRegression to what it was originally. Move the delete and retry code in to TestFileUtils.deleteDirWithRetry --- src/org/labkey/test/TestFileUtils.java | 74 +++++++++++++++++++ .../AssayTransformMissingParentDirTest.java | 51 +------------ 2 files changed, 76 insertions(+), 49 deletions(-) diff --git a/src/org/labkey/test/TestFileUtils.java b/src/org/labkey/test/TestFileUtils.java index d73ad973a1..8841bf1f86 100644 --- a/src/org/labkey/test/TestFileUtils.java +++ b/src/org/labkey/test/TestFileUtils.java @@ -21,6 +21,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.pdfbox.Loader; @@ -40,6 +41,7 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.util.FileUtil; import org.jetbrains.annotations.Nullable; +import org.labkey.api.util.StringUtilsLabKey; import org.openqa.selenium.NotFoundException; import java.io.BufferedInputStream; @@ -57,6 +59,7 @@ import java.io.Writer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -71,10 +74,13 @@ import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import static org.labkey.api.util.DebugInfoDumper.dumpHeap; +import static org.labkey.test.WebDriverWrapper.sleep; import static org.labkey.test.util.TestDataGenerator.CHARSET_STRING; import static org.labkey.test.util.TestDataGenerator.randomInt; import static org.labkey.test.util.TestDataGenerator.randomName; @@ -445,6 +451,74 @@ public static void deleteDir(File dir) } } + /** + * Deletes a directory and all its contents, retrying up to 10 times with a 10-second delay between attempts. + *

+ * Before each attempt, the directory and all its children are marked writable to handle read-only files or + * directories. This is primarily intended to work around Windows file-locking issues where an external process + * may hold a lock on the directory or its contents. + *

+ * On the final failed attempt, a heap dump is captured for diagnostics. On Windows, the list of running + * processes is also logged to help identify what may be holding the lock. + * + * @param dir the directory to delete + * @throws AccessDeniedException if access is denied on all 10 attempts + * @throws IOException if an I/O error occurs on all 10 attempts + * @throws Exception if an unexpected error occurs + */ + public static void deleteDirWithRetry(File dir) throws Exception + { + // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is + // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. + for (int attempt = 1; attempt <= 10; attempt++) { + try + { + dir.setWritable(true, false); + + // Wrap in a try to close the stream. + try (Stream files = Files.walk(dir.toPath())) { + files.forEach(p -> p.toFile().setWritable(true, false)); + } + + FileUtils.deleteDirectory(dir); + LOG.info(String.format("Deletion of directory %s was successful.", dir)); + break; + } catch (AccessDeniedException deniedException) { + // Yes I know AccessDeniedException is a subset of an IOException, but I wanted to log explicitly a + // failure and retry because of an AccessDeniedException from some other IOException. + LOG.warn(String.format("Access denied trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", + dir, deniedException.getMessage(), attempt)); + if (attempt == 10) throw deniedException; + sleep(10_000); + } + catch (IOException ioException) { + LOG.warn(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", + dir, ioException.getMessage(), attempt)); + if (attempt == 10) + { + LOG.info("Dump the heap."); + dumpHeap(); + + if (SystemUtils.IS_OS_WINDOWS) { + try { + LOG.info("Lock diagnostic..."); + ProcessBuilder pb = new ProcessBuilder("tasklist"); + pb.redirectErrorStream(true); + Process p = pb.start(); + String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); + LOG.info("Running processes:\n" + output); + } catch (IOException diagnosticException) { + LOG.warn("Failed to run lock diagnostic: " + diagnosticException.getMessage()); + } + } + throw ioException; + } + sleep(10_000); + } + } + + } + private static void checkFileLocation(File file) { try diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index 5cd0267904..9f9793f279 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -52,55 +52,8 @@ public void testMissingParentDirectoryRegression() throws Exception assayDesignerPage.addTransformScript(transformFile); assayDesignerPage.clickSave(); - // Now delete the parent dir to ensure we handle it reasonably. - // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is - // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. - for (int attempt = 1; attempt <= 10; attempt++) { - try - { - parentDir.toFile().setWritable(true, false); - - // Wrap in a try to close the stream. - try (Stream files = Files.walk(parentDir)) { - files.forEach(p -> p.toFile().setWritable(true, false)); - } - - FileUtils.deleteDirectory(parentDir.toFile()); - log(String.format("Deletion of directory %s was successful.", parentDir)); - break; - } catch (AccessDeniedException deniedException) { - // Yes I know AccessDeniedException is a subset of an IOException, but I wanted to log explicitly a - // failure and retry because of an AccessDeniedException from some other IOException. - log(String.format("Access denied trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", - parentDir, deniedException.getMessage(), attempt)); - if (attempt == 10) throw deniedException; - sleep(10_000); - } - catch (IOException ioException) { - log(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", - parentDir, ioException.getMessage(), attempt)); - if (attempt == 10) - { - log("Dump the heap."); - dumpHeap(); - - if (SystemUtils.IS_OS_WINDOWS) { - try { - log("Lock diagnostic..."); - ProcessBuilder pb = new ProcessBuilder("tasklist"); - pb.redirectErrorStream(true); - Process p = pb.start(); - String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); - log("Running processes:\n" + output); - } catch (IOException diagnosticException) { - log("Failed to run lock diagnostic: " + diagnosticException.getMessage()); - } - } - throw ioException; - } - sleep(10_000); - } - } + // Now delete the parent dir to ensure we handle it reasonably + TestFileUtils.deleteDirWithRetry(parentDir.toFile()); // Attempt to import data and verify a reasonable error message is shown String importData = """ From d9d2906c9f872c8fd55e021188f1b3dfb6c64397 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Tue, 14 Apr 2026 18:17:17 -0700 Subject: [PATCH 07/10] Remove unused imports. --- .../tests/assay/AssayTransformMissingParentDirTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index 9f9793f279..48763c9eb8 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -1,11 +1,8 @@ package org.labkey.test.tests.assay; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.SystemUtils; import org.junit.Test; import org.junit.experimental.categories.Category; import org.labkey.api.util.FileUtil; -import org.labkey.api.util.StringUtilsLabKey; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; import org.labkey.test.categories.Assays; @@ -15,14 +12,10 @@ import org.labkey.test.pages.assay.AssayImportPage; import org.labkey.test.pages.assay.AssayRunsPage; import org.labkey.test.params.assay.GeneralAssayDesign; -import org.labkey.test.util.search.SearchAdminAPIHelper; import java.io.File; -import java.io.IOException; -import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.stream.Stream; /** * Issue 54156: Regression test to ensure a reasonable error message is shown when an assay design references From 50197305456596d1503563c5c06ae79e350284e4 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Tue, 14 Apr 2026 23:08:54 -0700 Subject: [PATCH 08/10] Don't leak resources. --- src/org/labkey/test/TestFileUtils.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/org/labkey/test/TestFileUtils.java b/src/org/labkey/test/TestFileUtils.java index 8841bf1f86..fee0fa9e74 100644 --- a/src/org/labkey/test/TestFileUtils.java +++ b/src/org/labkey/test/TestFileUtils.java @@ -505,8 +505,15 @@ public static void deleteDirWithRetry(File dir) throws Exception ProcessBuilder pb = new ProcessBuilder("tasklist"); pb.redirectErrorStream(true); Process p = pb.start(); - String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); - LOG.info("Running processes:\n" + output); + try { + String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); + LOG.info("Running processes:\n" + output); + } + finally + { + // Don't leak the process resource. + p.destroy(); + } } catch (IOException diagnosticException) { LOG.warn("Failed to run lock diagnostic: " + diagnosticException.getMessage()); } From b124b46876769fb6b1f06348c522432fbe1add38 Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Wed, 15 Apr 2026 09:27:10 -0700 Subject: [PATCH 09/10] Catch only the IOException. Dump the heap if running on TeamCity. Get the list of running processes if running on Linux as well. --- src/org/labkey/test/TestFileUtils.java | 61 +++++++++++++------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/org/labkey/test/TestFileUtils.java b/src/org/labkey/test/TestFileUtils.java index fee0fa9e74..92bd948ae7 100644 --- a/src/org/labkey/test/TestFileUtils.java +++ b/src/org/labkey/test/TestFileUtils.java @@ -471,8 +471,7 @@ public static void deleteDirWithRetry(File dir) throws Exception // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. for (int attempt = 1; attempt <= 10; attempt++) { - try - { + try { dir.setWritable(true, false); // Wrap in a try to close the stream. @@ -483,40 +482,42 @@ public static void deleteDirWithRetry(File dir) throws Exception FileUtils.deleteDirectory(dir); LOG.info(String.format("Deletion of directory %s was successful.", dir)); break; - } catch (AccessDeniedException deniedException) { - // Yes I know AccessDeniedException is a subset of an IOException, but I wanted to log explicitly a - // failure and retry because of an AccessDeniedException from some other IOException. - LOG.warn(String.format("Access denied trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", - dir, deniedException.getMessage(), attempt)); - if (attempt == 10) throw deniedException; - sleep(10_000); - } - catch (IOException ioException) { + } catch (IOException ioException) { LOG.warn(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", dir, ioException.getMessage(), attempt)); - if (attempt == 10) - { - LOG.info("Dump the heap."); - dumpHeap(); + if (attempt == 10) { + if (TestProperties.isTestRunningOnTeamCity()) { + LOG.info("Dump the heap."); + dumpHeap(); + } + + ProcessBuilder pb; if (SystemUtils.IS_OS_WINDOWS) { + pb = new ProcessBuilder("tasklist"); + } + else { + pb = new ProcessBuilder("ps", "-ef"); + } + + try { + LOG.info("Lock diagnostic..."); + pb.redirectErrorStream(true); + + // Tried to use "try (Process p = pb.start()) {" without the finally block but our build + // system didn't like that and complained that Process doesn't implement closeable (it does). + Process p = pb.start(); try { - LOG.info("Lock diagnostic..."); - ProcessBuilder pb = new ProcessBuilder("tasklist"); - pb.redirectErrorStream(true); - Process p = pb.start(); - try { - String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); - LOG.info("Running processes:\n" + output); - } - finally - { - // Don't leak the process resource. - p.destroy(); - } - } catch (IOException diagnosticException) { - LOG.warn("Failed to run lock diagnostic: " + diagnosticException.getMessage()); + String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); + LOG.info("Running processes:\n" + output); + } + finally { + // Don't leak the process resource. + p.destroy(); } + + } catch (IOException diagnosticException) { + LOG.warn("Failed to run lock diagnostic: " + diagnosticException.getMessage()); } throw ioException; } From 1dcf68d041bcb2f57bd0c2e13f992411bab8863a Mon Sep 17 00:00:00 2001 From: labkey-danield Date: Wed, 15 Apr 2026 09:30:48 -0700 Subject: [PATCH 10/10] Update the javadoc comment. --- src/org/labkey/test/TestFileUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/org/labkey/test/TestFileUtils.java b/src/org/labkey/test/TestFileUtils.java index 92bd948ae7..23b77c54a7 100644 --- a/src/org/labkey/test/TestFileUtils.java +++ b/src/org/labkey/test/TestFileUtils.java @@ -458,12 +458,10 @@ public static void deleteDir(File dir) * directories. This is primarily intended to work around Windows file-locking issues where an external process * may hold a lock on the directory or its contents. *

- * On the final failed attempt, a heap dump is captured for diagnostics. On Windows, the list of running + * On the final failed attempt, a heap dump is captured for diagnostics if running on TeamCity. The list of running * processes is also logged to help identify what may be holding the lock. * * @param dir the directory to delete - * @throws AccessDeniedException if access is denied on all 10 attempts - * @throws IOException if an I/O error occurs on all 10 attempts * @throws Exception if an unexpected error occurs */ public static void deleteDirWithRetry(File dir) throws Exception