Skip to content

Commit 5bcf018

Browse files
labkey-martypvagishalabkey-jeckelslabkey-sweta
authored
Issue 47144: Handle large files on Panorama Public via Symlinks (#344)
* - PanoramaPublicFileImporter logs to the job log, and throws an exception if any of the datafileurls could not be fixed. - PanoramaPublicSymlinkManager.moveAndSymLinkDirectory takes a Logger parameter so the log output can go to the job log - Added a PanoramaPublicMetadataImporter. I moved some of the code out of CopyExperimentFinalTask into this class. This creates a row in the panoramapublic.experimentannotations table.  It runs before PanoramaPublicFileImporter so that if there is an error, e.g. datafileurls cannot be fixed, the container can be deleted to move files back to the source container. - Updated test - import a document into a subfolder of the container file root. * - Fire symlink update events only when file / container being moved / renamed / deleted is in the Panorama Public project. We don't expect folders in other projects to contain symlink targets. - When handling folder rename (ContainerListener.propertyChange), pass the full paths of the old and renamed containers instead of just the folder names. Otherwise, it can lead to updating all symlinks that have the old folder name in the path. - When deleting a folder, use ExperimentAnnotationsManager.getExperimentIncludesContainer(c) to lookup the experiment. This method will return the experiment that contains runs from the folder even if it is a subfolder of the folder where the experiment was created. - When an experiment folder in Panorama Public is deleted, move the files back to next highest experiment version if one exists. Otherwise, move the files back to the source folder. * Rework datafile alignment * Scope datafile url to correct container * Removed PanoramaPublicFileWriter. * Limit the number of containers to look at when updating symlinks. This should only include the source container in the submitter's project as well as any containers with older versions of the data on Panorama Public. * Remove code to lookup runs in the source container when aligning datafileUrls. This should not be required anymore due to LabKey/targetedms#724. Set filePathRoot on the copied expRun to be the target container's file root. Log error if the data file path is unexpected, i.e. it does not contain "Run<runid>" Co-authored-by: vagisha <vagisha@gmail.com> Co-authored-by: Josh Eckels <jeckels@labkey.com> Co-authored-by: labkey-sweta <swetaj@labkey.com>
1 parent 8fe5720 commit 5bcf018

20 files changed

+1722
-314
lines changed

‎panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java‎

Lines changed: 58 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1499,6 +1499,20 @@ private boolean validateAction(CopyExperimentForm form, BindException errors)
14991499
return true;
15001500
}
15011501

1502+
private Path getExportFilesDir(Container c)
1503+
{
1504+
FileContentService fcs = FileContentService.get();
1505+
if(fcs != null)
1506+
{
1507+
Path fileRoot = fcs.getFileRootPath(c, FileContentService.ContentType.files);
1508+
if (fileRoot != null)
1509+
{
1510+
return fileRoot.resolve(PipelineService.EXPORT_DIR);
1511+
}
1512+
}
1513+
return null;
1514+
}
1515+
15021516
@Override
15031517
public boolean handlePost(CopyExperimentForm form, BindException errors)
15041518
{
@@ -1539,13 +1553,15 @@ public boolean handlePost(CopyExperimentForm form, BindException errors)
15391553
return false;
15401554
}
15411555

1556+
String previousVersionName = null;
15421557
Submission previousSubmission = _journalSubmission.getLatestCopiedSubmission();
15431558
if (previousSubmission != null)
15441559
{
15451560
// Target folder name is automatically populated in the copy experiment form. Unless the admin making the copy changed the
15461561
// folder name we expect the previous copy of the data to have the same folder name. Rename the old folder so that we can
15471562
// use the same folder name for the new copy.
1548-
if (!renamePreviousFolder(previousSubmission, destinationFolder, errors))
1563+
previousVersionName = renamePreviousFolder(previousSubmission, destinationFolder, errors);
1564+
if (previousVersionName == null)
15491565
{
15501566
return false;
15511567
}
@@ -1573,8 +1589,13 @@ public boolean handlePost(CopyExperimentForm form, BindException errors)
15731589
job.setUsePxTestDb(form.isUsePxTestDb());
15741590
job.setAssignDoi(form.isAssignDoi());
15751591
job.setUseDataCiteTestApi(form.isUseDataCiteTestApi());
1592+
job.setMoveAndSymlink(form.isMoveAndSymlink());
15761593
job.setReviewerEmailPrefix(form.getReviewerEmailPrefix());
15771594
job.setDeletePreviousCopy(form.isDeleteOldCopy());
1595+
job.setPreviousVersionName(previousVersionName);
1596+
job.setExportTargetPath(getExportFilesDir(target));
1597+
job.setExportSourceContainer(form.getContainer());
1598+
15781599
PipelineService.get().queueJob(job);
15791600

15801601
_successURL = PageFlowUtil.urlProvider(PipelineStatusUrls.class).urlBegin(target);
@@ -1586,12 +1607,14 @@ public boolean handlePost(CopyExperimentForm form, BindException errors)
15861607
}
15871608
}
15881609

1589-
private boolean renamePreviousFolder(Submission previousSubmission, String targetContainerName, BindException errors)
1610+
private String renamePreviousFolder(Submission previousSubmission, String targetContainerName, BindException errors)
15901611
{
1612+
String newPath = null;
15911613
ExperimentAnnotations previousCopy = ExperimentAnnotationsManager.get(previousSubmission.getCopiedExperimentId());
15921614
if (previousCopy != null)
15931615
{
15941616
Container previousContainer = previousCopy.getContainer();
1617+
newPath = previousContainer.getPath();
15951618
if (targetContainerName.equals(previousContainer.getName()))
15961619
{
15971620
try (DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction())
@@ -1601,21 +1624,23 @@ private boolean renamePreviousFolder(Submission previousSubmission, String targe
16011624
{
16021625
errors.reject(ERROR_MSG, "Previous experiment copy (Id: " + previousCopy.getId() + ") does not have a version. " +
16031626
"Cannot rename previous folder.");
1604-
return false;
1627+
return null;
16051628
}
16061629
// Rename the container where the old copy lives so that the same folder name can be used for the new copy.
16071630
String newName = previousContainer.getName() + " V." + version;
16081631
if (ContainerManager.getChild(previousContainer.getParent(), newName) != null)
16091632
{
16101633
errors.reject(ERROR_MSG, "Cannot rename previous folder to '" + newName + "'. A folder with that name already exists.");
1611-
return false;
1634+
return null;
16121635
}
16131636
ContainerManager.rename(previousContainer, getUser(), newName);
1637+
1638+
newPath = FileContentService.get().getFileRoot(previousContainer.getParent()) + File.separator + newName;
16141639
transaction.commit();
16151640
}
16161641
}
16171642
}
1618-
return true;
1643+
return newPath;
16191644
}
16201645

16211646
private ValidEmail getValidEmail(String email, String errMsg, BindException errors)
@@ -1683,6 +1708,8 @@ public static class CopyExperimentForm extends ExperimentIdForm
16831708
private boolean _usePxTestDb; // Use the test database for getting a PX ID if true
16841709
private boolean _assignDoi;
16851710
private boolean _useDataCiteTestApi;
1711+
1712+
private boolean _moveAndSymlink;
16861713
private boolean _deleteOldCopy;
16871714

16881715
static void setDefaults(CopyExperimentForm form, ExperimentAnnotations sourceExperiment, Submission currentSubmission)
@@ -1696,6 +1723,7 @@ static void setDefaults(CopyExperimentForm form, ExperimentAnnotations sourceExp
16961723
form.setUsePxTestDb(false);
16971724

16981725
form.setAssignDoi(true);
1726+
form.setMoveAndSymlink(true);
16991727
form.setUseDataCiteTestApi(false);
17001728

17011729
Container sourceExptContainer = sourceExperiment.getContainer();
@@ -1819,6 +1847,16 @@ public void setUseDataCiteTestApi(boolean useDataCiteTestApi)
18191847
_useDataCiteTestApi = useDataCiteTestApi;
18201848
}
18211849

1850+
public boolean isMoveAndSymlink()
1851+
{
1852+
return _moveAndSymlink;
1853+
}
1854+
1855+
public void setMoveAndSymlink(boolean moveAndSymlink)
1856+
{
1857+
_moveAndSymlink = moveAndSymlink;
1858+
}
1859+
18221860
public boolean isDeleteOldCopy()
18231861
{
18241862
return _deleteOldCopy;
@@ -5626,7 +5664,7 @@ public ExperimentAnnotationsDetails(User user, ExperimentAnnotations exptAnnotat
56265664
{
56275665
// Display the version only if there is more than one version of this dataset on Panorama Public
56285666
_version = _experimentAnnotations.getStringVersion(maxVersion);
5629-
if (_experimentAnnotations.getDataVersion().equals(maxVersion))
5667+
if (_experimentAnnotations.getDataVersion() != null && _experimentAnnotations.getDataVersion().equals(maxVersion))
56305668
{
56315669
// This is the current version; Display a link to see all published versions
56325670
_versionsUrl = new ActionURL(PanoramaPublicController.ShowPublishedVersions.class, _experimentAnnotations.getContainer());
@@ -9050,6 +9088,20 @@ public Pair<AttachmentParent, String> getAttachment(AttachmentForm form)
90509088
}
90519089
}
90529090

9091+
@RequiresPermission(ReadPermission.class)
9092+
public static class VerifySymlinksAction extends ReadOnlyApiAction<CatalogForm>
9093+
{
9094+
@Override
9095+
public Object execute(CatalogForm catalogForm, BindException errors) throws Exception
9096+
{
9097+
if (PanoramaPublicSymlinkManager.get().verifySymlinks())
9098+
return success();
9099+
9100+
errors.reject(ERROR_MSG, "Problems with symlink registration. See log for details.");
9101+
return null;
9102+
}
9103+
}
9104+
90539105
@RequiresPermission(ReadPermission.class)
90549106
public static class GetCatalogApiAction extends ReadOnlyApiAction<CatalogForm>
90559107
{

‎panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
package org.labkey.panoramapublic;
2+
3+
import org.apache.logging.log4j.Logger;
4+
import org.jetbrains.annotations.Nullable;
5+
import org.labkey.api.admin.AbstractFolderImportFactory;
6+
import org.labkey.api.admin.FolderImportContext;
7+
import org.labkey.api.admin.FolderImporter;
8+
import org.labkey.api.admin.ImportException;
9+
import org.labkey.api.admin.SubfolderWriter;
10+
import org.labkey.api.data.Container;
11+
import org.labkey.api.exp.api.ExpData;
12+
import org.labkey.api.exp.api.ExpRun;
13+
import org.labkey.api.exp.api.ExperimentService;
14+
import org.labkey.api.files.FileContentService;
15+
import org.labkey.api.pipeline.PipelineJob;
16+
import org.labkey.api.pipeline.PipelineService;
17+
import org.labkey.api.query.BatchValidationException;
18+
import org.labkey.api.security.User;
19+
import org.labkey.api.writer.VirtualFile;
20+
import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineJob;
21+
22+
import java.io.File;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.nio.file.Paths;
26+
import java.util.List;
27+
import java.util.Objects;
28+
29+
/**
30+
* This importer does a file move instead of copy to the temp directory and creates a symlink in place of the original
31+
* file.
32+
*/
33+
public class PanoramaPublicFileImporter implements FolderImporter
34+
{
35+
@Override
36+
public String getDataType()
37+
{
38+
return PanoramaPublicManager.PANORAMA_PUBLIC_FILES;
39+
}
40+
41+
@Override
42+
public String getDescription()
43+
{
44+
return "Panorama Public Files";
45+
}
46+
47+
@Override
48+
public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualFile root) throws Exception
49+
{
50+
Logger log = ctx.getLogger();
51+
52+
FileContentService fcs = FileContentService.get();
53+
if (null == fcs)
54+
return;
55+
56+
File targetRoot = fcs.getFileRoot(ctx.getContainer());
57+
58+
if (null == targetRoot)
59+
{
60+
log.error("File copy target folder not found: " + ctx.getContainer().getPath());
61+
return;
62+
}
63+
64+
if (null == job)
65+
{
66+
log.error("Pipeline job not found.");
67+
return;
68+
}
69+
70+
if (job instanceof CopyExperimentPipelineJob expJob)
71+
{
72+
File targetFiles = new File(targetRoot.getPath(), FileContentService.FILES_LINK);
73+
74+
// Get source files including resolving subfolders
75+
String divider = FileContentService.FILES_LINK + File.separator + PipelineService.EXPORT_DIR;
76+
String subProject = root.getLocation().substring(root.getLocation().lastIndexOf(divider) + divider.length());
77+
subProject = subProject.replace(File.separator + SubfolderWriter.DIRECTORY_NAME, "");
78+
79+
Path sourcePath = Paths.get(fcs.getFileRoot(expJob.getExportSourceContainer()).getPath(), subProject);
80+
File sourceFiles = Paths.get(sourcePath.toString(), FileContentService.FILES_LINK).toFile();
81+
82+
if (!targetFiles.exists())
83+
{
84+
log.warn("Panorama public file copy target not found. Creating directory: " + targetFiles);
85+
Files.createDirectories(targetFiles.toPath());
86+
}
87+
88+
log.info("Moving files and creating sym links in folder " + ctx.getContainer().getPath());
89+
PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, false, log);
90+
91+
alignDataFileUrls(expJob.getUser(), ctx.getContainer(), log);
92+
}
93+
}
94+
95+
private void alignDataFileUrls(User user, Container targetContainer, Logger log) throws BatchValidationException, ImportException
96+
{
97+
log.info("Aligning data files urls in folder: " + targetContainer.getPath());
98+
99+
FileContentService fcs = FileContentService.get();
100+
if (null == fcs)
101+
return;
102+
103+
ExperimentService expService = ExperimentService.get();
104+
List<? extends ExpRun> runs = expService.getExpRuns(targetContainer, null, null);
105+
boolean errors = false;
106+
107+
Path fileRootPath = fcs.getFileRootPath(targetContainer, FileContentService.ContentType.files);
108+
if(fileRootPath == null || !Files.exists(fileRootPath))
109+
{
110+
throw new ImportException("File root path for container " + targetContainer.getPath() + " does not exist: " + fileRootPath);
111+
}
112+
113+
for (ExpRun run : runs)
114+
{
115+
run.setFilePathRootPath(fileRootPath);
116+
run.save(user);
117+
log.debug("Setting filePathRoot on copied run: " + run.getName() + " to: " + fileRootPath);
118+
119+
for (ExpData data : run.getAllDataUsedByRun())
120+
{
121+
if (null != data.getRun() && data.getDataFileUrl().contains(FileContentService.FILES_LINK))
122+
{
123+
String[] parts = Objects.requireNonNull(data.getFilePath()).toString().split("Run\\d+");
124+
125+
if (parts.length > 1)
126+
{
127+
String fileName = parts[1];
128+
Path newDataPath = Paths.get(fileRootPath.toString(), fileName);
129+
130+
if (newDataPath.toFile().exists())
131+
{
132+
data.setDataFileURI(newDataPath.toUri());
133+
data.save(user);
134+
log.debug("Setting dataFileUri on copied data: " + data.getName() + " to: " + newDataPath);
135+
}
136+
else
137+
{
138+
log.error("Data file not found: " + newDataPath.toUri());
139+
errors = true;
140+
}
141+
}
142+
else
143+
{
144+
log.error("Unexpected data file path. Could not align dataFileUri. " + data.getFilePath().toString());
145+
errors = true;
146+
}
147+
}
148+
}
149+
}
150+
if (errors)
151+
{
152+
throw new ImportException("Data files urls could not be aligned.");
153+
}
154+
}
155+
156+
public static class Factory extends AbstractFolderImportFactory
157+
{
158+
@Override
159+
public FolderImporter create()
160+
{
161+
return new PanoramaPublicFileImporter();
162+
}
163+
164+
@Override
165+
public int getPriority()
166+
{
167+
// We want this to run last to do exp.data.datafileurl cleanup
168+
return PanoramaPublicManager.PRIORITY_PANORAMA_PUBLIC_FILES;
169+
}
170+
}
171+
}

‎panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package org.labkey.panoramapublic;
2+
3+
import org.jetbrains.annotations.NotNull;
4+
import org.jetbrains.annotations.Nullable;
5+
import org.labkey.api.data.Container;
6+
import org.labkey.api.data.SQLFragment;
7+
import org.labkey.api.exp.api.ExpData;
8+
import org.labkey.api.exp.api.ExperimentService;
9+
import org.labkey.api.files.FileListener;
10+
import org.labkey.api.security.User;
11+
12+
import java.io.File;
13+
import java.nio.file.Path;
14+
import java.util.Collection;
15+
import java.util.Collections;
16+
17+
public class PanoramaPublicFileListener implements FileListener
18+
{
19+
20+
@Override
21+
public String getSourceName()
22+
{
23+
return null;
24+
}
25+
26+
@Override
27+
public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container)
28+
{
29+
30+
}
31+
32+
@Override
33+
public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container)
34+
{
35+
// Update any symlinks targeting the file
36+
PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container);
37+
38+
ExpData data = ExperimentService.get().getExpDataByURL(src, null);
39+
if (null != data)
40+
data.setDataFileURI(dest.toURI());
41+
42+
return 0;
43+
}
44+
45+
@Override
46+
public void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container)
47+
{
48+
ExpData data = ExperimentService.get().getExpDataByURL(deleted, container);
49+
50+
if (null != data)
51+
data.delete(user);
52+
}
53+
54+
@Override
55+
public Collection<File> listFiles(@Nullable Container container)
56+
{
57+
return Collections.emptyList();
58+
}
59+
60+
@Override
61+
public SQLFragment listFilesQuery()
62+
{
63+
throw new UnsupportedOperationException("Not implemented");
64+
}
65+
}

0 commit comments

Comments
 (0)