refs #99 - don't write payload file to data directory if that file is in the fetch list

Author: johnscancella

src/main/java/gov/loc/repository/bagit/writer/PayloadWriter.java

@@ -4,13 +4,16 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.HashSet;
+import java.util.List;
 import java.util.ResourceBundle;
 import java.util.Set;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import gov.loc.repository.bagit.domain.Bag;
+import gov.loc.repository.bagit.domain.FetchItem;
 import gov.loc.repository.bagit.domain.Manifest;
 import gov.loc.repository.bagit.domain.Version;
 
@@ -35,12 +38,12 @@ static Path writeVersionDependentPayloadFiles(final Bag bag, final Path outputDi
     if(bag.getVersion().isSameOrNewer(VERSION_2_0)){
       bagitDir = outputDir.resolve(".bagit");
       Files.createDirectories(bagitDir);
-      writePayloadFiles(bag.getPayLoadManifests(), outputDir, bag.getRootDir());
+      writePayloadFiles(bag.getPayLoadManifests(), bag.getItemsToFetch(), outputDir, bag.getRootDir());
     }
     else{
       final Path dataDir = outputDir.resolve("data");
       Files.createDirectories(dataDir);
-      writePayloadFiles(bag.getPayLoadManifests(), dataDir, bag.getRootDir().resolve("data"));
+      writePayloadFiles(bag.getPayLoadManifests(), bag.getItemsToFetch(), dataDir, bag.getRootDir().resolve("data"));
     }
 
     return bagitDir;
@@ -50,25 +53,41 @@ static Path writeVersionDependentPayloadFiles(final Bag bag, final Path outputDi
   * Write the payload <b>file(s)</b> to the output directory
   * 
   * @param payloadManifests the set of objects representing the payload manifests
+  * @param fetchItems the list of items to exclude from writing in the output directory because they will be fetched
   * @param outputDir the data directory of the bag
   * @param bagDataDir the data directory of the bag
   * 
   * @throws IOException if there was a problem writing a file
   */
- public static void writePayloadFiles(final Set<Manifest> payloadManifests, final Path outputDir, final Path bagDataDir) throws IOException{
+ public static void writePayloadFiles(final Set<Manifest> payloadManifests, final List<FetchItem> fetchItems, final Path outputDir, final Path bagDataDir) throws IOException{
    logger.info(messages.getString("writing_payload_files"));
+   final Set<Path> fetchPaths = getFetchPaths(fetchItems);
+   
    for(final Manifest payloadManifest : payloadManifests){
      for(final Path payloadFile : payloadManifest.getFileToChecksumMap().keySet()){
-       final Path relativePayloadPath = bagDataDir.relativize(payloadFile); 
-           
-       final Path writeToPath = outputDir.resolve(relativePayloadPath);
-       logger.debug(messages.getString("writing_payload_file_to_path"), payloadFile, writeToPath);
-       final Path parent = writeToPath.getParent();
-       if(parent != null){
-         Files.createDirectories(parent);
+       final Path relativePayloadPath = bagDataDir.relativize(payloadFile);
+       
+       if(fetchPaths.contains(relativePayloadPath.normalize())) {
+         logger.info(messages.getString("skip_fetch_item_when_writing_payload"), payloadFile);
+       }
+       else {
+         final Path writeToPath = outputDir.resolve(relativePayloadPath);
+         logger.debug(messages.getString("writing_payload_file_to_path"), payloadFile, writeToPath);
+         final Path parent = writeToPath.getParent();
+         if(parent != null){
+           Files.createDirectories(parent);
+         }
+         Files.copy(payloadFile, writeToPath, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
        }
-       Files.copy(payloadFile, writeToPath, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
      }
    }
  }
+ 
+ private static Set<Path> getFetchPaths(final List<FetchItem> fetchItems){
+   final Set<Path> fetchPaths = new HashSet<>();
+   for(final FetchItem fetchItem : fetchItems) {
+     fetchPaths.add(fetchItem.getPath());
+   }
+   return fetchPaths;
+ }
 }

src/main/resources/MessageBundle.properties

@@ -217,4 +217,5 @@ writing_manifest_to_path=Writing manifest to [{}].
 writing_metadata_to_path=Writing bag metadata file [{}] to [{}].
 
 #for PayloadWriter.java
-writing_payload_file_to_path=Writing payload file [{}] to [{}].
+writing_payload_file_to_path=Writing payload file [{}] to [{}].
+skip_fetch_item_when_writing_payload=Skipping payload file {} because it is in the fetch list.

src/test/java/gov/loc/repository/bagit/writer/PayloadWriterTest.java

@@ -6,6 +6,8 @@
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -14,6 +16,7 @@
 import org.junit.rules.TemporaryFolder;
 
 import gov.loc.repository.bagit.PrivateConstructorTest;
+import gov.loc.repository.bagit.domain.FetchItem;
 import gov.loc.repository.bagit.domain.Manifest;
 import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms;
 
@@ -39,7 +42,23 @@ public void testWritePayloadFiles() throws IOException, URISyntaxException{
     File copiedFile = new File(outputDir, "data/dir1/test3.txt");
 
     assertFalse(copiedFile.exists() || copiedFile.getParentFile().exists());
-    PayloadWriter.writePayloadFiles(payloadManifests, Paths.get(outputDir.toURI()), rootDir);
+    PayloadWriter.writePayloadFiles(payloadManifests, new ArrayList<>(),Paths.get(outputDir.toURI()), rootDir);
     assertTrue(copiedFile.exists() && copiedFile.getParentFile().exists());
   }
+  
+  @Test
+  public void testWritePayloadFilesMinusFetchFiles() throws IOException, URISyntaxException{
+    Path rootDir = Paths.get(getClass().getClassLoader().getResource("bags/v0_97/bag").toURI());
+    Path testFile = Paths.get(getClass().getClassLoader().getResource("bags/v0_97/bag/data/dir1/test3.txt").toURI());
+    Manifest manifest = new Manifest(StandardSupportedAlgorithms.MD5);
+    manifest.getFileToChecksumMap().put(testFile, "someHashValue");
+    Set<Manifest> payloadManifests = new HashSet<>();
+    payloadManifests.add(manifest);
+    File outputDir = folder.newFolder();
+    File copiedFile = new File(outputDir, "data/dir1/test3.txt");
+    
+    assertFalse(copiedFile.exists() || copiedFile.getParentFile().exists());
+    PayloadWriter.writePayloadFiles(payloadManifests, Arrays.asList(new FetchItem(null, null, Paths.get("data/dir1/test3.txt"))),Paths.get(outputDir.toURI()), rootDir);
+    assertFalse(copiedFile.exists() && copiedFile.getParentFile().exists());
+  }
 }