Allow seurat merge in batches to reduce memory footprint

Author: bbimber

singlecell/resources/chunks/MergeSeurat.R

@@ -1,25 +1,49 @@
-if (length(seuratObjects) == 1) {
-    print('There is only one seurat object, no need to merge')
-    datasetId <- names(seuratObjects)[[1]]
-    saveData(seuratObjects[[datasetId]], datasetId)
-} else {
+doDiet <- exists('doDiet') && doDiet
+
+mergeBatch <- function(dat) {
     toMerge <- list()
-    for (datasetId in names(seuratObjects)) {
-        doDiet <- exists('doDiet') && doDiet
-        if (exists('doDiet') && doDiet) {
-            print('Running DietSeurat on inputs')
-            toMerge[[datasetId]] <- Seurat::DietSeurat(readRDS(seuratObjects[[datasetId]]))
+    for (datasetId in names(dat)) {
+        message(paste0('Loading: ', datasetId))
+        if (doDiet) {
+            toMerge[[datasetId]] <- Seurat::DietSeurat(readRDS(dat[[datasetId]]))
             gc()
         } else {
-            toMerge[[datasetId]] <- readRDS(seuratObjects[[datasetId]])
+            toMerge[[datasetId]] <- readRDS(dat[[datasetId]])
         }
     }
 
     seuratObj <- CellMembrane::MergeSeuratObjs(toMerge, projectName = projectName, doGC = doDiet, errorOnBarcodeSuffix = errorOnBarcodeSuffix)
-    rm(toMerge)
-    saveData(seuratObj, projectName)
+    return(seuratObj)
 }
 
-# Cleanup
-rm(seuratObjects)
-gc()
+if (length(seuratObjects) == 1) {
+    print('There is only one seurat object, no need to merge')
+    datasetId <- names(seuratObjects)[[1]]
+    saveData(seuratObjects[[datasetId]], datasetId)
+} else {
+    batchSize <- 20
+    numBatches <- ceiling(length(seuratObjects) / batchSize)
+    mergedObjects <- list()
+    for (i in 1:numBatches) {
+        message(paste0('Merging batch ', i, ' of ', numBatches))
+        start <- 1 + (i-1)*batchSize
+        end <- min(start+batchSize-1, length(seuratObjects))
+        message(paste0('processing: ', start, ' to ', end, ' of ', length(seuratObjects)))
+
+        mergedObjects[[i]] <- mergeBatch(seuratObjects[start:end])
+        gc()
+    }
+
+    message('Done with batches')
+    if (length(mergedObjects) == 1) {
+        seuratObj <- mergedObjects[[1]]
+    } else {
+        message('performing final merge')
+        seuratObj <- merge(x = mergedObjects[[1]], y = mergedObjects[2:length(mergedObjects)], project = mergedObjects[[1]]@project.name)
+    }
+
+    rm(mergedObjects)
+    gc()
+
+    saveData(seuratObj, projectName)
+}