Improving performance of bulk_get. When retrieving an object with a name ending in '.json', perform the slightly less expensive GetObjectsDetails (plural) call as opposed to the GetObjectsWithFullDetails call. When retrieving all other objects, perform the inexpensive GetObjectDetails (singular) call. Retrieving objects whose name ends in '.json' will incur a performance hit, but this bypasses the json payload format issue that otherwise prevents response payload unmarshaling.

Author: RachelTucker

build.gradle

@@ -32,7 +32,7 @@ buildscript {
 }
 
 allprojects {
-    version = '5.1.3'
+    version = '5.1.4'
     group = 'com.spectralogic'
 
     repositories {

ds3_java_cli/src/main/kotlin/com/spectralogic/ds3cli/PipeUtils.kt

@@ -18,7 +18,8 @@ package com.spectralogic.ds3cli
 import com.google.common.collect.ImmutableList
 import com.spectralogic.ds3cli.exceptions.CommandException
 import com.spectralogic.ds3client.Ds3Client
-import com.spectralogic.ds3client.commands.spectrads3.GetObjectsWithFullDetailsSpectraS3Request
+import com.spectralogic.ds3client.commands.spectrads3.GetObjectDetailsSpectraS3Request
+import com.spectralogic.ds3client.commands.spectrads3.GetObjectsDetailsSpectraS3Request
 import com.spectralogic.ds3client.models.Contents
 import com.spectralogic.ds3client.networking.FailedRequestException
 import kotlinx.coroutines.*
@@ -38,17 +39,28 @@ object PipeUtils {
                  pipedFileNames.map { objName ->
                     async(Dispatchers.IO) {
                         try {
-                            val result = client.getObjectsWithFullDetailsSpectraS3(
-                                    GetObjectsWithFullDetailsSpectraS3Request().withBucketId(bucketName).withName(objName));
-
-                            if (result.detailedS3ObjectListResult.detailedS3Objects == null || result.detailedS3ObjectListResult.detailedS3Objects.size == 0) {
-                                GetObjectResponse.NotFound(objName)
-                            } else if (result.detailedS3ObjectListResult.detailedS3Objects.size > 1) {
-                                GetObjectResponse.Error(IOException("There are multiple versions of object with name: $objName"))
+                            if (objName.endsWith(".json", ignoreCase = true)) {
+                                // If this is an object that ends in .json, then perform the more expensive call that
+                                // bypasses the json formatting feature of response payloads in BP
+                                val result = client.getObjectsDetailsSpectraS3(
+                                        GetObjectsDetailsSpectraS3Request().withBucketId(bucketName).withName(objName).withLatest(true))
+
+                                if (result.s3ObjectListResult.s3Objects == null || result.s3ObjectListResult.s3Objects.size == 0) {
+                                    GetObjectResponse.NotFound(objName)
+                                } else if (result.s3ObjectListResult.s3Objects.size > 1) {
+                                    GetObjectResponse.Error(IOException("There are multiple versions of object with name: $objName"))
+                                } else {
+                                    GetObjectResponse.Success(Contents().apply {
+                                        key = result.s3ObjectListResult.s3Objects[0].name
+                                        lastModified = result.s3ObjectListResult.s3Objects[0].creationDate
+                                    })
+                                }
                             } else {
+                                val result = client.getObjectDetailsSpectraS3(GetObjectDetailsSpectraS3Request(objName, bucketName)).s3ObjectResult
+
                                 GetObjectResponse.Success(Contents().apply {
-                                    key = result.detailedS3ObjectListResult.detailedS3Objects[0].name
-                                    lastModified = result.detailedS3ObjectListResult.detailedS3Objects[0].creationDate
+                                    key = result.name
+                                    lastModified = result.creationDate
                                 })
                             }
                         } catch (f: FailedRequestException) {