HDDS-16362. GetObjectAttributes ObjectParts should return Part entries for FSO buckets

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java

@@ -20,6 +20,7 @@
 import java.util.Collections;
 import java.util.NavigableMap;
 import java.util.Objects;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 
 /**
  * Head metadata and completed multipart part sizes from a single S3 {@code GetKeyInfo} call.
@@ -28,13 +29,16 @@ public final class S3HeadObjectAttributes {
 
   private final OzoneKey key;
   private final NavigableMap<Integer, Long> completedMultipartPartSizes;
+  private final BucketLayout bucketLayout;
 
   public S3HeadObjectAttributes(OzoneKey key,
-      NavigableMap<Integer, Long> completedMultipartPartSizes) {
+      NavigableMap<Integer, Long> completedMultipartPartSizes,
+      BucketLayout bucketLayout) {
     this.key = Objects.requireNonNull(key, "key == null");
     this.completedMultipartPartSizes = completedMultipartPartSizes == null
         ? Collections.emptyNavigableMap()
         : completedMultipartPartSizes;
+    this.bucketLayout = bucketLayout == null ? BucketLayout.DEFAULT : bucketLayout;
   }
 
   public OzoneKey getKey() {
@@ -44,4 +48,8 @@ public OzoneKey getKey() {
   public NavigableMap<Integer, Long> getCompletedMultipartPartSizes() {
     return completedMultipartPartSizes;
   }
+
+  public BucketLayout getBucketLayout() {
+    return bucketLayout;
+  }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -1955,7 +1955,9 @@ public OzoneKey headS3Object(String bucketName, String keyName,
   @Override
   public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName, String keyName)
       throws IOException {
-    OmKeyInfo keyInfo = getS3KeyInfo(bucketName, keyName, true);
+    KeyInfoWithVolumeContext keyInfoWithS3Context =
+        getS3KeyInfoWithVolumeContext(bucketName, keyName, true);
+    OmKeyInfo keyInfo = keyInfoWithS3Context.getKeyInfo();
     OmKeyLocationInfoGroup locationGroup = keyInfo.getLatestVersionLocations();
     NavigableMap<Integer, Long> partSizes = Collections.emptyNavigableMap();
     if (locationGroup != null && locationGroup.isMultipartKey()) {
@@ -1967,7 +1969,9 @@ public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName, String k
         }
       }
     }
-    return new S3HeadObjectAttributes(OzoneKey.fromKeyInfo(keyInfo), partSizes);
+    BucketLayout bucketLayout = keyInfoWithS3Context.getBucketLayout()
+        .orElse(BucketLayout.DEFAULT);
+    return new S3HeadObjectAttributes(OzoneKey.fromKeyInfo(keyInfo), partSizes, bucketLayout);
   }
 
   private OmKeyInfo getS3PartOmKeyInfo(String bucketName, String keyName,
@@ -1993,6 +1997,12 @@ private OmKeyInfo getS3PartOmKeyInfo(String bucketName, String keyName,
   @Nonnull
   private OmKeyInfo getS3KeyInfo(
       String bucketName, String keyName, boolean isHeadOp) throws IOException {
+    return getS3KeyInfoWithVolumeContext(bucketName, keyName, isHeadOp).getKeyInfo();
+  }
+
+  @Nonnull
+  private KeyInfoWithVolumeContext getS3KeyInfoWithVolumeContext(
+      String bucketName, String keyName, boolean isHeadOp) throws IOException {
     verifyBucketName(bucketName);
     Objects.requireNonNull(keyName, "keyName == null");
 
@@ -2010,7 +2020,7 @@ private OmKeyInfo getS3KeyInfo(
     KeyInfoWithVolumeContext keyInfoWithS3Context =
         ozoneManagerClient.getKeyInfo(keyArgs, true);
     keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal);
-    return keyInfoWithS3Context.getKeyInfo();
+    return keyInfoWithS3Context;
   }
 
   @Nonnull

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java

@@ -37,12 +37,22 @@ public class KeyInfoWithVolumeContext {
 
   private final OmKeyInfo keyInfo;
 
+  private final Optional<BucketLayout> bucketLayout;
+
   public KeyInfoWithVolumeContext(OmVolumeArgs volumeArgs,
                                   String userPrincipal,
                                   OmKeyInfo keyInfo) {
+    this(volumeArgs, userPrincipal, keyInfo, null);
+  }
+
+  public KeyInfoWithVolumeContext(OmVolumeArgs volumeArgs,
+                                  String userPrincipal,
+                                  OmKeyInfo keyInfo,
+                                  BucketLayout bucketLayout) {
     this.volumeArgs = Optional.ofNullable(volumeArgs);
     this.userPrincipal = Optional.ofNullable(userPrincipal);
     this.keyInfo = keyInfo;
+    this.bucketLayout = Optional.ofNullable(bucketLayout);
   }
 
   public static KeyInfoWithVolumeContext fromProtobuf(
@@ -52,6 +62,8 @@ public static KeyInfoWithVolumeContext fromProtobuf(
             OmVolumeArgs.getFromProtobuf(proto.getVolumeInfo()) : null)
         .setUserPrincipal(proto.getUserPrincipal())
         .setKeyInfo(OmKeyInfo.getFromProtobuf(proto.getKeyInfo()))
+        .setBucketLayout(proto.hasBucketLayout()
+            ? BucketLayout.fromProto(proto.getBucketLayout()) : null)
         .build();
   }
 
@@ -60,6 +72,7 @@ public GetKeyInfoResponse toProtobuf(int clientVersion) {
     volumeArgs.ifPresent(v -> builder.setVolumeInfo(v.getProtobuf()));
     userPrincipal.ifPresent(builder::setUserPrincipal);
     builder.setKeyInfo(keyInfo.getProtobuf(clientVersion));
+    bucketLayout.ifPresent(layout -> builder.setBucketLayout(layout.toProto()));
     return builder.build();
   }
 
@@ -75,6 +88,10 @@ public Optional<String> getUserPrincipal() {
     return userPrincipal;
   }
 
+  public Optional<BucketLayout> getBucketLayout() {
+    return bucketLayout;
+  }
+
   public static Builder newBuilder() {
     return new Builder();
   }
@@ -86,6 +103,7 @@ public static class Builder {
     private OmVolumeArgs volumeArgs;
     private String userPrincipal;
     private OmKeyInfo keyInfo;
+    private BucketLayout bucketLayout;
 
     public Builder setVolumeArgs(OmVolumeArgs volumeArgs) {
       this.volumeArgs = volumeArgs;
@@ -102,8 +120,13 @@ public Builder setKeyInfo(OmKeyInfo keyInfo) {
       return this;
     }
 
+    public Builder setBucketLayout(BucketLayout layout) {
+      this.bucketLayout = layout;
+      return this;
+    }
+
     public KeyInfoWithVolumeContext build() {
-      return new KeyInfoWithVolumeContext(volumeArgs, userPrincipal, keyInfo);
+      return new KeyInfoWithVolumeContext(volumeArgs, userPrincipal, keyInfo, bucketLayout);
     }
   }
 }

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -2243,6 +2243,72 @@ public void testGetObjectAttributesMultipartObjectParts(@TempDir Path tempDir) t
     assertTrue(secondPage.objectParts().parts().isEmpty());
   }
 
+  /**
+   * Directory (FSO layout) buckets return per-part {@code Part} elements even when no additional
+   * checksum was stored at upload time, matching AWS S3 directory-bucket behavior.
+   */
+  @Test
+  public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir)
+      throws Exception {
+    final String bucketName = uniqueObjectName();
+    final String keyName = getKeyName();
+    final int partSize = (int) (5 * MB);
+
+    createFsoBucket(bucketName);
+    try {
+      File multipartUploadFile =
+          Files.createFile(tempDir.resolve("fso-get-object-attributes-mpu.txt")).toFile();
+      createFile(multipartUploadFile, (int) (15 * MB));
+      multipartUpload(bucketName, keyName, multipartUploadFile, partSize, new HashMap<>(),
+          Collections.emptyList());
+
+      GetObjectAttributesResponse attributesResponse = s3Client.getObjectAttributes(
+          GetObjectAttributesRequest.builder()
+              .bucket(bucketName)
+              .key(keyName)
+              .objectAttributes(ObjectAttributes.OBJECT_PARTS, ObjectAttributes.OBJECT_SIZE)
+              .build());
+
+      assertNotNull(attributesResponse.objectParts());
+      assertEquals(3, attributesResponse.objectParts().totalPartsCount());
+      assertFalse(attributesResponse.objectParts().isTruncated());
+      assertEquals(multipartUploadFile.length(), attributesResponse.objectSize());
+      assertEquals(3, attributesResponse.objectParts().parts().size());
+      for (int i = 0; i < 3; i++) {
+        assertEquals(i + 1, attributesResponse.objectParts().parts().get(i).partNumber());
+        assertEquals((long) partSize, attributesResponse.objectParts().parts().get(i).size());
+      }
+
+      GetObjectAttributesResponse firstPage = s3Client.getObjectAttributes(
+          GetObjectAttributesRequest.builder()
+              .bucket(bucketName)
+              .key(keyName)
+              .objectAttributes(ObjectAttributes.OBJECT_PARTS)
+              .maxParts(2)
+              .partNumberMarker(0)
+              .build());
+      assertTrue(firstPage.objectParts().isTruncated());
+      assertEquals(2, firstPage.objectParts().parts().size());
+      assertEquals(1, firstPage.objectParts().parts().get(0).partNumber());
+      assertEquals(2, firstPage.objectParts().parts().get(1).partNumber());
+
+      GetObjectAttributesResponse secondPage = s3Client.getObjectAttributes(
+          GetObjectAttributesRequest.builder()
+              .bucket(bucketName)
+              .key(keyName)
+              .objectAttributes(ObjectAttributes.OBJECT_PARTS)
+              .maxParts(2)
+              .partNumberMarker(2)
+              .build());
+      assertFalse(secondPage.objectParts().isTruncated());
+      assertEquals(1, secondPage.objectParts().parts().size());
+      assertEquals(3, secondPage.objectParts().parts().get(0).partNumber());
+    } finally {
+      s3Client.deleteObject(b -> b.bucket(bucketName).key(keyName));
+      deleteFsoBucket(bucketName);
+    }
+  }
+
   @Test
   public void testGetObjectAttributesNonContiguousMultipartObjectParts() throws Exception {
     final String bucketName = getBucketName();
@@ -3443,6 +3509,22 @@ private static void assertNotImplemented(Executable request) {
     assertEquals(S3ErrorTable.NOT_IMPLEMENTED.getErrorMessage(), exception.awsErrorDetails().errorMessage());
   }
 
+  private void createFsoBucket(String bucketName) throws Exception {
+    try (OzoneClient ozoneClient = cluster.newClient()) {
+      OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+      volume.createBucket(bucketName, BucketArgs.newBuilder()
+          .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+          .build());
+    }
+  }
+
+  private void deleteFsoBucket(String bucketName) throws Exception {
+    try (OzoneClient ozoneClient = cluster.newClient()) {
+      OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+      volume.deleteBucket(bucketName);
+    }
+  }
+
   private String multipartUpload(String bucketName, String key, File file, int partSize,
                                  Map<String, String> userMetadata, List<Tag> tags) throws Exception {
     String uploadId = initiateMultipartUpload(bucketName, key, userMetadata, tags);
@@ -4591,21 +4673,5 @@ public void testListBucketsIncludesFSOBuckets() throws Exception {
         deleteFsoBucket(fsoBucketName);
       }
     }
-
-    private void createFsoBucket(String bucketName) throws Exception {
-      try (OzoneClient ozoneClient = cluster.newClient()) {
-        OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
-        volume.createBucket(bucketName, BucketArgs.newBuilder()
-            .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
-            .build());
-      }
-    }
-
-    private void deleteFsoBucket(String bucketName) throws Exception {
-      try (OzoneClient ozoneClient = cluster.newClient()) {
-        OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
-        volume.deleteBucket(bucketName);
-      }
-    }
   }
 }

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -1419,6 +1419,7 @@ message GetKeyInfoResponse {
   optional KeyInfo keyInfo = 1;
   optional VolumeInfo volumeInfo = 2;
   optional string UserPrincipal = 3;
+  optional BucketLayoutProto bucketLayout = 4;
 }
 
 message RenameKeysRequest {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java

@@ -31,6 +31,7 @@
 import java.net.InetAddress;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -46,6 +47,7 @@
 import org.apache.hadoop.ozone.audit.OMAction;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
 import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
@@ -199,7 +201,9 @@ public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs args,
               OmMetadataReader.getClientAddress());
       KeyInfoWithVolumeContext.Builder builder = KeyInfoWithVolumeContext
           .newBuilder()
-          .setKeyInfo(keyInfo);
+          .setKeyInfo(keyInfo)
+          .setBucketLayout(Objects.requireNonNullElse(
+              bucket.bucketLayout(), BucketLayout.DEFAULT));
       s3VolumeContext.ifPresent(context -> {
         builder.setVolumeArgs(context.getOmVolumeArgs());
         builder.setUserPrincipal(context.getUserPrincipal());

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java

@@ -323,6 +323,7 @@ private KeyInfoWithVolumeContext denormalizeKeyInfoWithVolumeContext(
         .setKeyInfo(denormalizeOmKeyInfo(k.getKeyInfo()))
         .setVolumeArgs(k.getVolumeArgs().orElse(null))
         .setUserPrincipal(k.getUserPrincipal().orElse(null))
+        .setBucketLayout(k.getBucketLayout().orElse(null))
         .build();
   }