From 15e797305818e3e90f437a98d06e0e7a837cf384 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Tue, 15 Sep 2026 11:17:32 +0530 Subject: [PATCH 1/6] GetObjectAttributes ObjectParts should return Part entries for FSO buckets --- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 97 ++++++++++++++++--- .../s3/endpoint/ObjectAttributesHandler.java | 38 +++++--- .../s3/endpoint/TestObjectAttributesGet.java | 52 +++++++++- 3 files changed, 155 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index bfcf5b741bc3..ad7a6c17209c 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -1907,6 +1907,71 @@ public void testGetObjectAttributesMultipartObjectParts(@TempDir Path tempDir) t assertTrue(secondPage.objectParts().parts().isEmpty()); } + /** + * Directory (FSO layout) buckets return per-part {@code Part} elements even when no additional + * checksum was stored at upload time, matching AWS S3 directory-bucket behavior. + */ + @Test + public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir) + throws Exception { + final String bucketName = uniqueObjectName(); + final String keyName = getKeyName(); + final int partSize = (int) (5 * MB); + + createFsoBucket(bucketName); + try { + File multipartUploadFile = + Files.createFile(tempDir.resolve("fso-get-object-attributes-mpu.txt")).toFile(); + createFile(multipartUploadFile, (int) (15 * MB)); + multipartUpload(bucketName, keyName, multipartUploadFile, partSize, new HashMap<>(), + Collections.emptyList()); + + GetObjectAttributesResponse attributesResponse = s3Client.getObjectAttributes( + GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(keyName) + .objectAttributes(ObjectAttributes.OBJECT_PARTS, ObjectAttributes.OBJECT_SIZE) + .build()); + + assertNotNull(attributesResponse.objectParts()); + assertEquals(3, attributesResponse.objectParts().totalPartsCount()); + assertFalse(attributesResponse.objectParts().isTruncated()); + assertEquals(multipartUploadFile.length(), attributesResponse.objectSize()); + assertEquals(3, attributesResponse.objectParts().parts().size()); + for (int i = 0; i < 3; i++) { + assertEquals(i + 1, attributesResponse.objectParts().parts().get(i).partNumber()); + assertEquals((long) partSize, attributesResponse.objectParts().parts().get(i).size()); + } + + GetObjectAttributesResponse firstPage = s3Client.getObjectAttributes( + GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(keyName) + .objectAttributes(ObjectAttributes.OBJECT_PARTS) + .maxParts(2) + .partNumberMarker(0) + .build()); + assertTrue(firstPage.objectParts().isTruncated()); + assertEquals(2, firstPage.objectParts().parts().size()); + assertEquals(1, firstPage.objectParts().parts().get(0).partNumber()); + assertEquals(2, firstPage.objectParts().parts().get(1).partNumber()); + + GetObjectAttributesResponse secondPage = s3Client.getObjectAttributes( + GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(keyName) + .objectAttributes(ObjectAttributes.OBJECT_PARTS) + .maxParts(2) + .partNumberMarker(2) + .build()); + assertFalse(secondPage.objectParts().isTruncated()); + assertEquals(1, secondPage.objectParts().parts().size()); + assertEquals(3, secondPage.objectParts().parts().get(0).partNumber()); + } finally { + deleteFsoBucket(bucketName); + } + } + @Test public void testGetObjectAttributesNonContiguousMultipartObjectParts() throws Exception { final String bucketName = getBucketName(); @@ -3080,6 +3145,22 @@ private String getKeyName(String ignored) { return uniqueObjectName(); } + private void createFsoBucket(String bucketName) throws Exception { + try (OzoneClient ozoneClient = cluster.newClient()) { + OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); + volume.createBucket(bucketName, BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build()); + } + } + + private void deleteFsoBucket(String bucketName) throws Exception { + try (OzoneClient ozoneClient = cluster.newClient()) { + OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); + volume.deleteBucket(bucketName); + } + } + private String multipartUpload(String bucketName, String key, File file, int partSize, Map userMetadata, List tags) throws Exception { String uploadId = initiateMultipartUpload(bucketName, key, userMetadata, tags); @@ -4228,21 +4309,5 @@ public void testListBucketsIncludesFSOBuckets() throws Exception { deleteFsoBucket(fsoBucketName); } } - - private void createFsoBucket(String bucketName) throws Exception { - try (OzoneClient ozoneClient = cluster.newClient()) { - OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); - volume.createBucket(bucketName, BucketArgs.newBuilder() - .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .build()); - } - } - - private void deleteFsoBucket(String bucketName) throws Exception { - try (OzoneClient ozoneClient = cluster.newClient()) { - OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); - volume.deleteBucket(bucketName); - } - } } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java index 1a1319bc13ae..38242177fa4d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java @@ -57,10 +57,10 @@ *

Returns selected metadata about an object without transferring the object body. * Supported attributes: {@code ETag}, {@code ObjectSize}, {@code StorageClass}, {@code ObjectParts}. * - *

The {@code Checksum} attribute is not yet supported because Ozone does not store - * non-MD5 checksum algorithms in key metadata. For general-purpose buckets, {@code Part} - * elements under {@code ObjectParts} are omitted unless an additional checksum is stored - * on the object, matching AWS S3 behavior. Object versioning ({@code versionId}) and + *

For general-purpose buckets, individual {@code Part} elements under {@code ObjectParts} + * are omitted unless an additional checksum is stored on the object, matching AWS S3 behavior. + * For FSO layout buckets (Ozone directory buckets), {@code Part} elements are always returned, + * matching AWS S3 directory-bucket behavior. Object versioning ({@code versionId}) and * SSE-C encryption headers are also not supported and are silently ignored. * *

See https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html @@ -101,6 +101,15 @@ static boolean hasStoredAdditionalChecksum(OzoneKey key) { return false; } + /** + * Whether {@code ObjectParts} should include per-part {@code Part} elements in the response. + * Directory (FSO layout) buckets always include them; general-purpose buckets include them + * only when an additional checksum was stored at upload time. + */ + static boolean shouldIncludePartElements(OzoneKey key, boolean directoryBucketLayout) { + return directoryBucketLayout || hasStoredAdditionalChecksum(key); + } + @Override Response handleGetRequest(ObjectRequestContext context, String keyPath) throws IOException, OS3Exception { @@ -137,8 +146,12 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) throw ex; } + boolean directoryBucketLayout = + context.getBucket().getBucketLayout().isFileSystemOptimized(); + GetObjectAttributesResponse response = - buildResponse(keyPath, key, requestedAttributes, completedPartSizes); + buildResponse(keyPath, key, requestedAttributes, completedPartSizes, + directoryBucketLayout); Response.ResponseBuilder rb = Response.ok(response, MediaType.APPLICATION_XML_TYPE); ObjectEndpoint.addLastModifiedDate(rb, key); @@ -177,7 +190,8 @@ private Set parseAttributesHeader(String keyPath) throws OS3Exception { } private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, - Set requested, NavigableMap completedPartSizes) + Set requested, NavigableMap completedPartSizes, + boolean directoryBucketLayout) throws IOException, OS3Exception { GetObjectAttributesResponse resp = new GetObjectAttributesResponse(); @@ -205,7 +219,7 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, String partsCountStr = extractPartsCount(eTag); if (partsCountStr != null && completedPartSizes != null) { resp.setObjectParts(buildObjectParts(keyPath, Integer.parseInt(partsCountStr), - completedPartSizes, key)); + completedPartSizes, key, directoryBucketLayout)); } } } @@ -228,10 +242,12 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, * *

For general-purpose buckets, individual {@code Part} elements are returned only when the * object has a stored AWS additional checksum in key metadata; otherwise only {@code ObjectParts} - * summary and pagination fields are returned. + * summary and pagination fields are returned. For FSO layout (directory) buckets, {@code Part} + * elements are always returned. */ private GetObjectAttributesResponse.ObjectParts buildObjectParts(String keyPath, - int totalPartsCount, NavigableMap partSizes, OzoneKey key) + int totalPartsCount, NavigableMap partSizes, OzoneKey key, + boolean directoryBucketLayout) throws OS3Exception { int maxParts = parseMaxPartsHeader(keyPath); int marker = parsePartNumberMarkerHeader(keyPath); @@ -252,9 +268,7 @@ private GetObjectAttributesResponse.ObjectParts buildObjectParts(String keyPath, Iterator> partIterator = partSizes.tailMap(marker, false).entrySet().iterator(); - // TODO: For FSO (directory) buckets, always include Part entries per AWS - // directory-bucket GetObjectAttributes behavior, regardless of checksum metadata. - boolean includePartEntries = hasStoredAdditionalChecksum(key); + boolean includePartEntries = shouldIncludePartElements(key, directoryBucketLayout); Integer lastPartReturned = null; int partsOnPage = 0; while (partIterator.hasNext() && partsOnPage < maxParts) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java index 6c39ac89e1a0..0af6e4b105be 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java @@ -42,10 +42,14 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucketStub; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.S3Consts; @@ -60,6 +64,7 @@ public class TestObjectAttributesGet { private static final String CONTENT = "0123456789"; private static final String BUCKET_NAME = "b1"; + private static final String FSO_BUCKET_NAME = "fso-b1"; private static final String KEY_NAME = "key1"; private ObjectEndpoint rest; private OzoneBucket bucket; @@ -299,13 +304,52 @@ public void testGetObjectAttributesNonContiguousMultipartParts() throws IOExcept assertEquals(partThreeContent.length(), paginatedParts.getParts().get(0).getSize()); } - private void completeMultipartUploadWithParts(String key, String... partContents) + @Test + public void testGetObjectAttrFsoMPUPartsWithoutChecksum() throws IOException, OS3Exception { - String uploadID = initiateMultipartUpload(rest, BUCKET_NAME, key); + OzoneClient client = rest.getClient(); + String volumeName = rest.getOzoneConfiguration().get(OzoneConfigKeys.OZONE_S3_VOLUME_NAME, + OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + OzoneVolume volume = client.getObjectStore().getVolume(volumeName); + volume.createBucket(FSO_BUCKET_NAME, BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build()); + + final String key = "fso-mpu-key"; + completeMultipartUploadWithPartsInBucket(FSO_BUCKET_NAME, key, "part-one", "part-two"); + + Response response = getObjectAttributes(rest, FSO_BUCKET_NAME, key, "ObjectParts"); + + assertEquals(HTTP_OK, response.getStatus()); + GetObjectAttributesResponse.ObjectParts objectParts = + ((GetObjectAttributesResponse) response.getEntity()).getObjectParts(); + assertNotNull(objectParts); + assertEquals(2, objectParts.getPartsCount().intValue()); + assertEquals(2, objectParts.getParts().size()); + assertEquals(1, objectParts.getParts().get(0).getPartNumber()); + assertEquals("part-one".length(), objectParts.getParts().get(0).getSize()); + assertEquals(2, objectParts.getParts().get(1).getPartNumber()); + assertEquals("part-two".length(), objectParts.getParts().get(1).getSize()); + } + + @Test + public void testShouldIncludePartElements() { + assertFalse(ObjectAttributesHandler.shouldIncludePartElements(null, false)); + assertTrue(ObjectAttributesHandler.shouldIncludePartElements(null, true)); + } + + private void completeMultipartUploadWithPartsInBucket(String bucketName, String key, + String... partContents) throws IOException, OS3Exception { + String uploadID = initiateMultipartUpload(rest, bucketName, key); List partsList = new ArrayList<>(); for (int i = 0; i < partContents.length; i++) { - partsList.add(uploadPart(rest, BUCKET_NAME, key, i + 1, uploadID, partContents[i])); + partsList.add(uploadPart(rest, bucketName, key, i + 1, uploadID, partContents[i])); } - completeMultipartUpload(rest, BUCKET_NAME, key, uploadID, partsList); + completeMultipartUpload(rest, bucketName, key, uploadID, partsList); + } + + private void completeMultipartUploadWithParts(String key, String... partContents) + throws IOException, OS3Exception { + completeMultipartUploadWithPartsInBucket(BUCKET_NAME, key, partContents); } } From 2b86393089bcac6622ac21d3e734c313a2000810 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Tue, 15 Sep 2026 12:38:36 +0530 Subject: [PATCH 2/6] fix integration test failure --- .../apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index ad7a6c17209c..1cf08264772b 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -1968,6 +1968,7 @@ public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir) assertEquals(1, secondPage.objectParts().parts().size()); assertEquals(3, secondPage.objectParts().parts().get(0).partNumber()); } finally { + s3Client.deleteObject(b -> b.bucket(bucketName).key(keyName)); deleteFsoBucket(bucketName); } } From a8268400d8d2ab9850dd1b6dfe4d8df4fb7b61f0 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Thu, 17 Sep 2026 10:36:46 +0530 Subject: [PATCH 3/6] call context.getBucket() when ObjectParts is actually built --- .../ozone/s3/endpoint/ObjectAttributesHandler.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java index 38242177fa4d..fa2138c77b3e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java @@ -146,12 +146,8 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) throw ex; } - boolean directoryBucketLayout = - context.getBucket().getBucketLayout().isFileSystemOptimized(); - GetObjectAttributesResponse response = - buildResponse(keyPath, key, requestedAttributes, completedPartSizes, - directoryBucketLayout); + buildResponse(keyPath, key, requestedAttributes, completedPartSizes, context); Response.ResponseBuilder rb = Response.ok(response, MediaType.APPLICATION_XML_TYPE); ObjectEndpoint.addLastModifiedDate(rb, key); @@ -191,7 +187,7 @@ private Set parseAttributesHeader(String keyPath) throws OS3Exception { private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, Set requested, NavigableMap completedPartSizes, - boolean directoryBucketLayout) + ObjectRequestContext context) throws IOException, OS3Exception { GetObjectAttributesResponse resp = new GetObjectAttributesResponse(); @@ -218,6 +214,8 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, if (eTag != null) { String partsCountStr = extractPartsCount(eTag); if (partsCountStr != null && completedPartSizes != null) { + boolean directoryBucketLayout = + context.getBucket().getBucketLayout().isFileSystemOptimized(); resp.setObjectParts(buildObjectParts(keyPath, Integer.parseInt(partsCountStr), completedPartSizes, key, directoryBucketLayout)); } From c916cb3ff9d8b97379438b15b59e8c517d03eca6 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 23 Sep 2026 13:13:22 +0530 Subject: [PATCH 4/6] resolves layout in the getKeyInfo path --- .../ozone/client/S3HeadObjectAttributes.java | 10 +++++++- .../hadoop/ozone/client/rpc/RpcClient.java | 21 ++++++++++++++-- .../om/helpers/KeyInfoWithVolumeContext.java | 25 ++++++++++++++++++- .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/OmMetadataReader.java | 6 ++++- .../apache/hadoop/ozone/om/OmSnapshot.java | 1 + .../s3/endpoint/ObjectAttributesHandler.java | 10 +++++--- .../ozone/client/ClientProtocolStub.java | 2 +- .../s3/endpoint/TestObjectAttributesGet.java | 23 +++++++++++++++++ 9 files changed, 89 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java index 91a5ae701932..fd2a8e214ea9 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/S3HeadObjectAttributes.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.NavigableMap; import java.util.Objects; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; /** * Head metadata and completed multipart part sizes from a single S3 {@code GetKeyInfo} call. @@ -28,13 +29,16 @@ public final class S3HeadObjectAttributes { private final OzoneKey key; private final NavigableMap completedMultipartPartSizes; + private final BucketLayout bucketLayout; public S3HeadObjectAttributes(OzoneKey key, - NavigableMap completedMultipartPartSizes) { + NavigableMap completedMultipartPartSizes, + BucketLayout bucketLayout) { this.key = Objects.requireNonNull(key, "key == null"); this.completedMultipartPartSizes = completedMultipartPartSizes == null ? Collections.emptyNavigableMap() : completedMultipartPartSizes; + this.bucketLayout = bucketLayout == null ? BucketLayout.DEFAULT : bucketLayout; } public OzoneKey getKey() { @@ -44,4 +48,8 @@ public OzoneKey getKey() { public NavigableMap getCompletedMultipartPartSizes() { return completedMultipartPartSizes; } + + public BucketLayout getBucketLayout() { + return bucketLayout; + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 9c121bde4c89..54d4ea92c451 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1947,7 +1947,22 @@ public OzoneKey headS3Object(String bucketName, String keyName, @Override public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName, String keyName) throws IOException { - OmKeyInfo keyInfo = getS3KeyInfo(bucketName, keyName, true); + verifyBucketName(bucketName); + Objects.requireNonNull(keyName, "keyName == null"); + + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT) + .setBucketName(bucketName) + .setKeyName(keyName) + .setSortDatanodesInPipeline(topologyAwareReadEnabled) + .setLatestVersionLocation(getLatestVersionLocation) + .setForceUpdateContainerCacheFromSCM(false) + .setHeadOp(true) + .build(); + KeyInfoWithVolumeContext keyInfoWithS3Context = + ozoneManagerClient.getKeyInfo(keyArgs, true); + keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal); + OmKeyInfo keyInfo = keyInfoWithS3Context.getKeyInfo(); OmKeyLocationInfoGroup locationGroup = keyInfo.getLatestVersionLocations(); NavigableMap partSizes = Collections.emptyNavigableMap(); if (locationGroup != null && locationGroup.isMultipartKey()) { @@ -1959,7 +1974,9 @@ public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName, String k } } } - return new S3HeadObjectAttributes(OzoneKey.fromKeyInfo(keyInfo), partSizes); + BucketLayout bucketLayout = keyInfoWithS3Context.getBucketLayout() + .orElse(BucketLayout.DEFAULT); + return new S3HeadObjectAttributes(OzoneKey.fromKeyInfo(keyInfo), partSizes, bucketLayout); } private OmKeyInfo getS3PartOmKeyInfo(String bucketName, String keyName, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java index d6d54d3c174d..bc2a70942ac2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java @@ -37,12 +37,22 @@ public class KeyInfoWithVolumeContext { private final OmKeyInfo keyInfo; + private final Optional bucketLayout; + public KeyInfoWithVolumeContext(OmVolumeArgs volumeArgs, String userPrincipal, OmKeyInfo keyInfo) { + this(volumeArgs, userPrincipal, keyInfo, null); + } + + public KeyInfoWithVolumeContext(OmVolumeArgs volumeArgs, + String userPrincipal, + OmKeyInfo keyInfo, + BucketLayout bucketLayout) { this.volumeArgs = Optional.ofNullable(volumeArgs); this.userPrincipal = Optional.ofNullable(userPrincipal); this.keyInfo = keyInfo; + this.bucketLayout = Optional.ofNullable(bucketLayout); } public static KeyInfoWithVolumeContext fromProtobuf( @@ -52,6 +62,8 @@ public static KeyInfoWithVolumeContext fromProtobuf( OmVolumeArgs.getFromProtobuf(proto.getVolumeInfo()) : null) .setUserPrincipal(proto.getUserPrincipal()) .setKeyInfo(OmKeyInfo.getFromProtobuf(proto.getKeyInfo())) + .setBucketLayout(proto.hasBucketLayout() + ? BucketLayout.fromProto(proto.getBucketLayout()) : null) .build(); } @@ -60,6 +72,7 @@ public GetKeyInfoResponse toProtobuf(int clientVersion) { volumeArgs.ifPresent(v -> builder.setVolumeInfo(v.getProtobuf())); userPrincipal.ifPresent(builder::setUserPrincipal); builder.setKeyInfo(keyInfo.getProtobuf(clientVersion)); + bucketLayout.ifPresent(layout -> builder.setBucketLayout(layout.toProto())); return builder.build(); } @@ -75,6 +88,10 @@ public Optional getUserPrincipal() { return userPrincipal; } + public Optional getBucketLayout() { + return bucketLayout; + } + public static Builder newBuilder() { return new Builder(); } @@ -86,6 +103,7 @@ public static class Builder { private OmVolumeArgs volumeArgs; private String userPrincipal; private OmKeyInfo keyInfo; + private BucketLayout bucketLayout; public Builder setVolumeArgs(OmVolumeArgs volumeArgs) { this.volumeArgs = volumeArgs; @@ -102,8 +120,13 @@ public Builder setKeyInfo(OmKeyInfo keyInfo) { return this; } + public Builder setBucketLayout(BucketLayout layout) { + this.bucketLayout = layout; + return this; + } + public KeyInfoWithVolumeContext build() { - return new KeyInfoWithVolumeContext(volumeArgs, userPrincipal, keyInfo); + return new KeyInfoWithVolumeContext(volumeArgs, userPrincipal, keyInfo, bucketLayout); } } } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 3702b021ac20..65c7e041aebc 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1413,6 +1413,7 @@ message GetKeyInfoResponse { optional KeyInfo keyInfo = 1; optional VolumeInfo volumeInfo = 2; optional string UserPrincipal = 3; + optional BucketLayoutProto bucketLayout = 4; } message RenameKeysRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index 434d05132bf5..201b07553ef3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -31,6 +31,7 @@ import java.net.InetAddress; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -46,6 +47,7 @@ import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; @@ -198,7 +200,9 @@ public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs args, OmMetadataReader.getClientAddress()); KeyInfoWithVolumeContext.Builder builder = KeyInfoWithVolumeContext .newBuilder() - .setKeyInfo(keyInfo); + .setKeyInfo(keyInfo) + .setBucketLayout(Objects.requireNonNullElse( + bucket.bucketLayout(), BucketLayout.DEFAULT)); s3VolumeContext.ifPresent(context -> { builder.setVolumeArgs(context.getOmVolumeArgs()); builder.setUserPrincipal(context.getUserPrincipal()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java index 6d3a56f40ed0..070e85b02518 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java @@ -315,6 +315,7 @@ private KeyInfoWithVolumeContext denormalizeKeyInfoWithVolumeContext( .setKeyInfo(denormalizeOmKeyInfo(k.getKeyInfo())) .setVolumeArgs(k.getVolumeArgs().orElse(null)) .setUserPrincipal(k.getUserPrincipal().orElse(null)) + .setBucketLayout(k.getBucketLayout().orElse(null)) .build(); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java index fa2138c77b3e..8150720a86b3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java @@ -127,12 +127,15 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) OzoneKey key; NavigableMap completedPartSizes = null; + boolean directoryBucketLayout = false; try { if (requestedAttributes.contains(ATTR_OBJECT_PARTS)) { S3HeadObjectAttributes headAttributes = getClientProtocol().headS3ObjectAttributes(bucketName, keyPath); key = headAttributes.getKey(); completedPartSizes = headAttributes.getCompletedMultipartPartSizes(); + directoryBucketLayout = + headAttributes.getBucketLayout().isFileSystemOptimized(); } else { key = getClientProtocol().headS3Object(bucketName, keyPath); } @@ -147,7 +150,8 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) } GetObjectAttributesResponse response = - buildResponse(keyPath, key, requestedAttributes, completedPartSizes, context); + buildResponse(keyPath, key, requestedAttributes, completedPartSizes, + directoryBucketLayout); Response.ResponseBuilder rb = Response.ok(response, MediaType.APPLICATION_XML_TYPE); ObjectEndpoint.addLastModifiedDate(rb, key); @@ -187,7 +191,7 @@ private Set parseAttributesHeader(String keyPath) throws OS3Exception { private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, Set requested, NavigableMap completedPartSizes, - ObjectRequestContext context) + boolean directoryBucketLayout) throws IOException, OS3Exception { GetObjectAttributesResponse resp = new GetObjectAttributesResponse(); @@ -214,8 +218,6 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, if (eTag != null) { String partsCountStr = extractPartsCount(eTag); if (partsCountStr != null && completedPartSizes != null) { - boolean directoryBucketLayout = - context.getBucket().getBucketLayout().isFileSystemOptimized(); resp.setObjectParts(buildObjectParts(keyPath, Integer.parseInt(partsCountStr), completedPartSizes, key, directoryBucketLayout)); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 40e384d2c547..084733a0dae2 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -143,7 +143,7 @@ public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName, String k NavigableMap partSizes = bucket instanceof OzoneBucketStub ? ((OzoneBucketStub) bucket).getCompletedMultipartPartSizes(keyName) : Collections.emptyNavigableMap(); - return new S3HeadObjectAttributes(key, partSizes); + return new S3HeadObjectAttributes(key, partSizes, bucket.getBucketLayout()); } @Override diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java index 0af6e4b105be..a9c92b961847 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java @@ -38,16 +38,20 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucketStub; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; @@ -336,6 +340,25 @@ public void testGetObjectAttrFsoMPUPartsWithoutChecksum() public void testShouldIncludePartElements() { assertFalse(ObjectAttributesHandler.shouldIncludePartElements(null, false)); assertTrue(ObjectAttributesHandler.shouldIncludePartElements(null, true)); + + OzoneKey obsKeyNoChecksum = testOzoneKey(false); + assertFalse(ObjectAttributesHandler.shouldIncludePartElements(obsKeyNoChecksum, false)); + + OzoneKey obsKeyWithChecksum = testOzoneKey(true); + assertTrue(ObjectAttributesHandler.shouldIncludePartElements(obsKeyWithChecksum, false)); + + assertTrue(ObjectAttributesHandler.shouldIncludePartElements(obsKeyNoChecksum, true)); + assertTrue(ObjectAttributesHandler.shouldIncludePartElements(obsKeyWithChecksum, true)); + } + + private static OzoneKey testOzoneKey(boolean withAdditionalChecksum) { + OzoneKey key = new OzoneKey("vol", "b", "k", 0, 0, 0, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + true, "owner"); + if (withAdditionalChecksum) { + key.setMetadata(Collections.singletonMap("x-amz-checksum-crc32", "abc")); + } + return key; } private void completeMultipartUploadWithPartsInBucket(String bucketName, String key, From 0e5e46d00fade076aaffeb07d58542be8cec0cd1 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 23 Sep 2026 13:18:18 +0530 Subject: [PATCH 5/6] reduce duplicacy --- .../hadoop/ozone/client/rpc/RpcClient.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 54d4ea92c451..89753fb533df 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1947,21 +1947,8 @@ public OzoneKey headS3Object(String bucketName, String keyName, @Override public S3HeadObjectAttributes headS3ObjectAttributes(String bucketName, String keyName) throws IOException { - verifyBucketName(bucketName); - Objects.requireNonNull(keyName, "keyName == null"); - - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT) - .setBucketName(bucketName) - .setKeyName(keyName) - .setSortDatanodesInPipeline(topologyAwareReadEnabled) - .setLatestVersionLocation(getLatestVersionLocation) - .setForceUpdateContainerCacheFromSCM(false) - .setHeadOp(true) - .build(); KeyInfoWithVolumeContext keyInfoWithS3Context = - ozoneManagerClient.getKeyInfo(keyArgs, true); - keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal); + getS3KeyInfoWithVolumeContext(bucketName, keyName, true); OmKeyInfo keyInfo = keyInfoWithS3Context.getKeyInfo(); OmKeyLocationInfoGroup locationGroup = keyInfo.getLatestVersionLocations(); NavigableMap partSizes = Collections.emptyNavigableMap(); @@ -2002,6 +1989,12 @@ private OmKeyInfo getS3PartOmKeyInfo(String bucketName, String keyName, @Nonnull private OmKeyInfo getS3KeyInfo( String bucketName, String keyName, boolean isHeadOp) throws IOException { + return getS3KeyInfoWithVolumeContext(bucketName, keyName, isHeadOp).getKeyInfo(); + } + + @Nonnull + private KeyInfoWithVolumeContext getS3KeyInfoWithVolumeContext( + String bucketName, String keyName, boolean isHeadOp) throws IOException { verifyBucketName(bucketName); Objects.requireNonNull(keyName, "keyName == null"); @@ -2019,7 +2012,7 @@ private OmKeyInfo getS3KeyInfo( KeyInfoWithVolumeContext keyInfoWithS3Context = ozoneManagerClient.getKeyInfo(keyArgs, true); keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal); - return keyInfoWithS3Context.getKeyInfo(); + return keyInfoWithS3Context; } @Nonnull From 5aaf739ab50ee97c9a749dd5a4d1a3f9860849e1 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Mon, 5 Oct 2026 12:19:23 +0530 Subject: [PATCH 6/6] added regression test --- .../s3/endpoint/TestObjectAttributesGet.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java index a9c92b961847..343924f29ffa 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java @@ -35,24 +35,37 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucketStub; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.S3HeadObjectAttributes; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import org.apache.hadoop.ozone.s3.exception.OS3Exception; @@ -336,6 +349,64 @@ public void testGetObjectAttrFsoMPUPartsWithoutChecksum() assertEquals("part-two".length(), objectParts.getParts().get(1).getSize()); } + /** + * Regression: multipart {@code ObjectParts} must not call {@code context.getBucket()} + * (bucket READ). Layout and part sizes come from {@code headS3ObjectAttributes} only. + */ + @Test + public void testGetObjectAttributesObjectPartsWithoutBucketRead() + throws IOException, OS3Exception { + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OzoneConfigKeys.OZONE_S3_VOLUME_NAME, + OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + + ClientProtocol clientProtocol = mock(ClientProtocol.class); + OzoneClient client = mock(OzoneClient.class); + ObjectStore objectStore = mock(ObjectStore.class); + OzoneVolume volume = mock(OzoneVolume.class); + + when(client.getConfiguration()).thenReturn(ozoneConfiguration); + when(client.getObjectStore()).thenReturn(objectStore); + when(client.getProxy()).thenReturn(clientProtocol); + when(objectStore.getClientProxy()).thenReturn(clientProtocol); + when(objectStore.getS3Volume()).thenReturn(volume); + when(volume.getBucket(anyString())).thenThrow( + new OMException("bucket read denied", ResultCodes.ACCESS_DENIED)); + + final String mpuKey = "mpu-key"; + OzoneKey ozoneKey = new OzoneKey("s3v", BUCKET_NAME, mpuKey, 10, 0, 0, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + true, "owner"); + ozoneKey.setMetadata(Collections.singletonMap(OzoneConsts.ETAG, "\"abc-2\"")); + + NavigableMap partSizes = new TreeMap<>(); + partSizes.put(1, 5L); + partSizes.put(2, 5L); + S3HeadObjectAttributes headAttributes = new S3HeadObjectAttributes(ozoneKey, partSizes, + BucketLayout.OBJECT_STORE); + when(clientProtocol.headS3ObjectAttributes(BUCKET_NAME, mpuKey)).thenReturn(headAttributes); + + HttpHeaders headers = Mockito.mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("UNSIGNED-PAYLOAD"); + + ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder() + .setClient(client) + .setHeaders(headers) + .build(); + + Response response = getObjectAttributes(endpoint, BUCKET_NAME, mpuKey, "ObjectParts"); + + assertEquals(HTTP_OK, response.getStatus()); + GetObjectAttributesResponse.ObjectParts objectParts = + ((GetObjectAttributesResponse) response.getEntity()).getObjectParts(); + assertNotNull(objectParts); + assertEquals(2, objectParts.getPartsCount().intValue()); + assertTrue(objectParts.getParts().isEmpty()); + + verify(volume, never()).getBucket(anyString()); + verify(clientProtocol).headS3ObjectAttributes(BUCKET_NAME, mpuKey); + } + @Test public void testShouldIncludePartElements() { assertFalse(ObjectAttributesHandler.shouldIncludePartElements(null, false));