From 9f32edcba868d4efed2e89369efe6f54c5b42e03 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Aug 2025 07:18:38 +0800 Subject: [PATCH 1/7] 1 2 3 1 1 1 1 5 1 --- .../metastore/IcebergRestProperties.java | 48 +++++ .../AbstractS3CompatibleProperties.java | 7 + .../property/storage/S3Properties.java | 33 +++- .../metastore/IcebergRestPropertiesTest.java | 185 ++++++++++++++++++ .../property/metastore/RestCatalogTest.java | 110 +++++++++++ .../property/storage/S3PropertiesTest.java | 15 ++ 6 files changed, 395 insertions(+), 3 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java index f59f2138052faa..c55611f47ddb43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java @@ -130,6 +130,32 @@ public class IcebergRestProperties extends AbstractIcebergProperties { description = "The cache TTL for case insensitive name matching in ms.") private String icebergRestCaseInsensitiveNameMatchingCacheTtlMs = "0"; + // The following properties are specific to AWS Glue Rest Catalog + @ConnectorProperty(names = {"iceberg.rest.sigv4-enabled"}, + required = false, + description = "True for Glue Rest Catalog") + private String icebergRestSigV4Enabled = ""; + + @ConnectorProperty(names = {"iceberg.rest.signing-name"}, + required = false, + description = "The signing name for the iceberg rest catalog service.") + private String icebergRestSigningName = ""; + + @ConnectorProperty(names = {"iceberg.rest.signing-region"}, + required = false, + description = "The signing region for the iceberg rest catalog service.") + private String icebergRestSigningRegion = ""; + + @ConnectorProperty(names = {"iceberg.rest.access-key-id"}, + required = false, + description = "The access key ID for the iceberg rest catalog service.") + private String icebergRestAccessKeyId = ""; + + @ConnectorProperty(names = {"iceberg.rest.secret-access-key"}, + required = false, + description = "The secret access key for the iceberg rest catalog service.") + private String icebergRestSecretAccessKey = ""; + protected IcebergRestProperties(Map props) { super(props); } @@ -196,6 +222,15 @@ private ParamRules buildRules() { throw new IllegalArgumentException("OAuth2 requires either credential or token"); } } + + // Check for glue rest catalog specific properties + rules.requireIf(icebergRestSigningName, "glue", + new String[] {icebergRestSigningRegion, + icebergRestAccessKeyId, + icebergRestSecretAccessKey, + icebergRestSigV4Enabled}, + "Rest Catalog requires signing-region, access-key-id, secret-access-key " + + "and sigv4-enabled set to true when signing-name is glue"); return rules; } @@ -207,6 +242,8 @@ private void initIcebergRestCatalogProperties() { addOptionalProperties(); // Authentication properties addAuthenticationProperties(); + // Glue Rest Catalog specific properties + addGlueRestCatalogProperties(); } private void addCoreCatalogProperties() { @@ -253,6 +290,17 @@ private void addOAuth2Properties() { } } + private void addGlueRestCatalogProperties() { + if (Strings.isNotBlank(icebergRestSigningName) && icebergRestSigningName.equalsIgnoreCase("glue")) { + icebergRestCatalogProperties.put("rest.signing-name", "glue"); + icebergRestCatalogProperties.put("rest.sigv4-enabled", icebergRestSigV4Enabled); + icebergRestCatalogProperties.put("rest.access-key-id", icebergRestAccessKeyId); + icebergRestCatalogProperties.put("rest.secret-access-key", icebergRestSecretAccessKey); + icebergRestCatalogProperties.put("rest.signing-region", icebergRestSigningRegion); + } + } + + public Map getIcebergRestCatalogProperties() { return Collections.unmodifiableMap(icebergRestCatalogProperties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index b36e4309701b6c..c181a933388fc1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -239,12 +239,19 @@ private void setEndpointIfNotSet() { return; } String endpoint = S3PropertyUtils.constructEndpointFromUrl(origProps, usePathStyle, forceParsingByStandardUrl); + if (StringUtils.isBlank(endpoint)) { + endpoint = getEndpointFromRegion(); + } if (StringUtils.isBlank(endpoint)) { throw new IllegalArgumentException("endpoint is required"); } setEndpoint(endpoint); } + protected String getEndpointFromRegion() { + return ""; + } + @Override public String validateAndNormalizeUri(String uri) throws UserException { return S3PropertyUtils.validateAndNormalizeUri(uri, getUsePathStyle(), getForceParsingByStandardUrl()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 19a9195ec2ff41..992afdeef73f30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -50,6 +50,11 @@ public class S3Properties extends AbstractS3CompatibleProperties { "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", "aws.glue.endpoint" }; + + private static final String[] REGION_NAMES = { + "s3.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region" + }; + @Setter @Getter @ConnectorProperty(names = {"s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", @@ -61,21 +66,21 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Setter @Getter @ConnectorProperty(names = {"s3.region", "AWS_REGION", "region", "REGION", "aws.region", "glue.region", - "aws.glue.region"}, + "aws.glue.region", "iceberg.rest.signing-region"}, required = false, description = "The region of S3.") protected String region = ""; @Getter @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY", "glue.access_key", - "aws.glue.access-key", "client.credentials-provider.glue.access_key"}, + "aws.glue.access-key", "client.credentials-provider.glue.access_key", "iceberg.rest.access-key-id"}, required = false, description = "The access key of S3. Optional for anonymous access to public datasets.") protected String accessKey = ""; @Getter @ConnectorProperty(names = {"s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY", "glue.secret_key", - "aws.glue.secret-key", "client.credentials-provider.glue.secret_key"}, + "aws.glue.secret-key", "client.credentials-provider.glue.secret_key", "iceberg.rest.secret-access-key"}, required = false, description = "The secret key of S3. Optional for anonymous access to public datasets.") protected String secretKey = ""; @@ -203,6 +208,16 @@ protected static boolean guessIsMe(Map origProps) { if (!Strings.isNullOrEmpty(endpoint)) { return endpoint.contains("amazonaws.com"); } + + // guess from region + String region = Stream.of(REGION_NAMES) + .map(origProps::get) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + if (!Strings.isNullOrEmpty(region)) { + return true; + } Optional uriValue = origProps.entrySet().stream() .filter(e -> e.getKey().equalsIgnoreCase("uri")) .map(Map.Entry::getValue) @@ -278,4 +293,16 @@ public void initializeHadoopStorageConfig() { "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider"); } } + + @Override + protected String getEndpointFromRegion() { + if (!StringUtils.isBlank(endpoint)) { + return endpoint; + } + if (StringUtils.isBlank(region)) { + return ""; + } + return "https://s3." + region + ".amazonaws.com"; + } } + diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java index 16aae778adaa93..5dc0c6e9670489 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java @@ -231,4 +231,189 @@ public void testImmutablePropertiesMap() { catalogProps.put("test", "value"); }); } + + @Test + public void testGlueRestCatalogValidConfiguration() { + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "glue"); + props.put("iceberg.rest.signing-region", "us-east-1"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("iceberg.rest.sigv4-enabled", "true"); + + IcebergRestProperties restProps = new IcebergRestProperties(props); + restProps.initNormalizeAndCheckProps(); + + Map catalogProps = restProps.getIcebergRestCatalogProperties(); + Assertions.assertEquals("glue", catalogProps.get("iceberg.rest.signing-name")); + Assertions.assertEquals("us-east-1", catalogProps.get("iceberg.rest.signing-region")); + Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE", catalogProps.get("iceberg.rest.access-key-id")); + Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + catalogProps.get("iceberg.rest.secret-access-key")); + Assertions.assertEquals("true", catalogProps.get("iceberg.rest.sigv4-enabled")); + } + + @Test + public void testGlueRestCatalogCaseInsensitive() { + // Test that "GLUE" is also recognized (case insensitive) + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "GLUE"); + props.put("iceberg.rest.signing-region", "us-west-2"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("iceberg.rest.sigv4-enabled", "true"); + + IcebergRestProperties restProps = new IcebergRestProperties(props); + restProps.initNormalizeAndCheckProps(); + + Map catalogProps = restProps.getIcebergRestCatalogProperties(); + Assertions.assertEquals("glue", catalogProps.get("iceberg.rest.signing-name")); + Assertions.assertEquals("us-west-2", catalogProps.get("iceberg.rest.signing-region")); + } + + @Test + public void testGlueRestCatalogMissingSigningRegion() { + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "glue"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("iceberg.rest.sigv4-enabled", "true"); + // Missing signing-region + + IcebergRestProperties restProps = new IcebergRestProperties(props); + Assertions.assertThrows(IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + } + + @Test + public void testGlueRestCatalogMissingAccessKeyId() { + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "glue"); + props.put("iceberg.rest.signing-region", "us-east-1"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("iceberg.rest.sigv4-enabled", "true"); + // Missing access-key-id + + IcebergRestProperties restProps = new IcebergRestProperties(props); + Assertions.assertThrows(IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + } + + @Test + public void testGlueRestCatalogMissingSecretAccessKey() { + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "glue"); + props.put("iceberg.rest.signing-region", "us-east-1"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.sigv4-enabled", "true"); + // Missing secret-access-key + + IcebergRestProperties restProps = new IcebergRestProperties(props); + Assertions.assertThrows(IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + } + + @Test + public void testGlueRestCatalogMissingSigV4Enabled() { + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "glue"); + props.put("iceberg.rest.signing-region", "us-east-1"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + // Missing sigv4-enabled + + IcebergRestProperties restProps = new IcebergRestProperties(props); + Assertions.assertThrows(IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + } + + @Test + public void testNonGlueSigningNameDoesNotRequireAdditionalProperties() { + // Test that non-glue signing names don't require additional properties + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "custom-service"); + + IcebergRestProperties restProps = new IcebergRestProperties(props); + restProps.initNormalizeAndCheckProps(); // Should not throw + + Map catalogProps = restProps.getIcebergRestCatalogProperties(); + // Should not contain glue-specific properties + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-name")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-region")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.access-key-id")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.secret-access-key")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.sigv4-enabled")); + } + + @Test + public void testEmptySigningNameDoesNotAddGlueProperties() { + // Test that empty signing name doesn't add glue properties + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-region", "us-east-1"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("iceberg.rest.sigv4-enabled", "true"); + + IcebergRestProperties restProps = new IcebergRestProperties(props); + restProps.initNormalizeAndCheckProps(); // Should not throw + + Map catalogProps = restProps.getIcebergRestCatalogProperties(); + // Should not contain glue-specific properties since signing-name is not "glue" + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-name")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-region")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.access-key-id")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.secret-access-key")); + Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.sigv4-enabled")); + } + + @Test + public void testGlueRestCatalogWithOAuth2() { + // Test that Glue properties can be combined with OAuth2 + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.security.type", "oauth2"); + props.put("iceberg.rest.oauth2.token", "my-access-token"); + props.put("iceberg.rest.signing-name", "glue"); + props.put("iceberg.rest.signing-region", "us-east-1"); + props.put("iceberg.rest.access-key-id", "AKIAIOSFODNN7EXAMPLE"); + props.put("iceberg.rest.secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + props.put("iceberg.rest.sigv4-enabled", "true"); + + IcebergRestProperties restProps = new IcebergRestProperties(props); + restProps.initNormalizeAndCheckProps(); + + Map catalogProps = restProps.getIcebergRestCatalogProperties(); + // Should have both OAuth2 and Glue properties + Assertions.assertEquals("my-access-token", catalogProps.get(OAuth2Properties.TOKEN)); + Assertions.assertEquals("glue", catalogProps.get("iceberg.rest.signing-name")); + Assertions.assertEquals("us-east-1", catalogProps.get("iceberg.rest.signing-region")); + Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE", catalogProps.get("iceberg.rest.access-key-id")); + Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + catalogProps.get("iceberg.rest.secret-access-key")); + Assertions.assertEquals("true", catalogProps.get("iceberg.rest.sigv4-enabled")); + } + + @Test + public void testGlueRestCatalogMissingMultipleProperties() { + // Test error message when multiple required properties are missing + Map props = new HashMap<>(); + props.put("iceberg.rest.uri", "http://localhost:8080"); + props.put("iceberg.rest.signing-name", "glue"); + // Missing all required properties + + IcebergRestProperties restProps = new IcebergRestProperties(props); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, restProps::initNormalizeAndCheckProps); + + // The error message should mention the required properties + String errorMessage = exception.getMessage(); + Assertions.assertTrue(errorMessage.contains("signing-region") || + errorMessage.contains("access-key-id") || + errorMessage.contains("secret-access-key") || + errorMessage.contains("sigv4-enabled")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java new file mode 100644 index 00000000000000..8ccad6ae7f07c2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.metastore; + +import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Table; +import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.io.CloseableIterable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public class RestCatalogTest { + + // set your AWS access key and secret key + private String ak = ""; + private String sk = ""; + + @BeforeEach + protected void initCatalog() { + } + + private Catalog initGlueRestCatalog() { + Map options = Maps.newHashMap(); + options.put(CatalogUtil.ICEBERG_CATALOG_TYPE, CatalogUtil.ICEBERG_CATALOG_TYPE_REST); + options.put(CatalogProperties.URI, "https://glue.ap-east-1.amazonaws.com/iceberg"); + options.put(CatalogProperties.WAREHOUSE_LOCATION, "169698404049:s3tablescatalog/s3-table-bucket-hk-glue-test"); + // remove this endpoint prop, or, add https:// + options.put(S3FileIOProperties.ENDPOINT, "https://s3.ap-east-1.amazonaws.com"); + // must set: + // software.amazon.awssdk.core.exception.SdkClientException: Unable to load region from any of the providers in + // the chain software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain@627ff1b8: + // [software.amazon.awssdk.regions.providers.SystemSettingsRegionProvider@67d32a54: + // Unable to load region from system settings. Region must be specified either via environment variable + // (AWS_REGION) or system property (aws.region)., + // software.amazon.awssdk.regions.providers.AwsProfileRegionProvider@2792b416: No region provided in profile: + // default, software.amazon.awssdk.regions.providers.InstanceProfileRegionProvider@5cff6b74: + // Unable to contact EC2 metadata service.] + options.put(AwsClientProperties.CLIENT_REGION, "ap-east-1"); + // Forbidden: {"message":"Missing Authentication Token"} + options.put("rest.sigv4-enabled", "true"); + // Forbidden: {"message":"Credential should be scoped to correct service: 'glue'. "} + options.put("rest.signing-name", "glue"); + // Forbidden: {"message":"The security token included in the request is invalid."} + options.put("rest.access-key-id", ak); + // Forbidden: {"message":"The request signature we calculated does not match the signature you provided. + // Check your AWS Secret Access Key and signing method. Consult the service documentation for details."} + options.put("rest.secret-access-key", sk); + // same as AwsClientProperties.CLIENT_REGION, "ap-east-1" + options.put("rest.signing-region", "ap-east-1"); + // options.put("iceberg.catalog.warehouse", ":s3tablescatalog/"); + // 4. Build iceberg catalog + Configuration conf = new Configuration(); + return CatalogUtil.buildIcebergCatalog("glue_test", options, conf); + } + + @Test + public void testGlueRestCatalog() { + Catalog glueRestCatalog = initGlueRestCatalog(); + SupportsNamespaces nsCatalog = (SupportsNamespaces) glueRestCatalog; + // List namespaces and assert + nsCatalog.listNamespaces(Namespace.empty()).forEach(namespace1 -> { + System.out.println("Namespace: " + namespace1); + Assertions.assertNotNull(namespace1, "Namespace should not be null"); + + glueRestCatalog.listTables(namespace1).forEach(tableIdentifier -> { + System.out.println("Table: " + tableIdentifier.name()); + Assertions.assertNotNull(tableIdentifier, "TableIdentifier should not be null"); + + // Load table history and assert + Table iceTable = glueRestCatalog.loadTable(tableIdentifier); + iceTable.history().forEach(snapshot -> { + System.out.println("Snapshot: " + snapshot); + Assertions.assertNotNull(snapshot, "Snapshot should not be null"); + }); + + CloseableIterable tasks = iceTable.newScan().planFiles(); + tasks.forEach(task -> { + System.out.println("FileScanTask: " + task); + Assertions.assertNotNull(task, "FileScanTask should not be null"); + }); + }); + }); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index d81f51485730e6..23e3c99e309ac9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -20,6 +20,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; +import com.google.common.collect.Maps; import mockit.Expectations; import mockit.Mocked; import org.junit.jupiter.api.Assertions; @@ -383,4 +384,18 @@ public void testGetBackendConfigPropertiesWithEmptyRuntimeCredentials() throws U Assertions.assertEquals("base-access-key", result.get("AWS_ACCESS_KEY")); Assertions.assertEquals("base-secret-key", result.get("AWS_SECRET_KEY")); } + + @Test + public void testS3PropertiesFromIcebergRest() throws UserException { + Map props = Maps.newHashMap(); + props.put("iceberg.rest.access-key-id", "aaa"); + props.put("iceberg.rest.secret-access-key", "bbb"); + props.put("iceberg.rest.signing-region", "ap-east-1"); + + S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(props); + Assertions.assertEquals("ap-east-1", s3Properties.region); + Assertions.assertEquals("https://s3.ap-east-1.amazonaws.com", s3Properties.endpoint); + Assertions.assertEquals("aaa", s3Properties.accessKey); + Assertions.assertEquals("bbb", s3Properties.secretKey); + } } From 08a1f22807e9866e36700fa40ad8c917d68b14be Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Aug 2025 10:38:10 -0700 Subject: [PATCH 2/7] fix format --- .../property/metastore/IcebergRestPropertiesTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java index 5dc0c6e9670489..588530afa281f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java @@ -411,9 +411,9 @@ public void testGlueRestCatalogMissingMultipleProperties() { // The error message should mention the required properties String errorMessage = exception.getMessage(); - Assertions.assertTrue(errorMessage.contains("signing-region") || - errorMessage.contains("access-key-id") || - errorMessage.contains("secret-access-key") || - errorMessage.contains("sigv4-enabled")); + Assertions.assertTrue(errorMessage.contains("signing-region") + || errorMessage.contains("access-key-id") + || errorMessage.contains("secret-access-key") + || errorMessage.contains("sigv4-enabled")); } } From 35a21fc95380e963da9c65b793cf7bd816eb8209 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Aug 2025 12:23:47 -0700 Subject: [PATCH 3/7] 1 --- .../property/storage/AbstractS3CompatibleProperties.java | 3 +++ .../doris/datasource/property/storage/S3Properties.java | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index c181a933388fc1..4726fd53ca0d25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -248,6 +248,9 @@ private void setEndpointIfNotSet() { setEndpoint(endpoint); } + // This method should be overridden by subclasses to provide a default endpoint based on the region. + // Because for aws s3, only region is needed, the endpoint can be constructed from the region. + // But for other s3 compatible storage, the endpoint may need to be specified explicitly. protected String getEndpointFromRegion() { return ""; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 992afdeef73f30..2934708eebcf22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -46,12 +46,12 @@ public class S3Properties extends AbstractS3CompatibleProperties { - private static final String[] ENDPOINT_NAMES = { + private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", "aws.glue.endpoint" }; - private static final String[] REGION_NAMES = { + private static final String[] REGION_NAMES_FOR_GUESSING = { "s3.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region" }; @@ -194,7 +194,7 @@ public void initNormalizeAndCheckProps() { * @return */ protected static boolean guessIsMe(Map origProps) { - String endpoint = Stream.of(ENDPOINT_NAMES) + String endpoint = Stream.of(ENDPOINT_NAMES_FOR_GUESSING) .map(origProps::get) .filter(Objects::nonNull) .findFirst() @@ -210,7 +210,7 @@ protected static boolean guessIsMe(Map origProps) { } // guess from region - String region = Stream.of(REGION_NAMES) + String region = Stream.of(REGION_NAMES_FOR_GUESSING) .map(origProps::get) .filter(Objects::nonNull) .findFirst() From 5e40dd5b23b16306595bcc1c621985f7b324f945 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Aug 2025 21:38:29 -0700 Subject: [PATCH 4/7] fix ut --- .../metastore/IcebergRestPropertiesTest.java | 44 +++++++++---------- .../property/metastore/RestCatalogTest.java | 2 + 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java index 588530afa281f5..1d550e061cc445 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergRestPropertiesTest.java @@ -246,12 +246,12 @@ public void testGlueRestCatalogValidConfiguration() { restProps.initNormalizeAndCheckProps(); Map catalogProps = restProps.getIcebergRestCatalogProperties(); - Assertions.assertEquals("glue", catalogProps.get("iceberg.rest.signing-name")); - Assertions.assertEquals("us-east-1", catalogProps.get("iceberg.rest.signing-region")); - Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE", catalogProps.get("iceberg.rest.access-key-id")); + Assertions.assertEquals("glue", catalogProps.get("rest.signing-name")); + Assertions.assertEquals("us-east-1", catalogProps.get("rest.signing-region")); + Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE", catalogProps.get("rest.access-key-id")); Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - catalogProps.get("iceberg.rest.secret-access-key")); - Assertions.assertEquals("true", catalogProps.get("iceberg.rest.sigv4-enabled")); + catalogProps.get("rest.secret-access-key")); + Assertions.assertEquals("true", catalogProps.get("rest.sigv4-enabled")); } @Test @@ -269,8 +269,8 @@ public void testGlueRestCatalogCaseInsensitive() { restProps.initNormalizeAndCheckProps(); Map catalogProps = restProps.getIcebergRestCatalogProperties(); - Assertions.assertEquals("glue", catalogProps.get("iceberg.rest.signing-name")); - Assertions.assertEquals("us-west-2", catalogProps.get("iceberg.rest.signing-region")); + Assertions.assertEquals("glue", catalogProps.get("rest.signing-name")); + Assertions.assertEquals("us-west-2", catalogProps.get("rest.signing-region")); } @Test @@ -341,11 +341,11 @@ public void testNonGlueSigningNameDoesNotRequireAdditionalProperties() { Map catalogProps = restProps.getIcebergRestCatalogProperties(); // Should not contain glue-specific properties - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-name")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-region")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.access-key-id")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.secret-access-key")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.sigv4-enabled")); + Assertions.assertFalse(catalogProps.containsKey("rest.signing-name")); + Assertions.assertFalse(catalogProps.containsKey("rest.signing-region")); + Assertions.assertFalse(catalogProps.containsKey("rest.access-key-id")); + Assertions.assertFalse(catalogProps.containsKey("rest.secret-access-key")); + Assertions.assertFalse(catalogProps.containsKey("rest.sigv4-enabled")); } @Test @@ -363,11 +363,11 @@ public void testEmptySigningNameDoesNotAddGlueProperties() { Map catalogProps = restProps.getIcebergRestCatalogProperties(); // Should not contain glue-specific properties since signing-name is not "glue" - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-name")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.signing-region")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.access-key-id")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.secret-access-key")); - Assertions.assertFalse(catalogProps.containsKey("iceberg.rest.sigv4-enabled")); + Assertions.assertFalse(catalogProps.containsKey("rest.signing-name")); + Assertions.assertFalse(catalogProps.containsKey("rest.signing-region")); + Assertions.assertFalse(catalogProps.containsKey("rest.access-key-id")); + Assertions.assertFalse(catalogProps.containsKey("rest.secret-access-key")); + Assertions.assertFalse(catalogProps.containsKey("rest.sigv4-enabled")); } @Test @@ -389,12 +389,12 @@ public void testGlueRestCatalogWithOAuth2() { Map catalogProps = restProps.getIcebergRestCatalogProperties(); // Should have both OAuth2 and Glue properties Assertions.assertEquals("my-access-token", catalogProps.get(OAuth2Properties.TOKEN)); - Assertions.assertEquals("glue", catalogProps.get("iceberg.rest.signing-name")); - Assertions.assertEquals("us-east-1", catalogProps.get("iceberg.rest.signing-region")); - Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE", catalogProps.get("iceberg.rest.access-key-id")); + Assertions.assertEquals("glue", catalogProps.get("rest.signing-name")); + Assertions.assertEquals("us-east-1", catalogProps.get("rest.signing-region")); + Assertions.assertEquals("AKIAIOSFODNN7EXAMPLE", catalogProps.get("rest.access-key-id")); Assertions.assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - catalogProps.get("iceberg.rest.secret-access-key")); - Assertions.assertEquals("true", catalogProps.get("iceberg.rest.sigv4-enabled")); + catalogProps.get("rest.secret-access-key")); + Assertions.assertEquals("true", catalogProps.get("rest.sigv4-enabled")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java index 8ccad6ae7f07c2..dca4f91c789223 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/RestCatalogTest.java @@ -31,10 +31,12 @@ import org.apache.iceberg.io.CloseableIterable; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Map; +@Disabled("Disabled until fill ak/sk") public class RestCatalogTest { // set your AWS access key and secret key From c6ee5b86a072dc2aac9ce70002110019da60c1ed Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Aug 2025 21:51:17 -0700 Subject: [PATCH 5/7] fix guess me --- .../datasource/property/storage/S3Properties.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 2934708eebcf22..f8bbc6bc3e5c91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -209,6 +209,15 @@ protected static boolean guessIsMe(Map origProps) { return endpoint.contains("amazonaws.com"); } + // guess from URI + Optional uriValue = origProps.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase("uri")) + .map(Map.Entry::getValue) + .findFirst(); + if (uriValue.isPresent()) { + return uriValue.get().contains("amazonaws.com"); + } + // guess from region String region = Stream.of(REGION_NAMES_FOR_GUESSING) .map(origProps::get) @@ -218,11 +227,7 @@ protected static boolean guessIsMe(Map origProps) { if (!Strings.isNullOrEmpty(region)) { return true; } - Optional uriValue = origProps.entrySet().stream() - .filter(e -> e.getKey().equalsIgnoreCase("uri")) - .map(Map.Entry::getValue) - .findFirst(); - return uriValue.isPresent() && uriValue.get().contains("amazonaws.com"); + return false; } @Override From 118651612ea8bbef9abf811fe26d22262b64443c Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 7 Aug 2025 14:39:35 +0800 Subject: [PATCH 6/7] add test --- .../backup_restore_object_storage.groovy | 8 ++------ .../refactor_storage_param/s3_load.groovy | 7 +------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy index 179a33c19d1beb..920060c104bda5 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy @@ -199,17 +199,13 @@ suite("refactor_storage_backup_restore_object_storage", "p0,external,external_do shouldFail { createRepository("${failedRepoName}", "s3.endpoint", s3_endpoint, "s3.region", region, "AWS_ACCESS_KEY", ak, "AWS_SECRET_KEY", sk, "true", "s3://ck/" + System.currentTimeMillis()) } - //endpoint is empty - shouldFail { - createRepository("${failedRepoName}", "s3.endpoint", "", "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "", "s3://${bucket}/test_" + System.currentTimeMillis()) - } //region is empty shouldFail { createRepository("${failedRepoName}", "s3.endpoint", "", "s3.region", "", "s3.access_key", ak, "s3.secret_key", sk, "", "s3://${bucket}/test_" + System.currentTimeMillis()) } } /*-------------AWS S3--------------------------------*/ - String ak = context.config.otherConfigs.get("AWSAK") + String ak = context.config.otherConfigs.get("AWSAK") String sk = context.config.otherConfigs.get("AWSSK") String s3_endpoint = "s3.ap-northeast-1.amazonaws.com" String region = "ap-northeast-1" @@ -297,4 +293,4 @@ suite("refactor_storage_backup_restore_object_storage", "p0,external,external_do backupAndRestore("${oss_repoName2}", ossDbName2, s3table, "backup_${oss_repoName1}_test") -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy index d9ed6a401896c8..1b628bde854d79 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy @@ -24,7 +24,7 @@ suite("refactor_storage_param_s3_load", "p0,external,external_docker") { if (enabled == null || enabled.equalsIgnoreCase("false")) { return } - String ak = context.config.otherConfigs.get("AWSAK") + String ak = context.config.otherConfigs.get("AWSAK") String sk = context.config.otherConfigs.get("AWSSK") String endpoint = "s3.ap-northeast-1.amazonaws.com" String region = "ap-northeast-1" @@ -124,16 +124,12 @@ suite("refactor_storage_param_s3_load", "p0,external,external_docker") { s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "") s3Load("https://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") - shouldFail { - s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") - } shouldFail { s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "s3.region", region, "s3.access_key", "", "s3.secret_key", sk, "false") } shouldFail { s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "") - } shouldFail { s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") @@ -168,7 +164,6 @@ suite("refactor_storage_param_s3_load", "p0,external,external_docker") { shouldFail { s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") } - shouldFail { s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "obs.region", region, "obs.access_key", "", "obs.secret_key", sk, "false") } From f35539e930bf5e0c66a5cd3cca1c3ad809aa19d4 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 8 Aug 2025 03:45:20 +0800 Subject: [PATCH 7/7] add glue p2 test --- regression-test/conf/regression-conf.groovy | 3 + .../iceberg/test_glue_rest_s3tables.out | 33 + .../iceberg/test_glue_rest_s3tables.groovy | 667 ++++++++++++++++++ 3 files changed, 703 insertions(+) create mode 100644 regression-test/data/external_table_p2/iceberg/test_glue_rest_s3tables.out create mode 100644 regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 84431bfdfa2221..8fd23dc9c78fc4 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -280,3 +280,6 @@ icebergS3TablesCatalog = "" enableExternalHudiTest = false // The properties string of hudi catalog hudiEmrCatalog = "" +icebergS3TablesCatalog="" +icebergS3TablesCatalogGlueRest="" + diff --git a/regression-test/data/external_table_p2/iceberg/test_glue_rest_s3tables.out b/regression-test/data/external_table_p2/iceberg/test_glue_rest_s3tables.out new file mode 100644 index 00000000000000..40a714ac82b992 --- /dev/null +++ b/regression-test/data/external_table_p2/iceberg/test_glue_rest_s3tables.out @@ -0,0 +1,33 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 + +-- !q02 -- +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 {"key1":"value1"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 {"key1":"value1"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 + +-- !q03 -- +\N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 {"key1":"value1"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 {"key1":"value1"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 + +-- !q01 -- +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 + +-- !q02 -- +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 {"key1":"value1"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 {"key1":"value1"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 + +-- !q03 -- +\N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123457 2024-03-21T12:00:00.123457 {"key1":"value1"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {2.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [3.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240321 +false -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value 2024-03-22 2024-03-22T12:00 2024-03-22T12:00:00.123457 2024-03-22T12:00:00.123457 {"key1":"value1"} {3:20} {3:200000000000} {3.2:20.2} {3.2:20.2} {0:1} {3.2:2.2} {3.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890:2.34567890} {3.3456789012345679:2.3456789012345679} ["string1", "string2"] [4, 5, 6] [300000000000, 400000000000] [3.3, 4.4] [3.123456789, 4.123456789] [0, 1] [3.3, 4.4] [3.45, 4.56] [8.4567, 4.5678] [3.45678901, 4.56789012] [3.45678901, 4.56789012] [3.4567890123456789, 4.5678901234567890] {"s_bigint":-1234567890} {"key":[{"s_int":-123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":-123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":-123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":-123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value11", "value2", null] [null, null, null] 20240322 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 +true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 + diff --git a/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy b/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy new file mode 100644 index 00000000000000..1954ce0a231708 --- /dev/null +++ b/regression-test/suites/external_table_p2/iceberg/test_glue_rest_s3tables.groovy @@ -0,0 +1,667 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_glue_rest_s3tables", "p2,external,iceberg,external_remote,external_remote_iceberg") { + def format_compressions = ["parquet_zstd"] + + def q01 = { String format_compression, String catalog_name -> + def parts = format_compression.split("_") + def format = parts[0] + def compression = parts[1] + def all_types_table = "iceberg_glue_rest_${format_compression}_master" + sql """ DROP TABLE IF EXISTS `${all_types_table}`; """ + sql """ + CREATE TABLE `${all_types_table}`( + `boolean_col` boolean, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `date_col` date, + `timestamp_col1` datetime, + `timestamp_col2` datetime, + `timestamp_col3` datetime, + `t_map_string` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array, + `dt` int) ENGINE=iceberg + properties ( + "compression-codec" = ${compression}, + "write-format"=${format} + ) + """ + + sql """ + INSERT INTO ${all_types_table} + VALUES ( + 1, -- boolean_col + 2147483647, -- int_col + 9223372036854775807, -- bigint_col + 123.45, -- float_col + 123456.789, -- double_col + CAST(123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-20', -- date_col + '2024-03-20 12:00:00', -- timestamp_col1 + '2024-03-20 12:00:00.123456789', -- timestamp_col2 + '2024-03-20 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(1, 10), -- t_map_int + MAP(1, 100000000000), -- t_map_bigint + MAP(CAST(1.1 AS FLOAT), CAST(10.1 AS FLOAT)), -- t_map_float + MAP(CAST(1.1 AS DOUBLE), CAST(10.1 AS DOUBLE)), -- t_map_double + MAP(TRUE, FALSE), -- t_map_boolean + MAP(CAST(1.1 AS DECIMAL(2,1)), CAST(1.1 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(1.23 AS DECIMAL(4,2)), CAST(1.23 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(1.2345 AS DECIMAL(8,4)), CAST(1.2345 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(1.23456789 AS DECIMAL(17,8)), CAST(1.23456789 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(1.23456789 AS DECIMAL(18,8)), CAST(1.23456789 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(1.234567890123456789 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(1, 2, 3), -- t_ARRAY_int + ARRAY(100000000000, 200000000000), -- t_ARRAY_bigint + ARRAY(CAST(1.1 AS FLOAT), CAST(2.2 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(1.123456789 AS DOUBLE), CAST(2.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(TRUE, FALSE), -- t_ARRAY_boolean + ARRAY(CAST(1.1 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(1.23 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(1.2345 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(1.23456789 AS DECIMAL(17,8)), CAST(2.34567891 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(1.23456789, 2.34567891), -- t_ARRAY_decimal_precision_18 + ARRAY(1.234567890123456789, 2.345678901234567890), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', 1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', 123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', 123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', 123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', 123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value1', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240320 -- dt + ); + """ + order_qt_q01 """ select * from ${all_types_table}; + """ + + sql """ + INSERT INTO ${all_types_table} + VALUES ( + 1, -- boolean_col + 2147483647, -- int_col + 9223372036854775807, -- bigint_col + CAST(123.45 AS FLOAT), -- float_col + CAST(123456.789 AS DOUBLE), -- double_col + CAST(123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-20', -- date_col + '2024-03-20 12:00:00', -- timestamp_col1 + '2024-03-20 12:00:00.123456789', -- timestamp_col2 + '2024-03-20 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(1, 10), -- t_map_int + MAP(1, 100000000000), -- t_map_bigint + MAP(CAST(1.1 AS FLOAT), CAST(10.1 AS FLOAT)), -- t_map_float + MAP(CAST(1.1 AS DOUBLE), CAST(10.1 AS DOUBLE)), -- t_map_double + MAP(true, false), -- t_map_boolean + MAP(CAST(1.1 AS DECIMAL(2,1)), CAST(1.1 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(1.23 AS DECIMAL(4,2)), CAST(1.23 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(1.2345 AS DECIMAL(8,4)), CAST(1.2345 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(1.23456789 AS DECIMAL(17,8)), CAST(1.23456789 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(1.23456789 AS DECIMAL(18,8)), CAST(1.23456789 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(1.234567890123456789 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(1, 2, 3), -- t_ARRAY_int + ARRAY(100000000000, 200000000000), -- t_ARRAY_bigint + ARRAY(CAST(1.1 AS FLOAT), CAST(2.2 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(1.123456789 AS DOUBLE), CAST(2.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(true, false), -- t_ARRAY_boolean + ARRAY(CAST(1.1 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(1.23 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(1.2345 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(1.23456789 AS DECIMAL(17,8)), CAST(2.34567891 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(1.23456789 AS DECIMAL(18,8)), CAST(2.34567891 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', 1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', 123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', 123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', 123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', 123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value1', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240320 -- dt + ), + ( + 0, -- boolean_col + -2147483648, -- int_col + -9223372036854775808, -- bigint_col + CAST(-123.45 AS FLOAT), -- float_col + CAST(-123456.789 AS DOUBLE), -- double_col + CAST(-123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(-1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(-123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(-123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-21', -- date_col + '2024-03-21 12:00:00', -- timestamp_col1 + '2024-03-21 12:00:00.123456789', -- timestamp_col2 + '2024-03-21 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(2, 20), -- t_map_int + MAP(2, 200000000000), -- t_map_bigint + MAP(CAST(2.2 AS FLOAT), CAST(20.2 AS FLOAT)), -- t_map_float + MAP(CAST(2.2 AS DOUBLE), CAST(20.2 AS DOUBLE)), -- t_map_double + MAP(false, true), -- t_map_boolean + MAP(CAST(2.2 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(2.34 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(2.3456 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(2.34567890 AS DECIMAL(17,8)), CAST(2.34567890 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(2.34567890 AS DECIMAL(18,8)), CAST(2.34567890 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(2.345678901234567890 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(4, 5, 6), -- t_ARRAY_int + ARRAY(300000000000, 400000000000), -- t_ARRAY_bigint + ARRAY(CAST(3.3 AS FLOAT), CAST(4.4 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(3.123456789 AS DOUBLE), CAST(4.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(false, true), -- t_ARRAY_boolean + ARRAY(CAST(3.3 AS DECIMAL(2,1)), CAST(4.4 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(3.45 AS DECIMAL(4,2)), CAST(4.56 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(3.4567 AS DECIMAL(8,4)), CAST(4.5678 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(3.45678901 AS DECIMAL(17,8)), CAST(4.56789012 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(3.45678901 AS DECIMAL(18,8)), CAST(4.56789012 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(3.456789012345678901 AS DECIMAL(38,16)), CAST(4.567890123456789012 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', -1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', -123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', -123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', -123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', -123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value1', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240321 -- dt + ), + ( + 0, -- boolean_col + -2147483648, -- int_col + -9223372036854775808, -- bigint_col + CAST(-123.45 AS FLOAT), -- float_col + CAST(-123456.789 AS DOUBLE), -- double_col + CAST(-123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(-1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(-123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(-123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-22', -- date_col + '2024-03-22 12:00:00', -- timestamp_col1 + '2024-03-22 12:00:00.123456789', -- timestamp_col2 + '2024-03-22 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(3, 20), -- t_map_int + MAP(3, 200000000000), -- t_map_bigint + MAP(CAST(3.2 AS FLOAT), CAST(20.2 AS FLOAT)), -- t_map_float + MAP(CAST(3.2 AS DOUBLE), CAST(20.2 AS DOUBLE)), -- t_map_double + MAP(false, true), -- t_map_boolean + MAP(CAST(3.2 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(3.34 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(2.3456 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(2.34567890 AS DECIMAL(17,8)), CAST(2.34567890 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(2.34567890 AS DECIMAL(18,8)), CAST(2.34567890 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(3.345678901234567890 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(4, 5, 6), -- t_ARRAY_int + ARRAY(300000000000, 400000000000), -- t_ARRAY_bigint + ARRAY(CAST(3.3 AS FLOAT), CAST(4.4 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(3.123456789 AS DOUBLE), CAST(4.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(false, true), -- t_ARRAY_boolean + ARRAY(CAST(3.3 AS DECIMAL(2,1)), CAST(4.4 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(3.45 AS DECIMAL(4,2)), CAST(4.56 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(8.4567 AS DECIMAL(8,4)), CAST(4.5678 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(3.45678901 AS DECIMAL(17,8)), CAST(4.56789012 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(3.45678901 AS DECIMAL(18,8)), CAST(4.56789012 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(3.456789012345678901 AS DECIMAL(38,16)), CAST(4.567890123456789012 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', -1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', -123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', -123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', -123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', -123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value11', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240322 -- dt + ); + """ + order_qt_q02 """ select * from ${all_types_table}; + """ + + sql """ + INSERT INTO ${all_types_table}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls) + VALUES ( + CAST(123.45 AS FLOAT), -- float_col + MAP(1, 10), -- t_map_int + ARRAY(CAST(1.2345 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(null, 'value1', 'value2') -- t_ARRAY_string_starting_with_nulls + ); + """ + order_qt_q03 """ select * from ${all_types_table}; + """ + + sql """ DROP TABLE ${all_types_table}; """ + } + + def q03 = { String format_compression, String catalog_name -> + def parts = format_compression.split("_") + def format = parts[0] + def compression = parts[1] + def all_types_partition_table = "iceberg_all_types_par_glue_rest_${format_compression}_master" + sql """ DROP TABLE IF EXISTS `${all_types_partition_table}`; """ + sql """ + CREATE TABLE `${all_types_partition_table}`( + `boolean_col` boolean, + `int_col` int, + `bigint_col` bigint, + `float_col` float, + `double_col` double, + `decimal_col1` decimal(9,0), + `decimal_col2` decimal(8,4), + `decimal_col3` decimal(18,6), + `decimal_col4` decimal(38,12), + `string_col` string, + `date_col` date, + `timestamp_col1` datetime, + `timestamp_col2` datetime, + `timestamp_col3` datetime, + `t_map_string` map, + `t_map_int` map, + `t_map_bigint` map, + `t_map_float` map, + `t_map_double` map, + `t_map_boolean` map, + `t_map_decimal_precision_2` map, + `t_map_decimal_precision_4` map, + `t_map_decimal_precision_8` map, + `t_map_decimal_precision_17` map, + `t_map_decimal_precision_18` map, + `t_map_decimal_precision_38` map, + `t_array_string` array, + `t_array_int` array, + `t_array_bigint` array, + `t_array_float` array, + `t_array_double` array, + `t_array_boolean` array, + `t_array_decimal_precision_2` array, + `t_array_decimal_precision_4` array, + `t_array_decimal_precision_8` array, + `t_array_decimal_precision_17` array, + `t_array_decimal_precision_18` array, + `t_array_decimal_precision_38` array, + `t_struct_bigint` struct, + `t_complex` map>>, + `t_struct_nested` struct>, + `t_struct_null` struct, + `t_struct_non_nulls_after_nulls` struct, + `t_nested_struct_non_nulls_after_nulls` struct>, + `t_map_null_value` map, + `t_array_string_starting_with_nulls` array, + `t_array_string_with_nulls_in_between` array, + `t_array_string_ending_with_nulls` array, + `t_array_string_all_nulls` array, + `dt` int) ENGINE=iceberg + PARTITION BY LIST (dt) () + properties ( + "compression-codec" = ${compression}, + "write-format"=${format} + ); + """ + + sql """ + INSERT INTO ${all_types_partition_table} + VALUES ( + 1, -- boolean_col + 2147483647, -- int_col + 9223372036854775807, -- bigint_col + 123.45, -- float_col + 123456.789, -- double_col + 123456789, -- decimal_col1 + 1234.5678, -- decimal_col2 + CAST(123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-20', -- date_col + '2024-03-20 12:00:00', -- timestamp_col1 + '2024-03-20 12:00:00.123456789', -- timestamp_col2 + '2024-03-20 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(1, 10), -- t_map_int + MAP(1, 100000000000), -- t_map_bigint + MAP(CAST(1.1 AS FLOAT), CAST(10.1 AS FLOAT)), -- t_map_float + MAP(CAST(1.1 AS DOUBLE), CAST(10.1 AS DOUBLE)), -- t_map_double + MAP(true, false), -- t_map_boolean + MAP(CAST(1.1 AS DECIMAL(2,1)), CAST(1.1 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(1.23 AS DECIMAL(4,2)), CAST(1.23 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(1.2345 AS DECIMAL(8,4)), CAST(1.2345 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(1.23456789 AS DECIMAL(17,8)), CAST(1.23456789 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(1.23456789 AS DECIMAL(18,8)), CAST(1.23456789 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(1.234567890123456789 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(1, 2, 3), -- t_ARRAY_int + ARRAY(100000000000, 200000000000), -- t_ARRAY_bigint + ARRAY(CAST(1.1 AS FLOAT), CAST(2.2 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(1.123456789 AS DOUBLE), CAST(2.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(true, false), -- t_ARRAY_boolean + ARRAY(CAST(1.1 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(1.23 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(1.2345 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(1.23456789 AS DECIMAL(17,8)), CAST(2.34567891 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(1.23456789 AS DECIMAL(18,8)), CAST(2.34567891 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', 1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', 123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', 123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', 123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', 123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value1', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240320 -- dt + ); + """ + order_qt_q01 """ select * from ${all_types_partition_table}; + """ + + sql """ + INSERT INTO ${all_types_partition_table} + VALUES ( + 1, -- boolean_col + 2147483647, -- int_col + 9223372036854775807, -- bigint_col + CAST(123.45 AS FLOAT), -- float_col + CAST(123456.789 AS DOUBLE), -- double_col + CAST(123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-20', -- date_col + '2024-03-20 12:00:00', -- timestamp_col1 + '2024-03-20 12:00:00.123456789', -- timestamp_col2 + '2024-03-20 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(1, 10), -- t_map_int + MAP(1, 100000000000), -- t_map_bigint + MAP(CAST(1.1 AS FLOAT), CAST(10.1 AS FLOAT)), -- t_map_float + MAP(CAST(1.1 AS DOUBLE), CAST(10.1 AS DOUBLE)), -- t_map_double + MAP(true, false), -- t_map_boolean + MAP(CAST(1.1 AS DECIMAL(2,1)), CAST(1.1 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(1.23 AS DECIMAL(4,2)), CAST(1.23 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(1.2345 AS DECIMAL(8,4)), CAST(1.2345 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(1.23456789 AS DECIMAL(17,8)), CAST(1.23456789 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(1.23456789 AS DECIMAL(18,8)), CAST(1.23456789 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(1.234567890123456789 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(1, 2, 3), -- t_ARRAY_int + ARRAY(100000000000, 200000000000), -- t_ARRAY_bigint + ARRAY(CAST(1.1 AS FLOAT), CAST(2.2 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(1.123456789 AS DOUBLE), CAST(2.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(true, false), -- t_ARRAY_boolean + ARRAY(CAST(1.1 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(1.23 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(1.2345 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(1.23456789 AS DECIMAL(17,8)), CAST(2.34567891 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(1.23456789 AS DECIMAL(18,8)), CAST(2.34567891 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(1.234567890123456789 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', 1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', 123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', 123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', 123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', 123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value1', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240320 -- dt + ), + ( + 0, -- boolean_col + -2147483648, -- int_col + -9223372036854775808, -- bigint_col + CAST(-123.45 AS FLOAT), -- float_col + CAST(-123456.789 AS DOUBLE), -- double_col + CAST(-123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(-1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(-123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(-123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-21', -- date_col + '2024-03-21 12:00:00', -- timestamp_col1 + '2024-03-21 12:00:00.123456789', -- timestamp_col2 + '2024-03-21 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(2, 20), -- t_map_int + MAP(2, 200000000000), -- t_map_bigint + MAP(CAST(2.2 AS FLOAT), CAST(20.2 AS FLOAT)), -- t_map_float + MAP(CAST(2.2 AS DOUBLE), CAST(20.2 AS DOUBLE)), -- t_map_double + MAP(false, true), -- t_map_boolean + MAP(CAST(2.2 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(2.34 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(2.3456 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(2.34567890 AS DECIMAL(17,8)), CAST(2.34567890 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(2.34567890 AS DECIMAL(18,8)), CAST(2.34567890 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(2.345678901234567890 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(4, 5, 6), -- t_ARRAY_int + ARRAY(300000000000, 400000000000), -- t_ARRAY_bigint + ARRAY(CAST(3.3 AS FLOAT), CAST(4.4 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(3.123456789 AS DOUBLE), CAST(4.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(false, true), -- t_ARRAY_boolean + ARRAY(CAST(3.3 AS DECIMAL(2,1)), CAST(4.4 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(3.45 AS DECIMAL(4,2)), CAST(4.56 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(3.4567 AS DECIMAL(8,4)), CAST(4.5678 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(3.45678901 AS DECIMAL(17,8)), CAST(4.56789012 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(3.45678901 AS DECIMAL(18,8)), CAST(4.56789012 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(3.456789012345678901 AS DECIMAL(38,16)), CAST(4.567890123456789012 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', -1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', -123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', -123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', -123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', -123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value1', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240321 -- dt + ), + ( + 0, -- boolean_col + -2147483648, -- int_col + -9223372036854775808, -- bigint_col + -123.45, -- float_col + -123456.789, -- double_col + CAST(-123456789 AS DECIMAL(9,0)), -- decimal_col1 + CAST(-1234.5678 AS DECIMAL(8,4)), -- decimal_col2 + CAST(-123456.789012 AS DECIMAL(18,6)), -- decimal_col3 + CAST(-123456789.012345678901 AS DECIMAL(38,12)), -- decimal_col4 + 'string_value', -- string_col + '2024-03-22', -- date_col + '2024-03-22 12:00:00', -- timestamp_col1 + '2024-03-22 12:00:00.123456789', -- timestamp_col2 + '2024-03-22 12:00:00.123456789', -- timestamp_col3 + MAP('key1', 'value1'), -- t_map_string + MAP(3, 20), -- t_map_int + MAP(3, 200000000000), -- t_map_bigint + MAP(CAST(3.2 AS FLOAT), CAST(20.2 AS FLOAT)), -- t_map_float + MAP(CAST(3.2 AS DOUBLE), CAST(20.2 AS DOUBLE)), -- t_map_double + MAP(false, true), -- t_map_boolean + MAP(CAST(3.2 AS DECIMAL(2,1)), CAST(2.2 AS DECIMAL(2,1))), -- t_map_decimal_precision_2 + MAP(CAST(3.34 AS DECIMAL(4,2)), CAST(2.34 AS DECIMAL(4,2))), -- t_map_decimal_precision_4 + MAP(CAST(2.3456 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_map_decimal_precision_8 + MAP(CAST(2.34567890 AS DECIMAL(17,8)), CAST(2.34567890 AS DECIMAL(17,8))), -- t_map_decimal_precision_17 + MAP(CAST(2.34567890 AS DECIMAL(18,8)), CAST(2.34567890 AS DECIMAL(18,8))), -- t_map_decimal_precision_18 + MAP(CAST(3.345678901234567890 AS DECIMAL(38,16)), CAST(2.345678901234567890 AS DECIMAL(38,16))), -- t_map_decimal_precision_38 + ARRAY('string1', 'string2'), -- t_ARRAY_string + ARRAY(4, 5, 6), -- t_ARRAY_int + ARRAY(300000000000, 400000000000), -- t_ARRAY_bigint + ARRAY(CAST(3.3 AS FLOAT), CAST(4.4 AS FLOAT)), -- t_ARRAY_float + ARRAY(CAST(3.123456789 AS DOUBLE), CAST(4.123456789 AS DOUBLE)), -- t_ARRAY_double + ARRAY(false, true), -- t_ARRAY_boolean + ARRAY(CAST(3.3 AS DECIMAL(2,1)), CAST(4.4 AS DECIMAL(2,1))), -- t_ARRAY_decimal_precision_2 + ARRAY(CAST(3.45 AS DECIMAL(4,2)), CAST(4.56 AS DECIMAL(4,2))), -- t_ARRAY_decimal_precision_4 + ARRAY(CAST(8.4567 AS DECIMAL(8,4)), CAST(4.5678 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(CAST(3.45678901 AS DECIMAL(17,8)), CAST(4.56789012 AS DECIMAL(17,8))), -- t_ARRAY_decimal_precision_17 + ARRAY(CAST(3.45678901 AS DECIMAL(18,8)), CAST(4.56789012 AS DECIMAL(18,8))), -- t_ARRAY_decimal_precision_18 + ARRAY(CAST(3.456789012345678901 AS DECIMAL(38,16)), CAST(4.567890123456789012 AS DECIMAL(38,16))), -- t_ARRAY_decimal_precision_38 + NAMED_STRUCT('s_bigint', -1234567890), -- t_struct_bigint + MAP('key', ARRAY(NAMED_STRUCT('s_int', -123))), -- t_complex + NAMED_STRUCT('struct_field', ARRAY('value1', 'value2')), -- t_struct_nested + NAMED_STRUCT('struct_field_null', null, 'struct_field_null2', null), -- t_struct_null + NAMED_STRUCT('struct_non_nulls_after_nulls1', -123, 'struct_non_nulls_after_nulls2', 'value'), -- t_struct_non_nulls_after_nulls + NAMED_STRUCT('struct_field1', -123, 'struct_field2', 'value', 'strict_field3', NAMED_STRUCT('nested_struct_field1', -123, 'nested_struct_field2', 'nested_value')), -- t_nested_struct_non_nulls_after_nulls + MAP('null_key', null), -- t_map_null_value + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + ARRAY('value1', null, 'value2'), -- t_ARRAY_string_with_nulls_in_between + ARRAY('value11', 'value2', null), -- t_ARRAY_string_ending_with_nulls + ARRAY(null, null, null), -- t_ARRAY_string_all_nulls + 20240322 -- dt + ); + """ + order_qt_q02 """ select * from ${all_types_partition_table}; + """ + + sql """ + INSERT INTO ${all_types_partition_table}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls, dt) + VALUES ( + 123.45, -- float_col + MAP(1, 10), -- t_map_int + ARRAY(CAST(1.2345 AS DECIMAL(8,4)), CAST(2.3456 AS DECIMAL(8,4))), -- t_ARRAY_decimal_precision_8 + ARRAY(null, 'value1', 'value2'), -- t_ARRAY_string_starting_with_nulls + 20240321 -- dt + ); + """ + order_qt_q03 """ select * from ${all_types_partition_table}; + """ + + // just test + sql """ + SELECT + CASE + WHEN file_size_in_bytes BETWEEN 0 AND 8 * 1024 * 1024 THEN '0-8M' + WHEN file_size_in_bytes BETWEEN 8 * 1024 * 1024 + 1 AND 32 * 1024 * 1024 THEN '8-32M' + WHEN file_size_in_bytes BETWEEN 2 * 1024 * 1024 + 1 AND 128 * 1024 * 1024 THEN '32-128M' + WHEN file_size_in_bytes BETWEEN 128 * 1024 * 1024 + 1 AND 512 * 1024 * 1024 THEN '128-512M' + WHEN file_size_in_bytes > 512 * 1024 * 1024 THEN '> 512M' + ELSE 'Unknown' + END AS SizeRange, + COUNT(*) AS FileNum + FROM ${all_types_partition_table}\$data_files + GROUP BY + SizeRange; + """ + + sql """ DROP TABLE ${all_types_partition_table}; """ + } + + String enabled = context.config.otherConfigs.get("enableExternalIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String catalog_name = "test_s3tables_glue_rest" + String props = context.config.otherConfigs.get("icebergS3TablesCatalogGlueRest") + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog ${catalog_name} properties ( + ${props} + ); + """ + + sql """ switch ${catalog_name};""" + sql """ drop database if exists iceberg_s3tables_glue_rest force""" + sql """ create database iceberg_s3tables_glue_rest""" + sql """ use iceberg_s3tables_glue_rest;""" + sql """ set enable_fallback_to_original_planner=false """ + + try { + for (String format_compression in format_compressions) { + logger.info("Process format_compression " + format_compression) + q01(format_compression, catalog_name) + q03(format_compression, catalog_name) + } + } finally { + } +}