From fa396320288ca824e563c103a5adf0f25e8a969c Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sat, 6 Sep 2025 08:30:13 +0800 Subject: [PATCH 1/5] [opt](catalog) remove old properties (#55625) (cherry picked from commit 71c5f35aee5760f3b2d0e0afff78a154b284d597) --- .idea/vcs.xml | 46 +-- .../analysis/CreateStorageVaultStmt.java | 12 +- .../org/apache/doris/catalog/HMSResource.java | 8 +- .../org/apache/doris/catalog/HiveTable.java | 17 +- .../apache/doris/catalog/S3StorageVault.java | 3 +- .../doris/common/util/PrintableMap.java | 27 +- .../datasource/hive/HiveMetadataOps.java | 6 +- .../hive/ThriftHMSCachedClient.java | 8 +- .../iceberg/IcebergDLFExternalCatalog.java | 4 +- .../datasource/iceberg/IcebergUtils.java | 4 +- .../lakesoul/source/LakeSoulScanNode.java | 10 +- .../paimon/source/PaimonSource.java | 3 +- .../property/PropertyConverter.java | 352 ------------------ .../property/S3ClientBEProperties.java | 109 ------ .../property/constants/BaseProperties.java | 6 - .../property/constants/BosProperties.java | 86 ----- .../property/constants/CosProperties.java | 41 -- .../property/constants/GCSProperties.java | 40 -- .../property/constants/GlueProperties.java | 57 --- .../property/constants/HMSProperties.java | 33 -- .../property/constants/MCProperties.java | 1 - .../property/constants/MinioProperties.java | 40 -- .../property/constants/ObsProperties.java | 46 --- .../property/constants/OssProperties.java | 43 --- .../property/constants/PaimonProperties.java | 56 --- .../property/constants/S3Properties.java | 86 +---- .../constants/TrinoConnectorProperties.java | 22 -- .../AWSGlueMetaStoreBaseProperties.java | 1 + .../property/metastore/HMSBaseProperties.java | 7 +- .../property/storage/AzureProperties.java | 1 + .../property/storage/BrokerProperties.java | 18 +- .../property/storage/COSProperties.java | 1 + .../property/storage/GCSProperties.java | 1 + .../property/storage/MinioProperties.java | 2 + .../property/storage/OBSProperties.java | 1 + .../property/storage/OSSHdfsProperties.java | 1 + .../property/storage/OSSProperties.java | 4 +- .../property/storage/S3Properties.java | 3 +- .../property/storage/S3PropertyUtils.java | 13 + .../TrinoConnectorExternalCatalog.java | 4 +- .../plans/commands/info/CopyIntoInfo.java | 0 .../hive/metastore/HiveMetaStoreClient.java | 6 +- .../apache/doris/catalog/HiveTableTest.java | 6 +- .../iceberg/IcebergHadoopCatalogTest.java | 4 +- .../apache/doris/fs/obj/S3FileSystemTest.java | 4 +- 45 files changed, 140 insertions(+), 1103 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BosProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/HMSProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/TrinoConnectorProperties.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 0a185d9bdc7c46..11a42457d994f7 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,34 +1,16 @@ - - - - - - - - - - + + + + + + + \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java index 3a4e8ff9cb59b3..c001fba80ce287 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.analysis; +package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.StorageVault; @@ -28,7 +28,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -148,11 +148,11 @@ public void analyze(Analyzer analyzer) throws UserException { setStorageVaultType(StorageVault.StorageVaultType.fromString(type)); if (vaultType == StorageVault.StorageVaultType.S3 - && !properties.containsKey(PropertyConverter.USE_PATH_STYLE)) { + && !properties.containsKey(S3Properties.USE_PATH_STYLE)) { properties = ImmutableMap.builder() - .putAll(properties) - .put(PropertyConverter.USE_PATH_STYLE, "true") - .build(); + .putAll(properties) + .put(S3Properties.USE_PATH_STYLE, "true") + .build(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java index fccf2a6010b274..c3167921f12a99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java @@ -19,7 +19,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.proc.BaseProcResult; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -63,10 +63,8 @@ public void modifyProperties(Map properties) throws DdlException @Override protected void setProperties(ImmutableMap properties) throws DdlException { - for (String field : HMSProperties.REQUIRED_FIELDS) { - if (!properties.containsKey(field)) { - throw new DdlException("Missing [" + field + "] in properties."); - } + if (!properties.containsKey(HMSBaseProperties.HIVE_METASTORE_URIS)) { + throw new DdlException("Missing [" + HMSBaseProperties.HIVE_METASTORE_URIS + "] in properties."); } this.properties.putAll(properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java index 717c69f667c803..9600f2f81f8e67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java @@ -21,8 +21,8 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.security.authentication.AuthType; import org.apache.doris.common.security.authentication.AuthenticationConfig; -import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import org.apache.doris.thrift.THiveTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -105,18 +105,19 @@ private void validate(Map properties) throws DdlException { // check hive properties // hive.metastore.uris - String hiveMetaStoreUris = copiedProps.get(HMSProperties.HIVE_METASTORE_URIS); + String hiveMetaStoreUris = copiedProps.get(HMSBaseProperties.HIVE_METASTORE_URIS); if (Strings.isNullOrEmpty(hiveMetaStoreUris)) { throw new DdlException(String.format( - PROPERTY_MISSING_MSG, HMSProperties.HIVE_METASTORE_URIS, HMSProperties.HIVE_METASTORE_URIS)); + PROPERTY_MISSING_MSG, HMSBaseProperties.HIVE_METASTORE_URIS, + HMSBaseProperties.HIVE_METASTORE_URIS)); } - copiedProps.remove(HMSProperties.HIVE_METASTORE_URIS); - hiveProperties.put(HMSProperties.HIVE_METASTORE_URIS, hiveMetaStoreUris); + copiedProps.remove(HMSBaseProperties.HIVE_METASTORE_URIS); + hiveProperties.put(HMSBaseProperties.HIVE_METASTORE_URIS, hiveMetaStoreUris); // support multi hive version - String hiveVersion = copiedProps.get(HMSProperties.HIVE_VERSION); + String hiveVersion = copiedProps.get(HMSBaseProperties.HIVE_VERSION); if (!Strings.isNullOrEmpty(hiveVersion)) { - copiedProps.remove(HMSProperties.HIVE_VERSION); - hiveProperties.put(HMSProperties.HIVE_VERSION, hiveVersion); + copiedProps.remove(HMSBaseProperties.HIVE_VERSION); + hiveProperties.put(HMSBaseProperties.HIVE_VERSION, hiveVersion); } // check auth type diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java index 58097d3c1bc8fe..f04b45010b1b83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.S3Properties; import com.google.common.base.Preconditions; @@ -64,7 +63,7 @@ public class S3StorageVault extends StorageVault { public static class PropertyKey { public static final String ACCESS_KEY = S3Properties.ACCESS_KEY; public static final String SECRET_KEY = S3Properties.SECRET_KEY; - public static final String USE_PATH_STYLE = PropertyConverter.USE_PATH_STYLE; + public static final String USE_PATH_STYLE = S3Properties.USE_PATH_STYLE; public static final String ROOT_PATH = S3Properties.ROOT_PATH; public static final String PROVIDER = S3Properties.PROVIDER; public static final String REGION = S3Properties.REGION; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java index e035814f1399a5..bbfd82f827d4c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java @@ -18,14 +18,17 @@ package org.apache.doris.common.util; import org.apache.doris.datasource.property.ConnectorPropertiesUtils; -import org.apache.doris.datasource.property.constants.CosProperties; -import org.apache.doris.datasource.property.constants.GCSProperties; -import org.apache.doris.datasource.property.constants.GlueProperties; import org.apache.doris.datasource.property.constants.MCProperties; -import org.apache.doris.datasource.property.constants.ObsProperties; -import org.apache.doris.datasource.property.constants.OssProperties; import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.metastore.AWSGlueMetaStoreBaseProperties; import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties; +import org.apache.doris.datasource.property.storage.AzureProperties; +import org.apache.doris.datasource.property.storage.COSProperties; +import org.apache.doris.datasource.property.storage.GCSProperties; +import org.apache.doris.datasource.property.storage.MinioProperties; +import org.apache.doris.datasource.property.storage.OBSProperties; +import org.apache.doris.datasource.property.storage.OSSHdfsProperties; +import org.apache.doris.datasource.property.storage.OSSProperties; import com.google.common.collect.Sets; @@ -60,18 +63,20 @@ public class PrintableMap { SENSITIVE_KEY.addAll(Arrays.asList( S3Properties.SECRET_KEY, S3Properties.Env.SECRET_KEY, - ObsProperties.SECRET_KEY, - OssProperties.SECRET_KEY, - GCSProperties.SECRET_KEY, - CosProperties.SECRET_KEY, - GlueProperties.SECRET_KEY, MCProperties.SECRET_KEY)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AliyunDLFBaseProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AWSGlueMetaStoreBaseProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(GCSProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AzureProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OSSProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OSSHdfsProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(COSProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OBSProperties.class)); + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(MinioProperties.class)); HIDDEN_KEY = Sets.newHashSet(); HIDDEN_KEY.addAll(S3Properties.Env.FS_KEYS); - HIDDEN_KEY.addAll(GlueProperties.META_KEYS); } public PrintableMap(Map map, String keyValueSeparator, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index 15a5a6f9668cc6..beb818da32011a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -34,7 +34,7 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.operations.ExternalMetadataOps; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo; @@ -236,8 +236,8 @@ In hive, this table only can insert not update(not report error,but not actually } Map properties = catalog.getProperties(); - if (properties.containsKey(HMSProperties.HIVE_METASTORE_TYPE) - && properties.get(HMSProperties.HIVE_METASTORE_TYPE).equals(HMSProperties.DLF_TYPE)) { + if (properties.containsKey(HMSBaseProperties.HIVE_METASTORE_TYPE) + && properties.get(HMSBaseProperties.HIVE_METASTORE_TYPE).equals(HMSBaseProperties.DLF_TYPE)) { for (Column column : stmt.getColumns()) { if (column.hasDefaultValue()) { throw new UserException("Default values are not supported with `DLF` catalog."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java index 3341ac3423b025..6256b4f917acfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java @@ -24,7 +24,7 @@ import org.apache.doris.datasource.DatabaseMetadata; import org.apache.doris.datasource.TableMetadata; import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import com.aliyun.datalake.metastore.hive2.ProxyMetaStoreClient; import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient; @@ -634,11 +634,11 @@ private class ThriftHMSClient implements AutoCloseable { private volatile Throwable throwable; private ThriftHMSClient(HiveConf hiveConf) throws MetaException { - String type = hiveConf.get(HMSProperties.HIVE_METASTORE_TYPE); - if (HMSProperties.DLF_TYPE.equalsIgnoreCase(type)) { + String type = hiveConf.get(HMSBaseProperties.HIVE_METASTORE_TYPE); + if (HMSBaseProperties.DLF_TYPE.equalsIgnoreCase(type)) { client = RetryingMetaStoreClient.getProxy(hiveConf, DUMMY_HOOK_LOADER, ProxyMetaStoreClient.class.getName()); - } else if (HMSProperties.GLUE_TYPE.equalsIgnoreCase(type)) { + } else if (HMSBaseProperties.GLUE_TYPE.equalsIgnoreCase(type)) { client = RetryingMetaStoreClient.getProxy(hiveConf, DUMMY_HOOK_LOADER, AWSCatalogMetastoreClient.class.getName()); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java index cd9e9e62dcb784..83df9a7e34b897 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java @@ -22,7 +22,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogProperty; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; import java.util.Map; @@ -32,7 +32,7 @@ public class IcebergDLFExternalCatalog extends IcebergExternalCatalog { public IcebergDLFExternalCatalog(long catalogId, String name, String resource, Map props, String comment) { super(catalogId, name, comment); - props.put(HMSProperties.HIVE_METASTORE_TYPE, "dlf"); + props.put(HMSBaseProperties.HIVE_METASTORE_TYPE, "dlf"); catalogProperty = new CatalogProperty(resource, props); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 7661fdf5a7f3fb..736170f8b3a18b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -60,7 +60,7 @@ import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.thrift.TExprOpcode; @@ -940,7 +940,7 @@ public static HiveCatalog createIcebergHiveCatalog(ExternalCatalog externalCatal // Later, type checks will be performed when loading the table. catalogProperties.put(HiveCatalog.LIST_ALL_TABLES, "true"); } - String metastoreUris = catalogProperties.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); + String metastoreUris = catalogProperties.getOrDefault(HMSBaseProperties.HIVE_METASTORE_URIS, ""); catalogProperties.put(CatalogProperties.URI, metastoreUris); hiveCatalog.initialize(name, catalogProperties); return hiveCatalog; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java index 80f037239ce3f2..ddcb5ebb6b31d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java @@ -183,12 +183,12 @@ private void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulS if (catalogProps.get(S3Properties.Env.ENDPOINT) != null) { options.put(LakeSoulUtils.FS_S3A_ENDPOINT, catalogProps.get(S3Properties.Env.ENDPOINT)); - if (options.containsKey(MinioProperties.ENDPOINT)) { - // Use path style access for minio - options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "true"); - } else { - // use virtual hosted style access for all other s3 compatible storage services + if (!options.containsKey("oss.endpoint")) { + // Aliyun OSS requires virtual host style access options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "false"); + } else { + // use path style access for all other s3 compatible storage services + options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "true"); } if (catalogProps.get(S3Properties.Env.ACCESS_KEY) != null) { options.put(LakeSoulUtils.FS_S3A_ACCESS_KEY, catalogProps.get(S3Properties.Env.ACCESS_KEY)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java index 1c6b88b16ec57b..69ab8a7fbc2677 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java @@ -23,7 +23,6 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.paimon.PaimonExternalTable; -import org.apache.doris.datasource.property.constants.PaimonProperties; import org.apache.doris.thrift.TFileAttributes; import com.google.common.annotations.VisibleForTesting; @@ -69,6 +68,6 @@ public ExternalCatalog getCatalog() { } public String getFileFormatFromTableProperties() { - return originTable.options().getOrDefault(PaimonProperties.FILE_FORMAT, "parquet"); + return originTable.options().getOrDefault("file.format", "parquet"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index 8a48f56a5452c6..e69de29bb2d1d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -1,352 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property; - -import org.apache.doris.common.credentials.CloudCredential; -import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.common.util.LocationPath; -import org.apache.doris.datasource.CatalogMgr; -import org.apache.doris.datasource.InitCatalogLog.Type; -import org.apache.doris.datasource.property.constants.AzureProperties; -import org.apache.doris.datasource.property.constants.CosProperties; -import org.apache.doris.datasource.property.constants.GCSProperties; -import org.apache.doris.datasource.property.constants.MinioProperties; -import org.apache.doris.datasource.property.constants.ObsProperties; -import org.apache.doris.datasource.property.constants.OssProperties; -import org.apache.doris.datasource.property.constants.PaimonProperties; -import org.apache.doris.datasource.property.constants.S3Properties; - -import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import org.apache.hadoop.fs.CosFileSystem; -import org.apache.hadoop.fs.CosNConfigKeys; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem; -import org.apache.hadoop.fs.obs.OBSConstants; -import org.apache.hadoop.fs.obs.OBSFileSystem; -import org.apache.hadoop.fs.s3a.Constants; -import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.HashMap; -import java.util.Map; - -public class PropertyConverter { - - private static final Logger LOG = LogManager.getLogger(PropertyConverter.class); - public static final String USE_PATH_STYLE = "use_path_style"; - - /** - * Convert properties defined at doris to FE S3 client properties - * Support other cloud client here. - */ - public static Map convertToHadoopFSProperties(Map props) { - if (props.containsKey(ObsProperties.ENDPOINT)) { - return convertToOBSProperties(props, ObsProperties.getCredential(props)); - } else if (props.containsKey(GCSProperties.ENDPOINT)) { - return convertToGCSProperties(props, GCSProperties.getCredential(props)); - } else if (props.containsKey(OssProperties.ENDPOINT)) { - return convertToOSSProperties(props, OssProperties.getCredential(props)); - } else if (props.containsKey(CosProperties.ENDPOINT)) { - return convertToCOSProperties(props, CosProperties.getCredential(props)); - } else if (props.containsKey(MinioProperties.ENDPOINT)) { - return convertToMinioProperties(props, MinioProperties.getCredential(props)); - } else if (props.containsKey(AzureProperties.ENDPOINT)) { - return convertToAzureProperties(props, AzureProperties.getCredential(props)); - } else if (props.containsKey(S3Properties.ENDPOINT)) { - CloudCredential s3Credential = S3Properties.getCredential(props); - Map s3Properties = convertToS3Properties(props, s3Credential); - String s3CliEndpoint = props.get(S3Properties.ENDPOINT); - return convertToCompatibleS3Properties(props, s3CliEndpoint, s3Credential, s3Properties); - } else if (props.containsKey(S3Properties.Env.ENDPOINT)) { - // checkout env in the end - // compatible with the s3,obs,oss,cos when they use aws client. - CloudCredentialWithEndpoint envCredentials = S3Properties.getEnvironmentCredentialWithEndpoint(props); - Map s3Properties = convertToS3EnvProperties(props, envCredentials, false); - String s3CliEndpoint = envCredentials.getEndpoint(); - return convertToCompatibleS3Properties(props, s3CliEndpoint, envCredentials, s3Properties); - } - return props; - } - - private static Map convertToAzureProperties(Map props, CloudCredential credential) { - return null; - } - - private static Map convertToCompatibleS3Properties(Map props, - String s3CliEndpoint, - CloudCredential credential, - Map s3Properties) { - Map heteroProps = new HashMap<>(s3Properties); - Map copiedProps = new HashMap<>(props); - if (s3CliEndpoint.contains(CosProperties.COS_PREFIX)) { - copiedProps.putIfAbsent(CosProperties.ENDPOINT, s3CliEndpoint); - // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties. - heteroProps.putAll(convertToCOSProperties(copiedProps, credential)); - } else if (s3CliEndpoint.contains(ObsProperties.OBS_PREFIX)) { - copiedProps.putIfAbsent(ObsProperties.ENDPOINT, s3CliEndpoint); - heteroProps.putAll(convertToOBSProperties(copiedProps, credential)); - } else if (s3CliEndpoint.contains(OssProperties.OSS_REGION_PREFIX)) { - copiedProps.putIfAbsent(OssProperties.ENDPOINT, s3CliEndpoint); - heteroProps.putAll(convertToOSSProperties(copiedProps, credential)); - } - return heteroProps; - } - - - private static Map convertToOBSProperties(Map props, - CloudCredential credential) { - Map obsProperties = Maps.newHashMap(); - obsProperties.put(OBSConstants.ENDPOINT, props.get(ObsProperties.ENDPOINT)); - obsProperties.put("fs.obs.impl", getHadoopFSImplByScheme("obs")); - if (credential.isWhole()) { - obsProperties.put(OBSConstants.ACCESS_KEY, credential.getAccessKey()); - obsProperties.put(OBSConstants.SECRET_KEY, credential.getSecretKey()); - } - if (credential.isTemporary()) { - obsProperties.put(ObsProperties.FS.SESSION_TOKEN, credential.getSessionToken()); - } - for (Map.Entry entry : props.entrySet()) { - if (entry.getKey().startsWith(ObsProperties.OBS_FS_PREFIX)) { - obsProperties.put(entry.getKey(), entry.getValue()); - } - } - return obsProperties; - } - - public static String getHadoopFSImplByScheme(String fsScheme) { - if (fsScheme.equalsIgnoreCase("obs")) { - return OBSFileSystem.class.getName(); - } else if (fsScheme.equalsIgnoreCase("file")) { - return LocalFileSystem.class.getName(); - } else if (fsScheme.equalsIgnoreCase("oss")) { - return AliyunOSSFileSystem.class.getName(); - } else if (fsScheme.equalsIgnoreCase("cosn") || fsScheme.equalsIgnoreCase("lakefs")) { - return CosFileSystem.class.getName(); - } else { - return S3AFileSystem.class.getName(); - } - } - - private static Map convertToS3EnvProperties(Map properties, - CloudCredentialWithEndpoint credential, - boolean isMeta) { - // Old properties to new properties - properties.put(S3Properties.ENDPOINT, credential.getEndpoint()); - properties.put(S3Properties.REGION, - checkRegion(credential.getEndpoint(), credential.getRegion(), S3Properties.Env.REGION)); - properties.put(S3Properties.ACCESS_KEY, credential.getAccessKey()); - properties.put(S3Properties.SECRET_KEY, credential.getSecretKey()); - if (properties.containsKey(S3Properties.Env.TOKEN)) { - properties.put(S3Properties.SESSION_TOKEN, credential.getSessionToken()); - } - if (properties.containsKey(S3Properties.Env.MAX_CONNECTIONS)) { - properties.put(S3Properties.MAX_CONNECTIONS, properties.get(S3Properties.Env.MAX_CONNECTIONS)); - } - if (properties.containsKey(S3Properties.Env.REQUEST_TIMEOUT_MS)) { - properties.put(S3Properties.REQUEST_TIMEOUT_MS, properties.get(S3Properties.Env.REQUEST_TIMEOUT_MS)); - } - if (properties.containsKey(S3Properties.Env.CONNECTION_TIMEOUT_MS)) { - properties.put(S3Properties.REQUEST_TIMEOUT_MS, properties.get(S3Properties.Env.CONNECTION_TIMEOUT_MS)); - } - - if (properties.containsKey(S3Properties.Env.ROLE_ARN)) { - properties.put(S3Properties.ROLE_ARN, properties.get(S3Properties.Env.ROLE_ARN)); - } - - if (properties.containsKey(S3Properties.Env.EXTERNAL_ID)) { - properties.put(S3Properties.EXTERNAL_ID, properties.get(S3Properties.Env.EXTERNAL_ID)); - } - - if (isMeta) { - return properties; - } - return convertToS3Properties(properties, credential); - } - - private static Map convertToS3Properties(Map properties, - CloudCredential credential) { - // s3 property in paimon is personalized - String type = properties.get(CatalogMgr.CATALOG_TYPE_PROP); - if (type != null && type.equalsIgnoreCase(Type.PAIMON.toString())) { - return PaimonProperties.convertToS3Properties(properties, credential); - } - Map s3Properties = Maps.newHashMap(); - String endpoint = properties.get(S3Properties.ENDPOINT); - s3Properties.put(Constants.ENDPOINT, endpoint); - s3Properties.put(Constants.AWS_REGION, - checkRegion(endpoint, properties.get(S3Properties.REGION), S3Properties.REGION)); - if (properties.containsKey(S3Properties.MAX_CONNECTIONS)) { - s3Properties.put(Constants.MAXIMUM_CONNECTIONS, properties.get(S3Properties.MAX_CONNECTIONS)); - } - if (properties.containsKey(S3Properties.REQUEST_TIMEOUT_MS)) { - s3Properties.put(Constants.REQUEST_TIMEOUT, properties.get(S3Properties.REQUEST_TIMEOUT_MS)); - } - if (properties.containsKey(S3Properties.CONNECTION_TIMEOUT_MS)) { - s3Properties.put(Constants.SOCKET_TIMEOUT, properties.get(S3Properties.CONNECTION_TIMEOUT_MS)); - } - - setS3FsAccess(s3Properties, properties, credential); - s3Properties.putAll(properties); - // remove extra meta properties - S3Properties.FS_KEYS.forEach(s3Properties::remove); - - return s3Properties; - } - - public static String checkRegion(String endpoint, String region, String regionKey) { - if (Strings.isNullOrEmpty(region)) { - region = S3Properties.getRegionOfEndpoint(endpoint); - } - if (Strings.isNullOrEmpty(region)) { - String errorMsg = String.format("No '%s' info found, using SDK default region: us-east-1", regionKey); - LOG.warn(errorMsg); - return "us-east-1"; - } - return region; - } - - private static void setS3FsAccess(Map s3Properties, Map properties, - CloudCredential credential) { - s3Properties.put(Constants.MAX_ERROR_RETRIES, "2"); - s3Properties.putIfAbsent("fs.s3.impl", S3AFileSystem.class.getName()); - String credentialsProviders = getAWSCredentialsProviders(properties); - s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER, credentialsProviders); - if (credential.isWhole()) { - s3Properties.put(Constants.ACCESS_KEY, credential.getAccessKey()); - s3Properties.put(Constants.SECRET_KEY, credential.getSecretKey()); - } - if (credential.isTemporary()) { - s3Properties.put(Constants.SESSION_TOKEN, credential.getSessionToken()); - s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER, TemporaryAWSCredentialsProvider.class.getName()); - } - s3Properties.put(Constants.PATH_STYLE_ACCESS, properties.getOrDefault(USE_PATH_STYLE, "false")); - for (Map.Entry entry : properties.entrySet()) { - if (entry.getKey().startsWith(S3Properties.S3_FS_PREFIX)) { - s3Properties.put(entry.getKey(), entry.getValue()); - } - } - - if (properties.containsKey(S3Properties.ROLE_ARN) - && !Strings.isNullOrEmpty(properties.get(S3Properties.ROLE_ARN))) { - // refer to https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/assumed_roles.html - // https://issues.apache.org/jira/browse/HADOOP-19201 - s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.class.getName()); - s3Properties.put(Constants.ASSUMED_ROLE_ARN, properties.get(S3Properties.ROLE_ARN)); - s3Properties.put(Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER, - InstanceProfileCredentialsProvider.class.getName()); - - if (properties.containsKey(S3Properties.EXTERNAL_ID) - && !Strings.isNullOrEmpty(properties.get(S3Properties.EXTERNAL_ID))) { - LOG.warn("External ID is not supported for assumed role credential provider"); - } - } - } - - public static String getAWSCredentialsProviders(Map properties) { - String credentialsProviders; - String hadoopCredProviders = properties.get(Constants.AWS_CREDENTIALS_PROVIDER); - if (hadoopCredProviders != null) { - credentialsProviders = hadoopCredProviders; - } else { - String defaultProviderList = String.join(",", S3Properties.AWS_CREDENTIALS_PROVIDERS); - credentialsProviders = properties.getOrDefault(S3Properties.CREDENTIALS_PROVIDER, defaultProviderList); - } - return credentialsProviders; - } - - private static Map convertToGCSProperties(Map props, CloudCredential credential) { - // Now we use s3 client to access - return convertToS3Properties(S3Properties.prefixToS3(props), credential); - } - - private static Map convertToOSSProperties(Map props, CloudCredential credential) { - Map ossProperties = Maps.newHashMap(); - String endpoint = props.get(OssProperties.ENDPOINT); - if (endpoint.startsWith(OssProperties.OSS_PREFIX)) { - // may use oss.oss-cn-beijing.aliyuncs.com - endpoint = endpoint.replace(OssProperties.OSS_PREFIX, ""); - } - ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ENDPOINT_KEY, endpoint); - boolean hdfsEnabled = Boolean.parseBoolean(props.getOrDefault(OssProperties.OSS_HDFS_ENABLED, "false")); - if (LocationPath.isHdfsOnOssEndpoint(endpoint) || hdfsEnabled) { - // use endpoint or enable hdfs - rewriteHdfsOnOssProperties(ossProperties, endpoint); - } else { - ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss")); - } - if (credential.isWhole()) { - ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID, credential.getAccessKey()); - ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET, credential.getSecretKey()); - } - if (credential.isTemporary()) { - ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN, credential.getSessionToken()); - } - for (Map.Entry entry : props.entrySet()) { - if (entry.getKey().startsWith(OssProperties.OSS_FS_PREFIX)) { - ossProperties.put(entry.getKey(), entry.getValue()); - } - } - return ossProperties; - } - - private static void rewriteHdfsOnOssProperties(Map ossProperties, String endpoint) { - if (!LocationPath.isHdfsOnOssEndpoint(endpoint)) { - // just for robustness here, avoid wrong endpoint when oss-hdfs is enabled. - // convert "oss-cn-beijing.aliyuncs.com" to "cn-beijing.oss-dls.aliyuncs.com" - // reference link: https://www.alibabacloud.com/help/en/e-mapreduce/latest/oss-kusisurumen - String[] endpointSplit = endpoint.split("\\."); - if (endpointSplit.length > 0) { - String region = endpointSplit[0].replace("oss-", "").replace("-internal", ""); - ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ENDPOINT_KEY, - region + ".oss-dls.aliyuncs.com"); - } - } - ossProperties.put("fs.oss.impl", "com.aliyun.jindodata.oss.JindoOssFileSystem"); - ossProperties.put("fs.AbstractFileSystem.oss.impl", "com.aliyun.jindodata.oss.OSS"); - } - - private static Map convertToCOSProperties(Map props, CloudCredential credential) { - Map cosProperties = Maps.newHashMap(); - cosProperties.put(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY, props.get(CosProperties.ENDPOINT)); - cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn")); - cosProperties.put("fs.lakefs.impl", getHadoopFSImplByScheme("lakefs")); - if (credential.isWhole()) { - cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_ID_KEY, credential.getAccessKey()); - cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_KEY_KEY, credential.getSecretKey()); - } - // session token is unsupported - for (Map.Entry entry : props.entrySet()) { - if (entry.getKey().startsWith(CosProperties.COS_FS_PREFIX)) { - cosProperties.put(entry.getKey(), entry.getValue()); - } - } - return cosProperties; - } - - private static Map convertToMinioProperties(Map props, CloudCredential credential) { - if (!props.containsKey(MinioProperties.REGION)) { - props.put(MinioProperties.REGION, MinioProperties.DEFAULT_REGION); - } - return convertToS3Properties(S3Properties.prefixToS3(props), credential); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java deleted file mode 100644 index c4f4c01b0b0f3a..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property; - -import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; -import org.apache.doris.datasource.property.constants.CosProperties; -import org.apache.doris.datasource.property.constants.GCSProperties; -import org.apache.doris.datasource.property.constants.MinioProperties; -import org.apache.doris.datasource.property.constants.ObsProperties; -import org.apache.doris.datasource.property.constants.OssProperties; -import org.apache.doris.datasource.property.constants.S3Properties; -import org.apache.doris.datasource.property.constants.S3Properties.Env; - -import java.util.HashMap; -import java.util.Map; - -public class S3ClientBEProperties { - /** - * convert FE properties to BE S3 client properties - * On BE, should use properties like AWS_XXX. - */ - public static Map getBeFSProperties(Map properties) { - if (properties.containsKey(MinioProperties.ENDPOINT)) { - if (!properties.containsKey(MinioProperties.REGION)) { - properties.put(MinioProperties.REGION, MinioProperties.DEFAULT_REGION); - } - return getBeAWSPropertiesFromS3(S3Properties.prefixToS3(properties)); - } else if (properties.containsKey(S3Properties.ENDPOINT)) { - // s3,oss,cos,obs use this. - return getBeAWSPropertiesFromS3(properties); - } else if (properties.containsKey(ObsProperties.ENDPOINT) - || properties.containsKey(OssProperties.ENDPOINT) - || properties.containsKey(GCSProperties.ENDPOINT) - || properties.containsKey(CosProperties.ENDPOINT)) { - return getBeAWSPropertiesFromS3(S3Properties.prefixToS3(properties)); - } - return properties; - } - - private static Map getBeAWSPropertiesFromS3(Map properties) { - Map beProperties = new HashMap<>(); - if (properties.containsKey(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE) - && properties.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE).equals( - IcebergExternalCatalog.ICEBERG_S3_TABLES)) { - beProperties.put(Env.NEED_OVERRIDE_ENDPOINT, "false"); - } else { - beProperties.put(Env.NEED_OVERRIDE_ENDPOINT, "true"); - } - String endpoint = properties.get(S3Properties.ENDPOINT); - beProperties.put(S3Properties.Env.ENDPOINT, endpoint); - String region = PropertyConverter.checkRegion(endpoint, properties.get(S3Properties.Env.REGION), - S3Properties.Env.REGION); - beProperties.put(S3Properties.Env.REGION, properties.getOrDefault(S3Properties.REGION, region)); - if (properties.containsKey(S3Properties.ACCESS_KEY)) { - beProperties.put(S3Properties.Env.ACCESS_KEY, properties.get(S3Properties.ACCESS_KEY)); - } - if (properties.containsKey(S3Properties.SECRET_KEY)) { - beProperties.put(S3Properties.Env.SECRET_KEY, properties.get(S3Properties.SECRET_KEY)); - } - if (properties.containsKey(S3Properties.SESSION_TOKEN)) { - beProperties.put(S3Properties.Env.TOKEN, properties.get(S3Properties.SESSION_TOKEN)); - } - if (properties.containsKey(S3Properties.ROOT_PATH)) { - beProperties.put(S3Properties.Env.ROOT_PATH, properties.get(S3Properties.ROOT_PATH)); - } - if (properties.containsKey(S3Properties.BUCKET)) { - beProperties.put(S3Properties.Env.BUCKET, properties.get(S3Properties.BUCKET)); - } - if (properties.containsKey(S3Properties.MAX_CONNECTIONS)) { - beProperties.put(Env.MAX_CONNECTIONS, properties.get(S3Properties.MAX_CONNECTIONS)); - } - if (properties.containsKey(S3Properties.REQUEST_TIMEOUT_MS)) { - beProperties.put(Env.REQUEST_TIMEOUT_MS, properties.get(S3Properties.REQUEST_TIMEOUT_MS)); - } - if (properties.containsKey(S3Properties.CONNECTION_TIMEOUT_MS)) { - beProperties.put(Env.CONNECTION_TIMEOUT_MS, properties.get(S3Properties.CONNECTION_TIMEOUT_MS)); - } - if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) { - beProperties.put(PropertyConverter.USE_PATH_STYLE, properties.get(PropertyConverter.USE_PATH_STYLE)); - } - if (properties.containsKey(S3Properties.PROVIDER)) { - beProperties.put(S3Properties.PROVIDER, properties.get(S3Properties.PROVIDER)); - } - - if (properties.containsKey(S3Properties.ROLE_ARN)) { - beProperties.put(S3Properties.Env.ROLE_ARN, properties.get(S3Properties.ROLE_ARN)); - } - - if (properties.containsKey(S3Properties.EXTERNAL_ID)) { - beProperties.put(S3Properties.Env.EXTERNAL_ID, properties.get(S3Properties.EXTERNAL_ID)); - } - return beProperties; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java index 6ea98607df0f93..022ecebf53da71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java @@ -32,10 +32,4 @@ public static CloudCredential getCloudCredential(Map props, credential.setSessionToken(props.getOrDefault(sessionTokenName, "")); return credential; } - - public static CloudCredential getCompatibleCredential(Map props) { - // Compatible with older versions. - return getCloudCredential(props, S3Properties.Env.ACCESS_KEY, S3Properties.Env.SECRET_KEY, - S3Properties.Env.TOKEN); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BosProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BosProperties.java deleted file mode 100644 index 0683bca4cd01f0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BosProperties.java +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.analysis.StorageBackend; -import org.apache.doris.common.Config; - -import org.apache.commons.collections.map.CaseInsensitiveMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.utils.URIBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Map; - -public class BosProperties { - private static final Logger LOG = LoggerFactory.getLogger(BosProperties.class); - - private static final String BOS_ENDPOINT = "bos_endpoint"; - private static final String BOS_ACCESS_KEY = "bos_accesskey"; - private static final String BOS_SECRET_ACCESS_KEY = "bos_secret_accesskey"; - - public static boolean tryConvertBosToS3(Map properties, StorageBackend.StorageType storageType) { - if (!Config.enable_access_file_without_broker || storageType != StorageBackend.StorageType.BROKER) { - return false; - } - CaseInsensitiveMap ciProperties = new CaseInsensitiveMap(); - ciProperties.putAll(properties); - if (StringUtils.isNotEmpty(ciProperties.get(BOS_ENDPOINT).toString()) - && StringUtils.isNotEmpty(ciProperties.get(BOS_ACCESS_KEY).toString()) - && StringUtils.isNotEmpty(ciProperties.get(BOS_SECRET_ACCESS_KEY).toString())) { - // bos endpoint like http[s]://gz.bcebos.com, we want to extract region gz, - // and convert to s3 endpoint http[s]://s3.gz.bcebos.com - String bosEndpiont = ciProperties.get(BOS_ENDPOINT).toString(); - try { - URI uri = new URI(bosEndpiont); - String host = uri.getHost(); - String[] hostSplit = host.split("\\."); - if (hostSplit.length < 3) { - return false; - } - String region = hostSplit[0]; - String s3Endpoint = new URIBuilder(uri).setHost("s3." + host).build().toString(); - properties.clear(); - properties.put(S3Properties.Env.ENDPOINT, s3Endpoint); - properties.put(S3Properties.Env.REGION, region); - properties.put(S3Properties.Env.ACCESS_KEY, ciProperties.get(BOS_ACCESS_KEY).toString()); - properties.put(S3Properties.Env.SECRET_KEY, ciProperties.get(BOS_SECRET_ACCESS_KEY).toString()); - - LOG.info("skip BROKER and access S3 directly."); - return true; - } catch (URISyntaxException e) { - LOG.warn(BOS_ENDPOINT + ": " + bosEndpiont + " is invalid."); - } - } - return false; - } - - public static String convertPathToS3(String path) { - try { - URI orig = new URI(path); - URI s3url = new URI("s3", orig.getRawAuthority(), - orig.getRawPath(), orig.getRawQuery(), orig.getRawFragment()); - return s3url.toString(); - } catch (URISyntaxException e) { - return path; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java deleted file mode 100644 index bdf99a76fb7052..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.credentials.CloudCredential; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class CosProperties extends BaseProperties { - public static final String COS_PREFIX = "cos."; - public static final String COS_FS_PREFIX = "fs.cos"; - - public static final String ENDPOINT = "cos.endpoint"; - public static final String ACCESS_KEY = "cos.access_key"; - public static final String SECRET_KEY = "cos.secret_key"; - public static final String REGION = "cos.region"; - public static final String SESSION_TOKEN = "cos.session_token"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY); - - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java deleted file mode 100644 index 2ab005d020f4c9..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.credentials.CloudCredential; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class GCSProperties extends BaseProperties { - - public static final String GCS_PREFIX = "gs."; - - public static final String ENDPOINT = "gs.endpoint"; - public static final String REGION = "gs.region"; - public static final String ACCESS_KEY = "gs.access_key"; - public static final String SECRET_KEY = "gs.secret_key"; - public static final String SESSION_TOKEN = "gs.session_token"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY); - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java deleted file mode 100644 index ff6115bf35e104..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.credentials.CloudCredential; - -import com.amazonaws.glue.catalog.util.AWSGlueConfig; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class GlueProperties extends BaseProperties { - public static final String ENDPOINT = "glue.endpoint"; - public static final String REGION = "glue.region"; - public static final String ACCESS_KEY = "glue.access_key"; - public static final String SECRET_KEY = "glue.secret_key"; - public static final String SESSION_TOKEN = "glue.session_token"; - - public static final String CLIENT_CREDENTIALS_PROVIDER = "client.credentials-provider"; - public static final String CLIENT_CREDENTIALS_PROVIDER_AK = "client.credentials-provider.glue.access_key"; - public static final String CLIENT_CREDENTIALS_PROVIDER_SK = "client.credentials-provider.glue.secret_key"; - - public static final List META_KEYS = Arrays.asList(AWSGlueConfig.AWS_GLUE_ENDPOINT, - AWSGlueConfig.AWS_REGION, AWSGlueConfig.AWS_GLUE_ACCESS_KEY, AWSGlueConfig.AWS_GLUE_SECRET_KEY, - AWSGlueConfig.AWS_GLUE_SESSION_TOKEN, CLIENT_CREDENTIALS_PROVIDER, CLIENT_CREDENTIALS_PROVIDER_AK, - CLIENT_CREDENTIALS_PROVIDER_SK); - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } - - public static CloudCredential getCompatibleCredential(Map props) { - // Compatible with older versions. - CloudCredential credential = getCloudCredential(props, AWSGlueConfig.AWS_GLUE_ACCESS_KEY, - AWSGlueConfig.AWS_GLUE_SECRET_KEY, AWSGlueConfig.AWS_GLUE_SESSION_TOKEN); - if (!credential.isWhole()) { - credential = BaseProperties.getCompatibleCredential(props); - } - return credential; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/HMSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/HMSProperties.java deleted file mode 100644 index 81baf042faed37..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/HMSProperties.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import java.util.Collections; -import java.util.List; - -public class HMSProperties { - public static final String HIVE_METASTORE_TYPE = "hive.metastore.type"; - public static final String DLF_TYPE = "dlf"; - public static final String GLUE_TYPE = "glue"; - public static final String HIVE_VERSION = "hive.version"; - // required - public static final String HIVE_METASTORE_URIS = "hive.metastore.uris"; - public static final List REQUIRED_FIELDS = Collections.singletonList(HMSProperties.HIVE_METASTORE_URIS); - public static final String ENABLE_HMS_EVENTS_INCREMENTAL_SYNC = "hive.enable_hms_events_incremental_sync"; - public static final String HMS_EVENTIS_BATCH_SIZE_PER_RPC = "hive.hms_events_batch_size_per_rpc"; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java index 28375b4f371de2..744a36aed37258 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java @@ -33,7 +33,6 @@ public class MCProperties extends BaseProperties { public static final String ODPS_ENDPOINT = "mc.odps_endpoint"; public static final String TUNNEL_SDK_ENDPOINT = "mc.tunnel_endpoint"; - public static final String PROJECT = "mc.default.project"; public static final String SESSION_TOKEN = "mc.session_token"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java deleted file mode 100644 index 0499f5fd6fbcf6..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.credentials.CloudCredential; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class MinioProperties extends BaseProperties { - - public static final String MINIO_PREFIX = "minio."; - public static final String ENDPOINT = "minio.endpoint"; - public static final String REGION = "minio.region"; - public static final String ACCESS_KEY = "minio.access_key"; - public static final String SECRET_KEY = "minio.secret_key"; - public static final String SESSION_TOKEN = "minio.session_token"; - public static final String DEFAULT_REGION = "us-east-1"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY, REGION); - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java deleted file mode 100644 index 19ff097fc321d7..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - - -import org.apache.doris.common.credentials.CloudCredential; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class ObsProperties extends BaseProperties { - public static final String OBS_PREFIX = "obs."; - public static final String OBS_FS_PREFIX = "fs.obs"; - - public static final String ENDPOINT = "obs.endpoint"; - public static final String REGION = "obs.region"; - public static final String ACCESS_KEY = "obs.access_key"; - public static final String SECRET_KEY = "obs.secret_key"; - public static final String SESSION_TOKEN = "obs.session_token"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY); - - public static class FS { - public static final String SESSION_TOKEN = "fs.obs.session.token"; - public static final String IMPL_DISABLE_CACHE = "fs.obs.impl.disable.cache"; - } - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java deleted file mode 100644 index 56497748b8baaf..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.credentials.CloudCredential; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class OssProperties extends BaseProperties { - - public static final String OSS_PREFIX = "oss."; - public static final String OSS_REGION_PREFIX = "oss-"; - public static final String OSS_FS_PREFIX = "fs.oss"; - - public static final String ENDPOINT = "oss.endpoint"; - public static final String REGION = "oss.region"; - public static final String ACCESS_KEY = "oss.access_key"; - public static final String SECRET_KEY = "oss.secret_key"; - public static final String SESSION_TOKEN = "oss.session_token"; - public static final String OSS_HDFS_ENABLED = "oss.hdfs.enabled"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY); - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java deleted file mode 100644 index 1a430fc997caf7..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.credentials.CloudCredential; - -import com.google.common.collect.Maps; - -import java.util.Map; - -public class PaimonProperties { - public static final String WAREHOUSE = "warehouse"; - public static final String S3_PATH_STYLE = "s3.path.style.access"; - public static final String FILE_FORMAT = "file.format"; - public static final String PAIMON_PREFIX = "paimon."; - public static final String PAIMON_CATALOG_TYPE = "metastore"; - public static final String HIVE_METASTORE_URIS = "uri"; - public static final String PAIMON_S3_ENDPOINT = "s3.endpoint"; - public static final String PAIMON_S3_ACCESS_KEY = "s3.access-key"; - public static final String PAIMON_S3_SECRET_KEY = "s3.secret-key"; - public static final String PAIMON_OSS_ENDPOINT = org.apache.hadoop.fs.aliyun.oss.Constants.ENDPOINT_KEY; - public static final String PAIMON_OSS_ACCESS_KEY = org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID; - public static final String PAIMON_OSS_SECRET_KEY = org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET; - public static final String PAIMON_HMS_CATALOG = "hive"; - public static final String PAIMON_FILESYSTEM_CATALOG = "filesystem"; - public static final String PAIMON_METASTORE_CLIENT = "metastore.client.class"; - - - public static Map convertToS3Properties(Map properties, - CloudCredential credential) { - Map s3Properties = Maps.newHashMap(); - s3Properties.put(PAIMON_S3_ACCESS_KEY, properties.get(S3Properties.ACCESS_KEY)); - s3Properties.put(PAIMON_S3_SECRET_KEY, properties.get(S3Properties.SECRET_KEY)); - s3Properties.putAll(properties); - // remove extra meta properties - s3Properties.remove(S3Properties.ACCESS_KEY); - s3Properties.remove(S3Properties.SECRET_KEY); - return s3Properties; - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 82a71548b69293..0b7eccbb21791b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -20,26 +20,14 @@ import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB; import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.common.credentials.DataLakeAWSCredentialsProvider; -import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.thrift.TCredProviderType; import org.apache.doris.thrift.TS3StorageParam; -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import com.amazonaws.auth.SystemPropertiesCredentialsProvider; -import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; -import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import java.util.Arrays; import java.util.HashMap; @@ -49,10 +37,9 @@ public class S3Properties extends BaseProperties { + public static final String USE_PATH_STYLE = "use_path_style"; public static final String S3_PREFIX = "s3."; - public static final String S3_FS_PREFIX = "fs.s3"; - public static final String CREDENTIALS_PROVIDER = "s3.credentials.provider"; public static final String ENDPOINT = "s3.endpoint"; public static final String EXTERNAL_ENDPOINT = "s3.external_endpoint"; public static final String REGION = "s3.region"; @@ -66,7 +53,6 @@ public class S3Properties extends BaseProperties { public static final String MAX_CONNECTIONS = "s3.connection.maximum"; public static final String REQUEST_TIMEOUT_MS = "s3.connection.request.timeout"; public static final String CONNECTION_TIMEOUT_MS = "s3.connection.timeout"; - public static final String S3_PROVIDER = "S3"; // required by storage policy public static final String ROOT_PATH = "s3.root.path"; @@ -80,17 +66,6 @@ public class S3Properties extends BaseProperties { public static final List PROVIDERS = Arrays.asList("COS", "OSS", "S3", "OBS", "BOS", "AZURE", "GCP", "TOS"); - public static final List AWS_CREDENTIALS_PROVIDERS = Arrays.asList( - DataLakeAWSCredentialsProvider.class.getName(), - TemporaryAWSCredentialsProvider.class.getName(), - SimpleAWSCredentialsProvider.class.getName(), - EnvironmentVariableCredentialsProvider.class.getName(), - SystemPropertiesCredentialsProvider.class.getName(), - InstanceProfileCredentialsProvider.class.getName(), - ProfileCredentialsProvider.class.getName(), - WebIdentityTokenCredentialsProvider.class.getName(), - IAMInstanceCredentialsProvider.class.getName()); - private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})"); public static Map credentialToMap(CloudCredentialWithEndpoint credential) { @@ -147,7 +122,7 @@ public static CloudCredentialWithEndpoint getEnvironmentCredentialWithEndpoint(M } String endpoint = props.get(Env.ENDPOINT); String region = props.getOrDefault(Env.REGION, S3Properties.getRegionOfEndpoint(endpoint)); - props.putIfAbsent(Env.REGION, PropertyConverter.checkRegion(endpoint, region, Env.REGION)); + props.putIfAbsent(Env.REGION, checkRegion(endpoint, region, Env.REGION)); return new CloudCredentialWithEndpoint(endpoint, region, credential); } @@ -169,43 +144,6 @@ public static String getRegionOfEndpoint(String endpoint) { return endpointSplit[1]; } - public static Map prefixToS3(Map properties) { - Map s3Properties = Maps.newHashMap(); - for (Map.Entry entry : properties.entrySet()) { - if (entry.getKey().startsWith(OssProperties.OSS_PREFIX)) { - String s3Key = entry.getKey().replace(OssProperties.OSS_PREFIX, S3Properties.S3_PREFIX); - s3Properties.put(s3Key, entry.getValue()); - } else if (entry.getKey().startsWith(GCSProperties.GCS_PREFIX)) { - String s3Key = entry.getKey().replace(GCSProperties.GCS_PREFIX, S3Properties.S3_PREFIX); - s3Properties.put(s3Key, entry.getValue()); - } else if (entry.getKey().startsWith(CosProperties.COS_PREFIX)) { - String s3Key = entry.getKey().replace(CosProperties.COS_PREFIX, S3Properties.S3_PREFIX); - s3Properties.put(s3Key, entry.getValue()); - } else if (entry.getKey().startsWith(ObsProperties.OBS_PREFIX)) { - String s3Key = entry.getKey().replace(ObsProperties.OBS_PREFIX, S3Properties.S3_PREFIX); - s3Properties.put(s3Key, entry.getValue()); - } else if (entry.getKey().startsWith(MinioProperties.MINIO_PREFIX)) { - String s3Key = entry.getKey().replace(MinioProperties.MINIO_PREFIX, S3Properties.S3_PREFIX); - s3Properties.put(s3Key, entry.getValue()); - } else { - s3Properties.put(entry.getKey(), entry.getValue()); - } - } - return s3Properties; - } - - public static Map requiredS3TVFProperties(Map properties) - throws AnalysisException { - try { - for (String field : S3Properties.TVF_REQUIRED_FIELDS) { - checkRequiredProperty(properties, field); - } - } catch (DdlException e) { - throw new AnalysisException(e.getMessage(), e); - } - return properties; - } - private static void checkProvider(Map properties) throws DdlException { if (properties.containsKey(PROVIDER)) { properties.put(PROVIDER, properties.get(PROVIDER).toUpperCase()); @@ -290,8 +228,8 @@ public static void convertToStdProperties(Map properties) { if (properties.containsKey(S3Properties.Env.BUCKET)) { properties.putIfAbsent(S3Properties.BUCKET, properties.get(S3Properties.Env.BUCKET)); } - if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) { - properties.putIfAbsent(PropertyConverter.USE_PATH_STYLE, properties.get(PropertyConverter.USE_PATH_STYLE)); + if (properties.containsKey(USE_PATH_STYLE)) { + properties.putIfAbsent(USE_PATH_STYLE, properties.get(USE_PATH_STYLE)); } if (properties.containsKey(S3Properties.Env.ROLE_ARN)) { @@ -331,7 +269,7 @@ public static TS3StorageParam getS3TStorageParam(Map properties) String connTimeoutMs = properties.get(S3Properties.CONNECTION_TIMEOUT_MS); s3Info.setConnTimeoutMs(Integer.parseInt(connTimeoutMs == null ? S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS : connTimeoutMs)); - String usePathStyle = properties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false"); + String usePathStyle = properties.getOrDefault(USE_PATH_STYLE, "false"); s3Info.setUsePathStyle(Boolean.parseBoolean(usePathStyle)); return s3Info; } @@ -366,8 +304,8 @@ public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map props) { @Override public void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); - this.brokerParams = new HashMap<>(origProps); - //why need this convert - this.brokerParams.putAll(PropertyConverter.convertToHadoopFSProperties(origProps)); + this.brokerParams = Maps.newHashMap(extractBrokerProperties()); } @Override @@ -93,4 +93,14 @@ public String getStorageName() { public void initializeHadoopStorageConfig() { // do nothing } + + private Map extractBrokerProperties() { + Map brokerProperties = new HashMap<>(); + for (String key : origProps.keySet()) { + if (key.startsWith(BROKER_PREFIX)) { + brokerProperties.put(key.substring(BROKER_PREFIX.length()), origProps.get(key)); + } + } + return brokerProperties; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java index 57576aef44e40d..af4c332442481d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java @@ -59,6 +59,7 @@ public class COSProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"cos.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, required = false, + sensitive = true, description = "The secret key of COS.") protected String secretKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java index f222097d1d24ba..dffd18d0d58ec2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java @@ -83,6 +83,7 @@ public class GCSProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"gs.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, required = false, + sensitive = true, description = "The secret key of GCS.") protected String secretKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java index dd8d2735324c8f..1f52a84d392c52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java @@ -47,12 +47,14 @@ public class MinioProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"minio.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, required = false, + sensitive = true, description = "The secret key of Minio.") protected String secretKey = ""; @Getter @ConnectorProperty(names = {"minio.session_token", "s3.session_token", "session_token"}, required = false, + sensitive = true, description = "The session token of Minio.") protected String sessionToken = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java index eea6c24ddc42aa..e2adb19895c105 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java @@ -52,6 +52,7 @@ public class OBSProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"obs.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, required = false, + sensitive = true, description = "The secret key of OBS.") protected String secretKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java index edc00c48d8399b..e5919d85286931 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java @@ -59,6 +59,7 @@ public class OSSHdfsProperties extends HdfsCompatibleProperties { protected String accessKey = ""; @ConnectorProperty(names = {"oss.hdfs.secret_key", "dlf.secret_key", "dlf.catalog.secret_key", "oss.secret_key"}, + sensitive = true, description = "The secret key of OSS.") protected String secretKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 8e68891ce33925..3ddb26e696927b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -56,7 +56,8 @@ public class OSSProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"oss.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY", - "dlf.secret_key", "dlf.catalog.secret_key", "fs.oss.accessKeySecret"}, + "dlf.secret_key", "dlf.catalog.secret_key","fs.oss.accessKeySecret"}, + sensitive = true, required = false, description = "The secret key of OSS.") protected String secretKey = ""; @@ -76,6 +77,7 @@ public class OSSProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"oss.session_token", "s3.session_token", "session_token", "fs.oss.securityToken"}, required = false, + sensitive = true, description = "The session token of OSS.") protected String sessionToken = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 71fd1a2112c20f..64b3c9086e650c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -44,6 +44,7 @@ import java.util.stream.Stream; public class S3Properties extends AbstractS3CompatibleProperties { + public static final String USE_PATH_STYLE = "use_path_style"; private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", @@ -121,7 +122,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Setter @Getter - @ConnectorProperty(names = {"use_path_style", "s3.path-style-access"}, required = false, + @ConnectorProperty(names = {USE_PATH_STYLE, "s3.path-style-access"}, required = false, description = "Whether to use path style URL for the storage.") protected String usePathStyle = "false"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java index bb66e66f5dcf81..49c7c5612a747e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java @@ -25,6 +25,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Map; import java.util.Optional; @@ -167,4 +169,15 @@ public static String validateAndGetUri(Map props) { } return uriOptional.get(); } + + public static String convertPathToS3(String path) { + try { + URI orig = new URI(path); + URI s3url = new URI("s3", orig.getRawAuthority(), + orig.getRawPath(), orig.getRawQuery(), orig.getRawFragment()); + return s3url.toString(); + } catch (URISyntaxException e) { + return path; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java index 8dcb37269f1d94..3627aff636db77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java @@ -22,7 +22,6 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InitCatalogLog.Type; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.property.constants.TrinoConnectorProperties; import org.apache.doris.trinoconnector.TrinoConnectorServicesProvider; import com.google.common.collect.ImmutableList; @@ -96,9 +95,10 @@ public class TrinoConnectorExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(TrinoConnectorExternalCatalog.class); private static final String TRINO_CONNECTOR_PROPERTIES_PREFIX = "trino."; + public static final String TRINO_CONNECTOR_NAME = "trino.connector.name"; private static final List TRINO_CONNECTOR_REQUIRED_PROPERTIES = ImmutableList.of( - TrinoConnectorProperties.TRINO_CONNECTOR_NAME + TRINO_CONNECTOR_NAME ); private CatalogHandle trinoCatalogHandle; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 30af287bca1e72..1eb01a6faa21fc 100644 --- a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -20,7 +20,7 @@ import org.apache.doris.datasource.hive.HiveVersionUtil; import org.apache.doris.datasource.hive.HiveVersionUtil.HiveVersion; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -354,8 +354,8 @@ public HiveMetaStoreClient(Configuration conf, HiveMetaHookLoader hookLoader, Bo this.conf = new Configuration(conf); } - hiveVersion = HiveVersionUtil.getVersion(conf.get(HMSProperties.HIVE_VERSION)); - LOG.info("Loading Doris HiveMetaStoreClient. Hive version: " + conf.get(HMSProperties.HIVE_VERSION)); + hiveVersion = HiveVersionUtil.getVersion(conf.get(HMSBaseProperties.HIVE_VERSION)); + LOG.info("Loading Doris HiveMetaStoreClient. Hive version: " + conf.get(HMSBaseProperties.HIVE_VERSION)); // For hive 2.3.7, there is no ClientCapability.INSERT_ONLY_TABLES if (hiveVersion == HiveVersion.V1_0 || hiveVersion == HiveVersion.V2_0 || hiveVersion == HiveVersion.V2_3) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/HiveTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/HiveTableTest.java index abfcc68790cf7e..06ba92d1ddfc70 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/HiveTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/HiveTableTest.java @@ -18,7 +18,7 @@ package org.apache.doris.catalog; import org.apache.doris.common.DdlException; -import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.metastore.HMSBaseProperties; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -81,8 +81,8 @@ public void testNoHiveMetastoreUris() throws DdlException { @Test() public void testVersion() throws DdlException { - properties.put(HMSProperties.HIVE_VERSION, "2.1.2"); + properties.put(HMSBaseProperties.HIVE_VERSION, "2.1.2"); HiveTable table = new HiveTable(1000, "hive_table", columns, properties); - Assert.assertEquals("2.1.2", table.getHiveProperties().get(HMSProperties.HIVE_VERSION)); + Assert.assertEquals("2.1.2", table.getHiveProperties().get(HMSBaseProperties.HIVE_VERSION)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java index eedd70be56787a..f371e83af47293 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java @@ -18,7 +18,6 @@ package org.apache.doris.external.iceberg; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.remote.dfs.DFSFileSystem; @@ -49,9 +48,8 @@ public void testHadoopCatalogListNamespaces() throws UserException, IOException properties.put("cos.secret_key", "yyy"); properties.put("cos.endpoint", "cos.ap-beijing.myqcloud.com"); properties.put("cos.region", "ap-beijing"); - Map hadoopProps = PropertyConverter.convertToHadoopFSProperties(properties); String pathStr = "cosn://bucket1/namespace"; - DFSFileSystem fs = (DFSFileSystem) FileSystemFactory.get(StorageProperties.createPrimary(hadoopProps)); + DFSFileSystem fs = (DFSFileSystem) FileSystemFactory.get(StorageProperties.createPrimary(properties)); nativeFs = fs.nativeFileSystem(new Path(pathStr)); RemoteIterator it = nativeFs.listStatusIterator(new Path(pathStr)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java index f3db1561490d08..558b14f4d105ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java @@ -21,7 +21,7 @@ import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; -import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemFactory; @@ -73,7 +73,7 @@ public void setUp() throws Exception { properties.put("AWS_ACCESS_KEY", System.getenv().getOrDefault("AWS_AK", "")); properties.put("AWS_SECRET_KEY", System.getenv().getOrDefault("AWS_SK", "")); properties.put("AWS_ENDPOINT", "http://s3.ap-northeast-1.amazonaws.com"); - properties.put(PropertyConverter.USE_PATH_STYLE, "false"); + properties.put(S3Properties.USE_PATH_STYLE, "false"); properties.put("AWS_REGION", "bj"); content = "O wild West Wind, thou breath of Autumn's being\n" From b618330fcf84fcb6b25128afccc9d5a88a33e14e Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 26 Sep 2025 14:02:28 +0800 Subject: [PATCH 2/5] [Chore](auto-pick)4.0.x autopick --- .../java/org/apache/doris/analysis/CreateStorageVaultStmt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java index c001fba80ce287..eb76d826fc3008 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.trees.plans.commands; +package org.apache.doris.analysis; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.StorageVault; From aa3a4c2a2b0a1b73c74301abcbd8a8888f23e119 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 18 Sep 2025 15:02:53 +0800 Subject: [PATCH 3/5] [chore](refactor-params)remove old properties (#56163) remove oldcode (cherry picked from commit 87f8248962f0e32353e47985c45e21d18dc33e0b) --- .../doris/analysis/AlterRepositoryStmt.java | 11 - .../doris/analysis/CreateResourceStmt.java | 5 +- .../analysis/CreateStorageVaultStmt.java | 3 +- .../apache/doris/backup/BackupHandler.java | 36 +- .../org/apache/doris/backup/Repository.java | 35 -- .../apache/doris/catalog/AzureResource.java | 2 +- .../doris/catalog/HdfsStorageVault.java | 9 +- .../org/apache/doris/catalog/HiveTable.java | 6 +- .../org/apache/doris/catalog/S3Resource.java | 2 +- .../apache/doris/catalog/S3StorageVault.java | 9 +- .../apache/doris/catalog/StorageVaultMgr.java | 11 +- .../doris/cloud/storage/AzureRemote.java | 2 +- .../doris/common/util/PrintableMap.java | 7 +- .../lakesoul/source/LakeSoulScanNode.java | 18 +- .../property/constants/AzureProperties.java | 70 ---- .../property/constants/S3Properties.java | 336 ------------------ .../property/storage/AzureProperties.java | 5 +- .../property/storage/S3Properties.java | 274 ++++++++++++++ .../doris/load/loadv2/BrokerLoadJob.java | 2 +- .../apache/doris/policy/StoragePolicy.java | 2 +- .../doris/task/PushStoragePolicyTask.java | 2 +- .../apache/doris/catalog/S3ResourceTest.java | 2 +- .../cloud/catalog/HdfsStorageVaultTest.java | 2 +- .../constants/AzurePropertiesTest.java | 55 --- .../doris/fs/obj/AzureObjStorageTest.java | 2 +- .../apache/doris/fs/obj/S3FileSystemTest.java | 2 +- .../fs/obj/S3ObjStorageGlobListTest.java | 2 +- 27 files changed, 323 insertions(+), 589 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AzureProperties.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/constants/AzurePropertiesTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java index ca834aaa653e93..044c4171106e59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java @@ -23,7 +23,6 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -64,16 +63,6 @@ public void analyze(Analyzer analyzer) throws UserException { if (copyProperties.size() == 0) { throw new UserException("alter repository need contains ak/sk/token info of s3."); } - copyProperties.remove(S3Properties.ACCESS_KEY); - copyProperties.remove(S3Properties.SECRET_KEY); - copyProperties.remove(S3Properties.SESSION_TOKEN); - copyProperties.remove(S3Properties.Env.ACCESS_KEY); - copyProperties.remove(S3Properties.Env.SECRET_KEY); - copyProperties.remove(S3Properties.Env.TOKEN); - if (copyProperties.size() != 0) { - throw new UserException("alter repository only support ak/sk/token info of s3." - + " unsupported properties: " + copyProperties.keySet()); - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java index 8e3a2f3eedeeeb..e44a56149e6838 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java @@ -25,8 +25,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.AzureProperties; +import org.apache.doris.datasource.property.storage.AzureProperties; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -82,7 +81,7 @@ public void analyzeResourceType() throws UserException { throw new AnalysisException("Resource type can't be null"); } - if (AzureProperties.checkAzureProviderPropertyExist(properties)) { + if (AzureProperties.guessIsMe(properties)) { resourceType = ResourceType.AZURE; return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java index eb76d826fc3008..65dabf9c896f00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java @@ -27,8 +27,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 5db2e28fd070c4..c5e6b180f065bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -52,9 +52,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.remote.AzureFileSystem; import org.apache.doris.fs.remote.RemoteFileSystem; -import org.apache.doris.fs.remote.S3FileSystem; import org.apache.doris.persist.BarrierLog; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; @@ -237,12 +235,10 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { * * @param repoName The name of the repository to alter. * @param newProps The new properties to apply to the repository. - * @param strictCheck If true, only allows altering S3 or Azure repositories and validates properties accordingly. - * TODO: Investigate why only S3 and Azure repositories are supported for alter operation * @throws DdlException if the repository does not exist, fails to apply properties, or cannot connect * to the updated repository. */ - public void alterRepository(String repoName, Map newProps, boolean strictCheck) + public void alterRepository(String repoName, Map newProps) throws DdlException { tryLock(); try { @@ -251,7 +247,7 @@ public void alterRepository(String repoName, Map newProps, boole throw new DdlException("Repository does not exist"); } // Merge new properties with the existing repository's properties - Map mergedProps = mergeProperties(oldRepo, newProps, strictCheck); + Map mergedProps = mergeProperties(oldRepo, newProps); // Create new remote file system with merged properties RemoteFileSystem fileSystem = FileSystemFactory.get(StorageProperties.createPrimary(mergedProps)); // Create new Repository instance with updated file system @@ -278,34 +274,16 @@ public void alterRepository(String repoName, Map newProps, boole /** * Merges new user-provided properties into the existing repository's configuration. - * In strict mode, only supports S3 or Azure repositories and applies internal S3 merge logic. * * @param repo The existing repository. * @param newProps New user-specified properties. - * @param strictCheck Whether to enforce S3/Azure-only and validate the new properties. * @return A complete set of merged properties. - * @throws DdlException if the merge fails or the repository type is unsupported. */ - private Map mergeProperties(Repository repo, Map newProps, boolean strictCheck) - throws DdlException { - if (strictCheck) { - if (!(repo.getRemoteFileSystem() instanceof S3FileSystem - || repo.getRemoteFileSystem() instanceof AzureFileSystem)) { - throw new DdlException("Only support altering S3 or Azure repository"); - } - // Let the repository validate and enrich the new S3/Azure properties - Map propsCopy = new HashMap<>(newProps); - Status status = repo.alterRepositoryS3Properties(propsCopy); - if (!status.ok()) { - throw new DdlException("Failed to merge S3 properties: " + status.getErrMsg()); - } - return propsCopy; - } else { - // General case: just override old props with new ones - Map combined = new HashMap<>(repo.getRemoteFileSystem().getProperties()); - combined.putAll(newProps); - return combined; - } + private Map mergeProperties(Repository repo, Map newProps) { + // General case: just override old props with new ones + Map combined = new HashMap<>(repo.getRemoteFileSystem().getProperties()); + combined.putAll(newProps); + return combined; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 7e4fa7e9b19852..04aa01635c86a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -30,7 +30,6 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; @@ -39,7 +38,6 @@ import org.apache.doris.fs.remote.BrokerFileSystem; import org.apache.doris.fs.remote.RemoteFile; import org.apache.doris.fs.remote.RemoteFileSystem; -import org.apache.doris.fs.remote.S3FileSystem; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.system.Backend; @@ -69,7 +67,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.UUID; /* @@ -209,38 +206,6 @@ public static Repository read(DataInput in) throws IOException { } } - //todo why only support alter S3 properties - public Status alterRepositoryS3Properties(Map properties) { - if (this.fileSystem instanceof S3FileSystem) { - Map oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties()); - oldProperties.remove(S3Properties.ACCESS_KEY); - oldProperties.remove(S3Properties.SECRET_KEY); - oldProperties.remove(S3Properties.SESSION_TOKEN); - oldProperties.remove(S3Properties.Env.ACCESS_KEY); - oldProperties.remove(S3Properties.Env.SECRET_KEY); - oldProperties.remove(S3Properties.Env.TOKEN); - for (Map.Entry entry : properties.entrySet()) { - if (Objects.equals(entry.getKey(), S3Properties.ACCESS_KEY) - || Objects.equals(entry.getKey(), S3Properties.Env.ACCESS_KEY)) { - oldProperties.putIfAbsent(S3Properties.ACCESS_KEY, entry.getValue()); - } - if (Objects.equals(entry.getKey(), S3Properties.SECRET_KEY) - || Objects.equals(entry.getKey(), S3Properties.Env.SECRET_KEY)) { - oldProperties.putIfAbsent(S3Properties.SECRET_KEY, entry.getValue()); - } - if (Objects.equals(entry.getKey(), S3Properties.SESSION_TOKEN) - || Objects.equals(entry.getKey(), S3Properties.Env.TOKEN)) { - oldProperties.putIfAbsent(S3Properties.SESSION_TOKEN, entry.getValue()); - } - } - properties.clear(); - properties.putAll(oldProperties); - return Status.OK; - } else { - return new Status(ErrCode.COMMON_ERROR, "Only support alter s3 repository"); - } - } - @Override public void gsonPostProcess() { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java index 2a6c39f2c4452d..ef278e6dcdf3b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java @@ -21,8 +21,8 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.AzureProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.obj.AzureObjStorage; import org.apache.doris.fs.obj.ObjStorage; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java index 8476c9f4a5ef6d..c41d4540716a65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java @@ -22,7 +22,6 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.security.authentication.AuthenticationConfig; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.remote.dfs.DFSFileSystem; @@ -59,7 +58,7 @@ */ public class HdfsStorageVault extends StorageVault { private static final Logger LOG = LogManager.getLogger(HdfsStorageVault.class); - + public static final String S3_VALIDITY_CHECK = "s3_validity_check"; public static final String HADOOP_FS_PREFIX = "dfs."; public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit"; public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path"; @@ -87,7 +86,7 @@ public static class PropertyKey { * such as `type`, `path_prefix`, etc. */ private static final Set NON_HDFS_CONF_PROPERTY_KEYS = - ImmutableSet.of(StorageVault.PropertyKey.TYPE, PropertyKey.VAULT_PATH_PREFIX, S3Properties.VALIDITY_CHECK) + ImmutableSet.of(StorageVault.PropertyKey.TYPE, PropertyKey.VAULT_PATH_PREFIX, S3_VALIDITY_CHECK) .stream().map(String::toLowerCase) .collect(ImmutableSet.toImmutableSet()); @@ -113,8 +112,8 @@ public Map getCopiedProperties() { } public static void checkConnectivity(Map newProperties) throws DdlException { - if (newProperties.containsKey(S3Properties.VALIDITY_CHECK) - && newProperties.get(S3Properties.VALIDITY_CHECK).equalsIgnoreCase("false")) { + if (newProperties.containsKey(S3_VALIDITY_CHECK) + && newProperties.get(S3_VALIDITY_CHECK).equalsIgnoreCase("false")) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java index 9600f2f81f8e67..f89d4223f7a1ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java @@ -21,8 +21,8 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.security.authentication.AuthType; import org.apache.doris.common.security.authentication.AuthenticationConfig; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.metastore.HMSBaseProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.thrift.THiveTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -49,6 +49,8 @@ public class HiveTable extends Table { private static final String PROPERTY_ERROR_MSG = "Hive table properties('%s'='%s')" + " is illegal or not supported. Please check it"; + public static final String AWS_PROPERTIES_PREFIX = "AWS"; + @SerializedName("hdb") private String hiveDb; @SerializedName("ht") @@ -164,7 +166,7 @@ private void validate(Map properties) throws DdlException { String key = entry.getKey(); if (key.startsWith(HdfsResource.HADOOP_FS_PREFIX) || key.startsWith(S3Properties.S3_PREFIX) - || key.startsWith(S3Properties.Env.PROPERTIES_PREFIX)) { + || key.startsWith(AWS_PROPERTIES_PREFIX)) { hiveProperties.put(key, entry.getValue()); iter.remove(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index b199e1a0b4249f..4e4d5b7b63b60d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -22,8 +22,8 @@ import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.obj.ObjStorage; import org.apache.doris.fs.obj.RemoteObjects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java index f04b45010b1b83..3816fbe0e4216a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java @@ -20,7 +20,8 @@ import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.datasource.property.storage.StorageProperties; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -58,14 +59,14 @@ public class S3StorageVault extends StorageVault { // Reuse all the code from S3Resource private Resource resource; - + public static final String S3_ROOT_PATH = "s3.root.path"; public static class PropertyKey { public static final String ACCESS_KEY = S3Properties.ACCESS_KEY; public static final String SECRET_KEY = S3Properties.SECRET_KEY; public static final String USE_PATH_STYLE = S3Properties.USE_PATH_STYLE; - public static final String ROOT_PATH = S3Properties.ROOT_PATH; - public static final String PROVIDER = S3Properties.PROVIDER; + public static final String ROOT_PATH = S3_ROOT_PATH; + public static final String PROVIDER = StorageProperties.FS_PROVIDER_KEY; public static final String REGION = S3Properties.REGION; public static final String ENDPOINT = S3Properties.ENDPOINT; public static final String BUCKET = S3Properties.BUCKET; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index 37730b077ecd5c..faa8b8b5827d33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -27,8 +27,8 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; -import org.apache.doris.datasource.property.constants.S3Properties; -import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.rpc.BackendServiceProxy; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.Backend; @@ -210,7 +210,7 @@ public void alterStorageVault(StorageVaultType type, Map propert properties.keySet().stream() .filter(key -> HdfsStorageVault.FORBID_ALTER_PROPERTIES.contains(key) || key.toLowerCase().contains(S3Properties.S3_PREFIX) - || key.toLowerCase().contains(S3Properties.PROVIDER)) + || key.toLowerCase().contains(StorageProperties.FS_PROVIDER_KEY)) .findAny() .ifPresent(key -> { throw new IllegalArgumentException("Alter property " + key + " is not allowed."); @@ -240,11 +240,6 @@ public void alterStorageVault(StorageVaultType type, Map propert } } - @VisibleForTesting - public void setDefaultStorageVault(SetDefaultStorageVaultStmt stmt) throws DdlException { - setDefaultStorageVault(stmt.getStorageVaultName()); - } - public void setDefaultStorageVault(String vaultName) throws DdlException { Cloud.AlterObjStoreInfoRequest.Builder builder = Cloud.AlterObjStoreInfoRequest.newBuilder(); Cloud.StorageVaultPB.Builder vaultBuilder = Cloud.StorageVaultPB.newBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/AzureRemote.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/AzureRemote.java index 86d5610bb0ef4b..6e6857ea5d13e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/AzureRemote.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/AzureRemote.java @@ -18,7 +18,7 @@ package org.apache.doris.cloud.storage; import org.apache.doris.common.DdlException; -import org.apache.doris.datasource.property.constants.AzureProperties; +import org.apache.doris.datasource.property.storage.AzureProperties; import com.azure.core.credential.AccessToken; import com.azure.core.credential.TokenCredential; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java index bbfd82f827d4c7..784432ba9fe8cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java @@ -19,7 +19,6 @@ import org.apache.doris.datasource.property.ConnectorPropertiesUtils; import org.apache.doris.datasource.property.constants.MCProperties; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.metastore.AWSGlueMetaStoreBaseProperties; import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties; import org.apache.doris.datasource.property.storage.AzureProperties; @@ -29,6 +28,7 @@ import org.apache.doris.datasource.property.storage.OBSProperties; import org.apache.doris.datasource.property.storage.OSSHdfsProperties; import org.apache.doris.datasource.property.storage.OSSProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import com.google.common.collect.Sets; @@ -61,10 +61,8 @@ public class PrintableMap { SENSITIVE_KEY.add("jdbc.password"); SENSITIVE_KEY.add("elasticsearch.password"); SENSITIVE_KEY.addAll(Arrays.asList( - S3Properties.SECRET_KEY, - S3Properties.Env.SECRET_KEY, MCProperties.SECRET_KEY)); - + SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(S3Properties.class)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AliyunDLFBaseProperties.class)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AWSGlueMetaStoreBaseProperties.class)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(GCSProperties.class)); @@ -74,7 +72,6 @@ public class PrintableMap { SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(COSProperties.class)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OBSProperties.class)); SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(MinioProperties.class)); - HIDDEN_KEY = Sets.newHashSet(); HIDDEN_KEY.addAll(S3Properties.Env.FS_KEYS); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java index ddcb5ebb6b31d0..72366c46b91f8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java @@ -25,8 +25,6 @@ import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.lakesoul.LakeSoulExternalTable; import org.apache.doris.datasource.lakesoul.LakeSoulUtils; -import org.apache.doris.datasource.property.constants.MinioProperties; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; @@ -181,8 +179,8 @@ private void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulS LOG.debug("{}", catalogProps); } - if (catalogProps.get(S3Properties.Env.ENDPOINT) != null) { - options.put(LakeSoulUtils.FS_S3A_ENDPOINT, catalogProps.get(S3Properties.Env.ENDPOINT)); + if (catalogProps.get("AWS_ENDPOINT") != null) { + options.put(LakeSoulUtils.FS_S3A_ENDPOINT, catalogProps.get("AWS_ENDPOINT")); if (!options.containsKey("oss.endpoint")) { // Aliyun OSS requires virtual host style access options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "false"); @@ -190,14 +188,14 @@ private void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulS // use path style access for all other s3 compatible storage services options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "true"); } - if (catalogProps.get(S3Properties.Env.ACCESS_KEY) != null) { - options.put(LakeSoulUtils.FS_S3A_ACCESS_KEY, catalogProps.get(S3Properties.Env.ACCESS_KEY)); + if (catalogProps.get("AWS_ACCESS_KEY") != null) { + options.put(LakeSoulUtils.FS_S3A_ACCESS_KEY, catalogProps.get("AWS_ACCESS_KEY")); } - if (catalogProps.get(S3Properties.Env.SECRET_KEY) != null) { - options.put(LakeSoulUtils.FS_S3A_SECRET_KEY, catalogProps.get(S3Properties.Env.SECRET_KEY)); + if (catalogProps.get("AWS_SECRET_KEY") != null) { + options.put(LakeSoulUtils.FS_S3A_SECRET_KEY, catalogProps.get("AWS_SECRET_KEY")); } - if (catalogProps.get(S3Properties.Env.REGION) != null) { - options.put(LakeSoulUtils.FS_S3A_REGION, catalogProps.get(S3Properties.Env.REGION)); + if (catalogProps.get("AWS_REGION") != null) { + options.put(LakeSoulUtils.FS_S3A_REGION, catalogProps.get("AWS_REGION")); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AzureProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AzureProperties.java deleted file mode 100644 index 70ba490ccfd1e4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/AzureProperties.java +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.Config; -import org.apache.doris.common.credentials.CloudCredential; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class AzureProperties extends BaseProperties { - public static final String AZURE_PREFIX = "azure."; - public static final String AZURE_FS_PREFIX = "fs.azure"; - public static final String AZURE_NAME = "AZURE"; - - public static final String ENDPOINT = "azure.endpoint"; - public static final String REGION = "azure.region"; - public static final String ACCESS_KEY = "azure.access_key"; - public static final String SECRET_KEY = "azure.secret_key"; - public static final String SESSION_TOKEN = "azure.session_token"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY); - - public static final String AZURE_ENDPOINT_TEMPLATE = "https://%s.blob.core.windows.net"; - - public static class FS { - public static final String SESSION_TOKEN = "fs.azure.session.token"; - public static final String IMPL_DISABLE_CACHE = "fs.azure.impl.disable.cache"; - } - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } - - public static boolean checkAzureProviderPropertyExist(Map properties) { - for (Map.Entry entry : properties.entrySet()) { - if (entry.getKey().toLowerCase().contains(S3Properties.PROVIDER) - && entry.getValue().toUpperCase().equals(AzureProperties.AZURE_NAME)) { - return true; - } - } - return false; - } - - public static String formatAzureEndpoint(String endpoint, String accountName) { - if (Config.force_azure_blob_global_endpoint) { - return String.format(AZURE_ENDPOINT_TEMPLATE, accountName); - } - if (endpoint.contains("://")) { - return endpoint; - } - return "https://" + endpoint; - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java deleted file mode 100644 index 0b7eccbb21791b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ /dev/null @@ -1,336 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.cloud.proto.Cloud; -import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB; -import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.credentials.CloudCredential; -import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.thrift.TCredProviderType; -import org.apache.doris.thrift.TS3StorageParam; - -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -public class S3Properties extends BaseProperties { - - public static final String USE_PATH_STYLE = "use_path_style"; - public static final String S3_PREFIX = "s3."; - - public static final String ENDPOINT = "s3.endpoint"; - public static final String EXTERNAL_ENDPOINT = "s3.external_endpoint"; - public static final String REGION = "s3.region"; - public static final String ACCESS_KEY = "s3.access_key"; - public static final String SECRET_KEY = "s3.secret_key"; - public static final String SESSION_TOKEN = "s3.session_token"; - - public static final String ROLE_ARN = "s3.role_arn"; - public static final String EXTERNAL_ID = "s3.external_id"; - - public static final String MAX_CONNECTIONS = "s3.connection.maximum"; - public static final String REQUEST_TIMEOUT_MS = "s3.connection.request.timeout"; - public static final String CONNECTION_TIMEOUT_MS = "s3.connection.timeout"; - - // required by storage policy - public static final String ROOT_PATH = "s3.root.path"; - public static final String BUCKET = "s3.bucket"; - public static final String VALIDITY_CHECK = "s3_validity_check"; - public static final String PROVIDER = "provider"; - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT); - public static final List TVF_REQUIRED_FIELDS = Arrays.asList(ACCESS_KEY, SECRET_KEY); - public static final List FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN, - ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS); - - public static final List PROVIDERS = Arrays.asList("COS", "OSS", "S3", "OBS", "BOS", "AZURE", "GCP", "TOS"); - - private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})"); - - public static Map credentialToMap(CloudCredentialWithEndpoint credential) { - Map resMap = new HashMap<>(); - resMap.put(S3Properties.ENDPOINT, credential.getEndpoint()); - resMap.put(S3Properties.REGION, credential.getRegion()); - if (credential.isWhole()) { - resMap.put(S3Properties.ACCESS_KEY, credential.getAccessKey()); - resMap.put(S3Properties.SECRET_KEY, credential.getSecretKey()); - } - if (credential.isTemporary()) { - resMap.put(S3Properties.SESSION_TOKEN, credential.getSessionToken()); - } - return resMap; - } - - public static class Env { - public static final String PROPERTIES_PREFIX = "AWS"; - // required - public static final String ENDPOINT = "AWS_ENDPOINT"; - public static final String REGION = "AWS_REGION"; - public static final String ACCESS_KEY = "AWS_ACCESS_KEY"; - public static final String SECRET_KEY = "AWS_SECRET_KEY"; - public static final String TOKEN = "AWS_TOKEN"; - // required by storage policy - public static final String ROOT_PATH = "AWS_ROOT_PATH"; - public static final String BUCKET = "AWS_BUCKET"; - // optional - public static final String MAX_CONNECTIONS = "AWS_MAX_CONNECTIONS"; - public static final String REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS"; - public static final String CONNECTION_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS"; - public static final String DEFAULT_MAX_CONNECTIONS = "50"; - public static final String DEFAULT_REQUEST_TIMEOUT_MS = "3000"; - public static final String DEFAULT_CONNECTION_TIMEOUT_MS = "1000"; - public static final String NEED_OVERRIDE_ENDPOINT = "AWS_NEED_OVERRIDE_ENDPOINT"; - - public static final String ROLE_ARN = "AWS_ROLE_ARN"; - public static final String EXTERNAL_ID = "AWS_EXTERNAL_ID"; - - public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT); - public static final List FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, TOKEN, - ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS); - } - - public static CloudCredential getCredential(Map props) { - return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); - } - - public static CloudCredentialWithEndpoint getEnvironmentCredentialWithEndpoint(Map props) { - CloudCredential credential = getCloudCredential(props, Env.ACCESS_KEY, Env.SECRET_KEY, - Env.TOKEN); - if (!props.containsKey(Env.ENDPOINT)) { - throw new IllegalArgumentException("Missing 'AWS_ENDPOINT' property. "); - } - String endpoint = props.get(Env.ENDPOINT); - String region = props.getOrDefault(Env.REGION, S3Properties.getRegionOfEndpoint(endpoint)); - props.putIfAbsent(Env.REGION, checkRegion(endpoint, region, Env.REGION)); - return new CloudCredentialWithEndpoint(endpoint, region, credential); - } - - public static String getRegionOfEndpoint(String endpoint) { - if (IPV4_PORT_PATTERN.matcher(endpoint).find()) { - // if endpoint contains '192.168.0.1:8999', return null region - return null; - } - String[] endpointSplit = endpoint.replace("http://", "") - .replace("https://", "") - .split("\\."); - if (endpointSplit.length < 2) { - return null; - } - if (endpointSplit[0].contains("oss-")) { - // compatible with the endpoint: oss-cn-bejing.aliyuncs.com - return endpointSplit[0]; - } - return endpointSplit[1]; - } - - private static void checkProvider(Map properties) throws DdlException { - if (properties.containsKey(PROVIDER)) { - properties.put(PROVIDER, properties.get(PROVIDER).toUpperCase()); - // S3 Provider properties should be case insensitive. - if (!PROVIDERS.stream().anyMatch(s -> s.equals(properties.get(PROVIDER).toUpperCase()))) { - throw new DdlException("Provider must be one of OSS, OBS, AZURE, BOS, COS, S3, GCP"); - } - } - } - - public static void requiredS3Properties(Map properties) throws DdlException { - // Try to convert env properties to uniform properties - // compatible with old version - S3Properties.convertToStdProperties(properties); - if (properties.containsKey(S3Properties.Env.ENDPOINT) - && !properties.containsKey(S3Properties.ENDPOINT)) { - for (String field : S3Properties.Env.REQUIRED_FIELDS) { - checkRequiredProperty(properties, field); - } - } else { - for (String field : S3Properties.REQUIRED_FIELDS) { - checkRequiredProperty(properties, field); - } - } - checkProvider(properties); - } - - public static void requiredS3PingProperties(Map properties) throws DdlException { - requiredS3Properties(properties); - checkRequiredProperty(properties, S3Properties.BUCKET); - } - - public static void checkRequiredProperty(Map properties, String propertyKey) - throws DdlException { - String value = properties.get(propertyKey); - if (Strings.isNullOrEmpty(value)) { - throw new DdlException("Missing [" + propertyKey + "] in properties."); - } - } - - public static void optionalS3Property(Map properties) { - properties.putIfAbsent(S3Properties.MAX_CONNECTIONS, S3Properties.Env.DEFAULT_MAX_CONNECTIONS); - properties.putIfAbsent(S3Properties.REQUEST_TIMEOUT_MS, S3Properties.Env.DEFAULT_REQUEST_TIMEOUT_MS); - properties.putIfAbsent(S3Properties.CONNECTION_TIMEOUT_MS, S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS); - // compatible with old version - properties.putIfAbsent(S3Properties.Env.MAX_CONNECTIONS, S3Properties.Env.DEFAULT_MAX_CONNECTIONS); - properties.putIfAbsent(S3Properties.Env.REQUEST_TIMEOUT_MS, S3Properties.Env.DEFAULT_REQUEST_TIMEOUT_MS); - properties.putIfAbsent(S3Properties.Env.CONNECTION_TIMEOUT_MS, S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS); - } - - public static void convertToStdProperties(Map properties) { - if (properties.containsKey(S3Properties.Env.ENDPOINT)) { - properties.putIfAbsent(S3Properties.ENDPOINT, properties.get(S3Properties.Env.ENDPOINT)); - } - if (properties.containsKey(S3Properties.Env.REGION)) { - properties.putIfAbsent(S3Properties.REGION, properties.get(S3Properties.Env.REGION)); - } - if (properties.containsKey(S3Properties.Env.ACCESS_KEY)) { - properties.putIfAbsent(S3Properties.ACCESS_KEY, properties.get(S3Properties.Env.ACCESS_KEY)); - } - if (properties.containsKey(S3Properties.Env.SECRET_KEY)) { - properties.putIfAbsent(S3Properties.SECRET_KEY, properties.get(S3Properties.Env.SECRET_KEY)); - } - if (properties.containsKey(S3Properties.Env.TOKEN)) { - properties.putIfAbsent(S3Properties.SESSION_TOKEN, properties.get(S3Properties.Env.TOKEN)); - } - if (properties.containsKey(S3Properties.Env.MAX_CONNECTIONS)) { - properties.putIfAbsent(S3Properties.MAX_CONNECTIONS, properties.get(S3Properties.Env.MAX_CONNECTIONS)); - } - if (properties.containsKey(S3Properties.Env.REQUEST_TIMEOUT_MS)) { - properties.putIfAbsent(S3Properties.REQUEST_TIMEOUT_MS, - properties.get(S3Properties.Env.REQUEST_TIMEOUT_MS)); - - } - if (properties.containsKey(S3Properties.Env.CONNECTION_TIMEOUT_MS)) { - properties.putIfAbsent(S3Properties.CONNECTION_TIMEOUT_MS, - properties.get(S3Properties.Env.CONNECTION_TIMEOUT_MS)); - } - if (properties.containsKey(S3Properties.Env.ROOT_PATH)) { - properties.putIfAbsent(S3Properties.ROOT_PATH, properties.get(S3Properties.Env.ROOT_PATH)); - } - if (properties.containsKey(S3Properties.Env.BUCKET)) { - properties.putIfAbsent(S3Properties.BUCKET, properties.get(S3Properties.Env.BUCKET)); - } - if (properties.containsKey(USE_PATH_STYLE)) { - properties.putIfAbsent(USE_PATH_STYLE, properties.get(USE_PATH_STYLE)); - } - - if (properties.containsKey(S3Properties.Env.ROLE_ARN)) { - properties.putIfAbsent(S3Properties.ROLE_ARN, properties.get(S3Properties.Env.ROLE_ARN)); - } - - if (properties.containsKey(S3Properties.Env.EXTERNAL_ID)) { - properties.putIfAbsent(S3Properties.EXTERNAL_ID, properties.get(S3Properties.Env.EXTERNAL_ID)); - } - } - - public static TS3StorageParam getS3TStorageParam(Map properties) { - TS3StorageParam s3Info = new TS3StorageParam(); - - if (properties.containsKey(S3Properties.ROLE_ARN)) { - s3Info.setRoleArn(properties.get(S3Properties.ROLE_ARN)); - if (properties.containsKey(S3Properties.EXTERNAL_ID)) { - s3Info.setExternalId(properties.get(S3Properties.EXTERNAL_ID)); - } - s3Info.setCredProviderType(TCredProviderType.INSTANCE_PROFILE); - } - - s3Info.setEndpoint(properties.get(S3Properties.ENDPOINT)); - s3Info.setRegion(properties.get(S3Properties.REGION)); - s3Info.setAk(properties.get(S3Properties.ACCESS_KEY)); - s3Info.setSk(properties.get(S3Properties.SECRET_KEY)); - s3Info.setToken(properties.get(S3Properties.SESSION_TOKEN)); - - s3Info.setRootPath(properties.get(S3Properties.ROOT_PATH)); - s3Info.setBucket(properties.get(S3Properties.BUCKET)); - String maxConnections = properties.get(S3Properties.MAX_CONNECTIONS); - s3Info.setMaxConn(Integer.parseInt(maxConnections == null - ? S3Properties.Env.DEFAULT_MAX_CONNECTIONS : maxConnections)); - String requestTimeoutMs = properties.get(S3Properties.REQUEST_TIMEOUT_MS); - s3Info.setRequestTimeoutMs(Integer.parseInt(requestTimeoutMs == null - ? S3Properties.Env.DEFAULT_REQUEST_TIMEOUT_MS : requestTimeoutMs)); - String connTimeoutMs = properties.get(S3Properties.CONNECTION_TIMEOUT_MS); - s3Info.setConnTimeoutMs(Integer.parseInt(connTimeoutMs == null - ? S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS : connTimeoutMs)); - String usePathStyle = properties.getOrDefault(USE_PATH_STYLE, "false"); - s3Info.setUsePathStyle(Boolean.parseBoolean(usePathStyle)); - return s3Info; - } - - public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map properties) { - Cloud.ObjectStoreInfoPB.Builder builder = Cloud.ObjectStoreInfoPB.newBuilder(); - if (properties.containsKey(S3Properties.ENDPOINT)) { - builder.setEndpoint(properties.get(S3Properties.ENDPOINT)); - } - if (properties.containsKey(S3Properties.REGION)) { - builder.setRegion(properties.get(S3Properties.REGION)); - } - if (properties.containsKey(S3Properties.ACCESS_KEY)) { - builder.setAk(properties.get(S3Properties.ACCESS_KEY)); - } - if (properties.containsKey(S3Properties.SECRET_KEY)) { - builder.setSk(properties.get(S3Properties.SECRET_KEY)); - } - if (properties.containsKey(S3Properties.ROOT_PATH)) { - Preconditions.checkArgument(!Strings.isNullOrEmpty(properties.get(S3Properties.ROOT_PATH)), - "%s cannot be empty", S3Properties.ROOT_PATH); - builder.setPrefix(properties.get(S3Properties.ROOT_PATH)); - } - if (properties.containsKey(S3Properties.BUCKET)) { - builder.setBucket(properties.get(S3Properties.BUCKET)); - } - if (properties.containsKey(S3Properties.EXTERNAL_ENDPOINT)) { - builder.setExternalEndpoint(properties.get(S3Properties.EXTERNAL_ENDPOINT)); - } - if (properties.containsKey(S3Properties.PROVIDER)) { - // S3 Provider properties should be case insensitive. - builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER).toUpperCase())); - } - - if (properties.containsKey(USE_PATH_STYLE)) { - String value = properties.get(USE_PATH_STYLE); - Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "use_path_style cannot be empty"); - Preconditions.checkArgument(value.equalsIgnoreCase("true") - || value.equalsIgnoreCase("false"), - "Invalid use_path_style value: %s only 'true' or 'false' is acceptable", value); - builder.setUsePathStyle(value.equalsIgnoreCase("true")); - } - - if (properties.containsKey(S3Properties.ROLE_ARN)) { - builder.setRoleArn(properties.get(S3Properties.ROLE_ARN)); - if (properties.containsKey(S3Properties.EXTERNAL_ID)) { - builder.setExternalId(properties.get(S3Properties.EXTERNAL_ID)); - } - builder.setCredProviderType(CredProviderTypePB.INSTANCE_PROFILE); - } - - return builder; - } - - public static String checkRegion(String endpoint, String region, String regionKey) { - if (Strings.isNullOrEmpty(region)) { - region = S3Properties.getRegionOfEndpoint(endpoint); - } - if (Strings.isNullOrEmpty(region)) { - return "us-east-1"; - } - return region; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java index e70026dd760834..581ca235e31b29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java @@ -94,7 +94,6 @@ public class AzureProperties extends StorageProperties { @Getter protected String forceParsingByStandardUrl = "false"; - public AzureProperties(Map origProps) { super(Type.AZURE, origProps); } @@ -109,7 +108,7 @@ public void initNormalizeAndCheckProps() { throw new IllegalArgumentException(String.format("Endpoint '%s' is not valid. It should end with '%s'.", endpoint, AZURE_ENDPOINT_SUFFIX)); } - this.endpoint = formatAzureEndpoint(endpoint); + this.endpoint = formatAzureEndpoint(endpoint, accessKey); } public static boolean guessIsMe(Map origProps) { @@ -144,7 +143,7 @@ public Map getBackendConfigProperties() { public static final String AZURE_ENDPOINT_TEMPLATE = "https://%s.blob.core.windows.net"; - private String formatAzureEndpoint(String endpoint) { + public static String formatAzureEndpoint(String endpoint, String accessKey) { if (Config.force_azure_blob_global_endpoint) { return String.format(AZURE_ENDPOINT_TEMPLATE, accessKey); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 64b3c9086e650c..130240ad53d703 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -17,9 +17,17 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB; +import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider; +import org.apache.doris.common.DdlException; import org.apache.doris.datasource.property.ConnectorPropertiesUtils; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.thrift.TCredProviderType; +import org.apache.doris.thrift.TS3StorageParam; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import lombok.Getter; import lombok.Setter; @@ -36,6 +44,8 @@ import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -44,6 +54,7 @@ import java.util.stream.Stream; public class S3Properties extends AbstractS3CompatibleProperties { + public static final String USE_PATH_STYLE = "use_path_style"; private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { @@ -84,6 +95,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { "aws.glue.secret-key", "client.credentials-provider.glue.secret_key", "iceberg.rest.secret-access-key", "s3.secret-access-key"}, required = false, + sensitive = true, description = "The secret key of S3. Optional for anonymous access to public datasets.") protected String secretKey = ""; @@ -330,5 +342,267 @@ protected String getEndpointFromRegion() { } return "https://s3." + region + ".amazonaws.com"; } + + /** + * =========================================== + * NOTICE: + * This parameter is still used for Cloud-related features, + * although it is no longer recommended. + * + * Reason: + * - Cloud may access S3-compatible object storage via the S3 protocol. + * - The exact behavior has not yet been fully clarified. + * + * Therefore: + * - We cannot directly replace it with the new parameter. + * - This redundant parameter is temporarily kept for compatibility. + * =========================================== + */ + + public static final String S3_PREFIX = "s3."; + + public static final String ENDPOINT = "s3.endpoint"; + public static final String EXTERNAL_ENDPOINT = "s3.external_endpoint"; + public static final String REGION = "s3.region"; + public static final String ACCESS_KEY = "s3.access_key"; + public static final String SECRET_KEY = "s3.secret_key"; + public static final String SESSION_TOKEN = "s3.session_token"; + public static final String MAX_CONNECTIONS = "s3.connection.maximum"; + public static final String REQUEST_TIMEOUT_MS = "s3.connection.request.timeout"; + public static final String CONNECTION_TIMEOUT_MS = "s3.connection.timeout"; + + public static final String ROLE_ARN = "s3.role_arn"; + public static final String EXTERNAL_ID = "s3.external_id"; + public static final String ROOT_PATH = "s3.root.path"; + public static final String BUCKET = "s3.bucket"; + public static final String VALIDITY_CHECK = "s3_validity_check"; + + public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT); + + public static class Env { + public static final String PROPERTIES_PREFIX = "AWS"; + // required + public static final String ENDPOINT = "AWS_ENDPOINT"; + public static final String REGION = "AWS_REGION"; + public static final String ACCESS_KEY = "AWS_ACCESS_KEY"; + public static final String SECRET_KEY = "AWS_SECRET_KEY"; + public static final String TOKEN = "AWS_TOKEN"; + // required by storage policy + public static final String ROOT_PATH = "AWS_ROOT_PATH"; + public static final String BUCKET = "AWS_BUCKET"; + // optional + public static final String MAX_CONNECTIONS = "AWS_MAX_CONNECTIONS"; + public static final String REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS"; + public static final String CONNECTION_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS"; + public static final String DEFAULT_MAX_CONNECTIONS = "50"; + public static final String DEFAULT_REQUEST_TIMEOUT_MS = "3000"; + public static final String DEFAULT_CONNECTION_TIMEOUT_MS = "1000"; + + public static final String ROLE_ARN = "AWS_ROLE_ARN"; + public static final String EXTERNAL_ID = "AWS_EXTERNAL_ID"; + + public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT); + public static final List FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, TOKEN, + ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS); + } + + public static void requiredS3Properties(Map properties) throws DdlException { + // Try to convert env properties to uniform properties + // compatible with old version + convertToStdProperties(properties); + if (properties.containsKey(Env.ENDPOINT) + && !properties.containsKey(ENDPOINT)) { + for (String field : Env.REQUIRED_FIELDS) { + checkRequiredProperty(properties, field); + } + } else { + for (String field : REQUIRED_FIELDS) { + checkRequiredProperty(properties, field); + } + } + if (StringUtils.isNotBlank(properties.get(StorageProperties.FS_PROVIDER_KEY))) { + // S3 Provider properties should be case insensitive. + if (!PROVIDERS.stream().anyMatch(s -> s.equals(properties.get(FS_PROVIDER_KEY).toUpperCase()))) { + throw new DdlException("Provider must be one of OSS, OBS, AZURE, BOS, COS, S3, GCP"); + } + } + + } + + public static final List PROVIDERS = Arrays.asList("COS", "OSS", "S3", "OBS", "BOS", "AZURE", "GCP", "TOS"); + + public static void checkRequiredProperty(Map properties, String propertyKey) + throws DdlException { + String value = properties.get(propertyKey); + if (StringUtils.isBlank(value)) { + throw new DdlException("Missing [" + propertyKey + "] in properties."); + } + } + + public static void requiredS3PingProperties(Map properties) throws DdlException { + requiredS3Properties(properties); + checkRequiredProperty(properties, BUCKET); + } + + public static void convertToStdProperties(Map properties) { + if (properties.containsKey(Env.ENDPOINT)) { + properties.putIfAbsent(ENDPOINT, properties.get(Env.ENDPOINT)); + } + if (properties.containsKey(Env.REGION)) { + properties.putIfAbsent(REGION, properties.get(Env.REGION)); + } + if (properties.containsKey(Env.ACCESS_KEY)) { + properties.putIfAbsent(ACCESS_KEY, properties.get(Env.ACCESS_KEY)); + } + if (properties.containsKey(Env.SECRET_KEY)) { + properties.putIfAbsent(SECRET_KEY, properties.get(Env.SECRET_KEY)); + } + if (properties.containsKey(Env.TOKEN)) { + properties.putIfAbsent(SESSION_TOKEN, properties.get(Env.TOKEN)); + } + if (properties.containsKey(Env.MAX_CONNECTIONS)) { + properties.putIfAbsent(MAX_CONNECTIONS, properties.get(Env.MAX_CONNECTIONS)); + } + if (properties.containsKey(Env.REQUEST_TIMEOUT_MS)) { + properties.putIfAbsent(REQUEST_TIMEOUT_MS, + properties.get(Env.REQUEST_TIMEOUT_MS)); + + } + if (properties.containsKey(Env.CONNECTION_TIMEOUT_MS)) { + properties.putIfAbsent(CONNECTION_TIMEOUT_MS, + properties.get(Env.CONNECTION_TIMEOUT_MS)); + } + if (properties.containsKey(Env.ROOT_PATH)) { + properties.putIfAbsent(ROOT_PATH, properties.get(Env.ROOT_PATH)); + } + if (properties.containsKey(Env.BUCKET)) { + properties.putIfAbsent(BUCKET, properties.get(Env.BUCKET)); + } + if (properties.containsKey(USE_PATH_STYLE)) { + properties.putIfAbsent(USE_PATH_STYLE, properties.get(USE_PATH_STYLE)); + } + + if (properties.containsKey(Env.ROLE_ARN)) { + properties.putIfAbsent(ROLE_ARN, properties.get(Env.ROLE_ARN)); + } + + if (properties.containsKey(Env.EXTERNAL_ID)) { + properties.putIfAbsent(EXTERNAL_ID, properties.get(Env.EXTERNAL_ID)); + } + } + + private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})"); + + public static String getRegionOfEndpoint(String endpoint) { + if (IPV4_PORT_PATTERN.matcher(endpoint).find()) { + // if endpoint contains '192.168.0.1:8999', return null region + return null; + } + String[] endpointSplit = endpoint.replace("http://", "") + .replace("https://", "") + .split("\\."); + if (endpointSplit.length < 2) { + return null; + } + if (endpointSplit[0].contains("oss-")) { + // compatible with the endpoint: oss-cn-bejing.aliyuncs.com + return endpointSplit[0]; + } + return endpointSplit[1]; + } + + public static void optionalS3Property(Map properties) { + properties.putIfAbsent(MAX_CONNECTIONS, Env.DEFAULT_MAX_CONNECTIONS); + properties.putIfAbsent(REQUEST_TIMEOUT_MS, Env.DEFAULT_REQUEST_TIMEOUT_MS); + properties.putIfAbsent(CONNECTION_TIMEOUT_MS, Env.DEFAULT_CONNECTION_TIMEOUT_MS); + // compatible with old version + properties.putIfAbsent(Env.MAX_CONNECTIONS, Env.DEFAULT_MAX_CONNECTIONS); + properties.putIfAbsent(Env.REQUEST_TIMEOUT_MS, Env.DEFAULT_REQUEST_TIMEOUT_MS); + properties.putIfAbsent(Env.CONNECTION_TIMEOUT_MS, Env.DEFAULT_CONNECTION_TIMEOUT_MS); + } + + public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map properties) { + Cloud.ObjectStoreInfoPB.Builder builder = Cloud.ObjectStoreInfoPB.newBuilder(); + if (properties.containsKey(S3Properties.ENDPOINT)) { + builder.setEndpoint(properties.get(S3Properties.ENDPOINT)); + } + if (properties.containsKey(S3Properties.REGION)) { + builder.setRegion(properties.get(S3Properties.REGION)); + } + if (properties.containsKey(S3Properties.ACCESS_KEY)) { + builder.setAk(properties.get(S3Properties.ACCESS_KEY)); + } + if (properties.containsKey(S3Properties.SECRET_KEY)) { + builder.setSk(properties.get(S3Properties.SECRET_KEY)); + } + if (properties.containsKey(S3Properties.ROOT_PATH)) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(properties.get(S3Properties.ROOT_PATH)), + "%s cannot be empty", S3Properties.ROOT_PATH); + builder.setPrefix(properties.get(S3Properties.ROOT_PATH)); + } + if (properties.containsKey(S3Properties.BUCKET)) { + builder.setBucket(properties.get(S3Properties.BUCKET)); + } + if (properties.containsKey(S3Properties.EXTERNAL_ENDPOINT)) { + builder.setExternalEndpoint(properties.get(S3Properties.EXTERNAL_ENDPOINT)); + } + if (properties.containsKey(StorageProperties.FS_PROVIDER_KEY)) { + // S3 Provider properties should be case insensitive. + builder.setProvider(Provider.valueOf(properties.get(StorageProperties.FS_PROVIDER_KEY).toUpperCase())); + } + + if (properties.containsKey(S3Properties.USE_PATH_STYLE)) { + String value = properties.get(S3Properties.USE_PATH_STYLE); + Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "use_path_style cannot be empty"); + Preconditions.checkArgument(value.equalsIgnoreCase("true") + || value.equalsIgnoreCase("false"), + "Invalid use_path_style value: %s only 'true' or 'false' is acceptable", value); + builder.setUsePathStyle(value.equalsIgnoreCase("true")); + } + + if (properties.containsKey(S3Properties.ROLE_ARN)) { + builder.setRoleArn(properties.get(S3Properties.ROLE_ARN)); + if (properties.containsKey(S3Properties.EXTERNAL_ID)) { + builder.setExternalId(properties.get(S3Properties.EXTERNAL_ID)); + } + builder.setCredProviderType(CredProviderTypePB.INSTANCE_PROFILE); + } + + return builder; + } + + public static TS3StorageParam getS3TStorageParam(Map properties) { + TS3StorageParam s3Info = new TS3StorageParam(); + + if (properties.containsKey(S3Properties.ROLE_ARN)) { + s3Info.setRoleArn(properties.get(S3Properties.ROLE_ARN)); + if (properties.containsKey(S3Properties.EXTERNAL_ID)) { + s3Info.setExternalId(properties.get(S3Properties.EXTERNAL_ID)); + } + s3Info.setCredProviderType(TCredProviderType.INSTANCE_PROFILE); + } + + s3Info.setEndpoint(properties.get(S3Properties.ENDPOINT)); + s3Info.setRegion(properties.get(S3Properties.REGION)); + s3Info.setAk(properties.get(S3Properties.ACCESS_KEY)); + s3Info.setSk(properties.get(S3Properties.SECRET_KEY)); + s3Info.setToken(properties.get(S3Properties.SESSION_TOKEN)); + + s3Info.setRootPath(properties.get(S3Properties.ROOT_PATH)); + s3Info.setBucket(properties.get(S3Properties.BUCKET)); + String maxConnections = properties.get(S3Properties.MAX_CONNECTIONS); + s3Info.setMaxConn(Integer.parseInt(maxConnections == null + ? S3Properties.Env.DEFAULT_MAX_CONNECTIONS : maxConnections)); + String requestTimeoutMs = properties.get(S3Properties.REQUEST_TIMEOUT_MS); + s3Info.setRequestTimeoutMs(Integer.parseInt(requestTimeoutMs == null + ? S3Properties.Env.DEFAULT_REQUEST_TIMEOUT_MS : requestTimeoutMs)); + String connTimeoutMs = properties.get(S3Properties.CONNECTION_TIMEOUT_MS); + s3Info.setConnTimeoutMs(Integer.parseInt(connTimeoutMs == null + ? S3Properties.Env.DEFAULT_CONNECTION_TIMEOUT_MS : connTimeoutMs)); + String usePathStyle = properties.getOrDefault(S3Properties.USE_PATH_STYLE, "false"); + s3Info.setUsePathStyle(Boolean.parseBoolean(usePathStyle)); + return s3Info; + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index aaff70917f61cd..f5644650257ba7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -44,7 +44,7 @@ import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.load.BrokerFileGroup; import org.apache.doris.load.BrokerFileGroupAggInfo.FileGroupAggKey; import org.apache.doris.load.EtlJobType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java index 6e21a711e85eca..aab2e43a0a6ab9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java @@ -26,7 +26,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java index 4f1bec5d870567..19f462169ea26f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.HdfsResource; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.Resource.ResourceType; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.thrift.TPushStoragePolicyReq; diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java index b39e2fbfef84b5..be447ffb28c4f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java @@ -24,7 +24,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.meta.MetaContext; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java index 6183360cd92807..c2fd12d83e5852 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java @@ -30,7 +30,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; -import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.SystemInfoService; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/constants/AzurePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/constants/AzurePropertiesTest.java deleted file mode 100644 index e155b5434d8242..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/constants/AzurePropertiesTest.java +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.constants; - -import org.apache.doris.common.Config; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class AzurePropertiesTest { - - @Test - public static void testFormatAzureEndpointGlobal() { - Config.force_azure_blob_global_endpoint = true; - String endpoint = AzureProperties.formatAzureEndpoint("ANY-ENDPOINT", "ak"); - Assertions.assertEquals("https://ak.blob.core.windows.net", endpoint); - } - - @Test - public static void testFormatAzureEndpoint() { - Config.force_azure_blob_global_endpoint = false; - String endpoint = AzureProperties.formatAzureEndpoint("ak.blob.core.chinacloudapi.cn", "ANY-ACCOUNT"); - Assertions.assertEquals("https://ak.blob.core.chinacloudapi.cn", endpoint); - } - - @Test - public static void testFormatAzureEndpointHTTPS() { - Config.force_azure_blob_global_endpoint = false; - String endpoint = AzureProperties.formatAzureEndpoint("https://ak.blob.core.chinacloudapi.cn", "ANY-ACCOUNT"); - Assertions.assertEquals("https://ak.blob.core.chinacloudapi.cn", endpoint); - } - - @Test - public static void testFormatAzureEndpointHTTP() { - Config.force_azure_blob_global_endpoint = false; - String endpoint = AzureProperties.formatAzureEndpoint("http://ak.blob.core.chinacloudapi.cn", "ANY-ACCOUNT"); - Assertions.assertEquals("http://ak.blob.core.chinacloudapi.cn", endpoint); - } - -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java index 3f6ba202ed885b..5e1681adbda6ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/AzureObjStorageTest.java @@ -18,8 +18,8 @@ package org.apache.doris.fs.obj; import org.apache.doris.backup.Status; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.AzureProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.remote.RemoteFile; diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java index 558b14f4d105ae..702070a7fa69ad 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java @@ -21,8 +21,8 @@ import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.remote.RemoteFile; diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageGlobListTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageGlobListTest.java index 1401e892ba38cd..0cff4d4a090657 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageGlobListTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageGlobListTest.java @@ -18,8 +18,8 @@ package org.apache.doris.fs.obj; import org.apache.doris.backup.Status; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.remote.RemoteFile; From 7bfee8b7ccb0ba3b6dcbba24f77826ebddfd56a0 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 26 Sep 2025 17:36:23 +0800 Subject: [PATCH 4/5] fix --- .../main/java/org/apache/doris/analysis/CopyStmt.java | 9 +++++---- .../org/apache/doris/analysis/CreateResourceStmt.java | 1 + .../apache/doris/analysis/CreateStorageVaultStmt.java | 1 + .../java/org/apache/doris/catalog/StorageVaultMgr.java | 6 ++++++ .../doris/datasource/property/PropertyConverter.java | 0 .../doris/datasource/property/storage/OSSProperties.java | 2 +- .../trees/plans/commands/info/BulkStorageDesc.java | 4 ++-- .../nereids/trees/plans/commands/info/CopyIntoInfo.java | 0 .../src/main/java/org/apache/doris/qe/DdlExecutor.java | 3 +-- 9 files changed, 17 insertions(+), 9 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java index c47c66a62e7ff3..e4ab52f1ac299f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java @@ -32,9 +32,10 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.datasource.property.constants.BosProperties; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.datasource.property.storage.S3PropertyUtils; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.load.loadv2.LoadTask.MergeType; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -166,7 +167,7 @@ private void analyze(String user, String db, boolean checkAuth) throws AnalysisE String path; for (int i = 0; i < dataDescription.getFilePaths().size(); i++) { path = dataDescription.getFilePaths().get(i); - dataDescription.getFilePaths().set(i, BosProperties.convertPathToS3(path)); + dataDescription.getFilePaths().set(i, S3PropertyUtils.convertPathToS3(path)); StorageBackend.checkPath(path, brokerDesc.getStorageType(), null); dataDescription.getFilePaths().set(i, path); } @@ -207,7 +208,7 @@ private void analyzeStagePB(StagePB stagePB) throws AnalysisException { brokerProperties.put(S3_BUCKET, objInfo.getBucket()); brokerProperties.put(S3_PREFIX, objInfo.getPrefix()); // S3 Provider properties should be case insensitive. - brokerProperties.put(S3Properties.PROVIDER, objInfo.getProvider().toString().toUpperCase()); + brokerProperties.put(StorageProperties.FS_PROVIDER_KEY, objInfo.getProvider().toString().toUpperCase()); StageProperties stageProperties = new StageProperties(stagePB.getPropertiesMap()); this.copyIntoProperties.mergeProperties(stageProperties); this.copyIntoProperties.analyze(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java index e44a56149e6838..70ec13fd9d16bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java @@ -25,6 +25,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.PrintableMap; import org.apache.doris.datasource.property.storage.AzureProperties; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java index 65dabf9c896f00..eb00cc5a4fc17c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java @@ -27,6 +27,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.PrintableMap; import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index faa8b8b5827d33..381ae085f16fee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -29,6 +29,7 @@ import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; import org.apache.doris.rpc.BackendServiceProxy; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.Backend; @@ -240,6 +241,11 @@ public void alterStorageVault(StorageVaultType type, Map propert } } + @VisibleForTesting + public void setDefaultStorageVault(SetDefaultStorageVaultStmt stmt) throws DdlException { + setDefaultStorageVault(stmt.getStorageVaultName()); + } + public void setDefaultStorageVault(String vaultName) throws DdlException { Cloud.AlterObjStoreInfoRequest.Builder builder = Cloud.AlterObjStoreInfoRequest.newBuilder(); Cloud.StorageVaultPB.Builder vaultBuilder = Cloud.StorageVaultPB.newBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 3ddb26e696927b..4e67f9800e8644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -56,7 +56,7 @@ public class OSSProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"oss.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY", - "dlf.secret_key", "dlf.catalog.secret_key","fs.oss.accessKeySecret"}, + "dlf.secret_key", "dlf.catalog.secret_key", "fs.oss.accessKeySecret"}, sensitive = true, required = false, description = "The secret key of OSS.") diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BulkStorageDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BulkStorageDesc.java index 38543c069df818..c530e2b804fd03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BulkStorageDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BulkStorageDesc.java @@ -20,7 +20,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.S3ClientBEProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Maps; @@ -81,7 +81,7 @@ public BulkStorageDesc(String name, StorageType type, Map proper this.properties = Maps.newHashMap(); } this.storageType = type; - this.properties.putAll(S3ClientBEProperties.getBeFSProperties(this.properties)); + this.properties.putAll(StorageProperties.createPrimary(this.properties).getBackendConfigProperties()); } public StorageType getStorageType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index e4737e81970bc5..e3ff4db33c9df6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -431,8 +431,7 @@ public static void execute(Env env, DdlStmt ddlStmt, boolean isProxy) throws Exc Env.getCurrentEnv().getAnalysisManager().dropAnalyzeJob(analyzeJobStmt); } else if (ddlStmt instanceof AlterRepositoryStmt) { AlterRepositoryStmt alterRepositoryStmt = (AlterRepositoryStmt) ddlStmt; - env.getBackupHandler().alterRepository(alterRepositoryStmt.getName(), alterRepositoryStmt.getProperties(), - false); + env.getBackupHandler().alterRepository(alterRepositoryStmt.getName(), alterRepositoryStmt.getProperties()); } else if (ddlStmt instanceof CreateStorageVaultStmt) { env.getStorageVaultMgr().createStorageVaultResource((CreateStorageVaultStmt) ddlStmt); } else if (ddlStmt instanceof CreateStageStmt) { From 0b0ed3771f579ca4f4aed934ff6029bc6c6eea75 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 26 Sep 2025 18:00:53 +0800 Subject: [PATCH 5/5] fix --- .idea/vcs.xml | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 11a42457d994f7..0a185d9bdc7c46 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,16 +1,34 @@ + - - - - - - - \ No newline at end of file + + + + + + + + +