branch-3.1: [opt](catalog) remove old properties (#55625 #56163)

fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java

@@ -23,7 +23,6 @@
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.qe.ConnectContext;
 
@@ -64,16 +63,6 @@ public void analyze(Analyzer analyzer) throws UserException {
         if (copyProperties.size() == 0) {
             throw new UserException("alter repository need contains ak/sk/token info of s3.");
         }
-        copyProperties.remove(S3Properties.ACCESS_KEY);
-        copyProperties.remove(S3Properties.SECRET_KEY);
-        copyProperties.remove(S3Properties.SESSION_TOKEN);
-        copyProperties.remove(S3Properties.Env.ACCESS_KEY);
-        copyProperties.remove(S3Properties.Env.SECRET_KEY);
-        copyProperties.remove(S3Properties.Env.TOKEN);
-        if (copyProperties.size() != 0) {
-            throw new UserException("alter repository only support ak/sk/token info of s3."
-                + " unsupported properties: " + copyProperties.keySet());
-        }
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java

@@ -32,9 +32,10 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.DebugUtil;
-import org.apache.doris.datasource.property.constants.BosProperties;
-import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.datasource.property.fileformat.FileFormatProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
+import org.apache.doris.datasource.property.storage.S3PropertyUtils;
+import org.apache.doris.datasource.property.storage.StorageProperties;
 import org.apache.doris.load.loadv2.LoadTask.MergeType;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.SessionVariable;
@@ -166,7 +167,7 @@ private void analyze(String user, String db, boolean checkAuth) throws AnalysisE
         String path;
         for (int i = 0; i < dataDescription.getFilePaths().size(); i++) {
             path = dataDescription.getFilePaths().get(i);
-            dataDescription.getFilePaths().set(i, BosProperties.convertPathToS3(path));
+            dataDescription.getFilePaths().set(i, S3PropertyUtils.convertPathToS3(path));
             StorageBackend.checkPath(path, brokerDesc.getStorageType(), null);
             dataDescription.getFilePaths().set(i, path);
         }
@@ -207,7 +208,7 @@ private void analyzeStagePB(StagePB stagePB) throws AnalysisException {
         brokerProperties.put(S3_BUCKET, objInfo.getBucket());
         brokerProperties.put(S3_PREFIX, objInfo.getPrefix());
         // S3 Provider properties should be case insensitive.
-        brokerProperties.put(S3Properties.PROVIDER, objInfo.getProvider().toString().toUpperCase());
+        brokerProperties.put(StorageProperties.FS_PROVIDER_KEY, objInfo.getProvider().toString().toUpperCase());
         StageProperties stageProperties = new StageProperties(stagePB.getPropertiesMap());
         this.copyIntoProperties.mergeProperties(stageProperties);
         this.copyIntoProperties.analyze();

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java

@@ -26,7 +26,7 @@
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.property.constants.AzureProperties;
+import org.apache.doris.datasource.property.storage.AzureProperties;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.qe.ConnectContext;
 
@@ -82,7 +82,7 @@ public void analyzeResourceType() throws UserException {
             throw new AnalysisException("Resource type can't be null");
         }
 
-        if (AzureProperties.checkAzureProviderPropertyExist(properties)) {
+        if (AzureProperties.guessIsMe(properties)) {
             resourceType = ResourceType.AZURE;
             return;
         }

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java

@@ -28,7 +28,7 @@
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.property.PropertyConverter;
+import org.apache.doris.datasource.property.storage.S3Properties;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.qe.ConnectContext;
 
@@ -148,11 +148,11 @@ public void analyze(Analyzer analyzer) throws UserException {
         setStorageVaultType(StorageVault.StorageVaultType.fromString(type));
 
         if (vaultType == StorageVault.StorageVaultType.S3
-                && !properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {
+                && !properties.containsKey(S3Properties.USE_PATH_STYLE)) {
             properties = ImmutableMap.<String, String>builder()
-                .putAll(properties)
-                .put(PropertyConverter.USE_PATH_STYLE, "true")
-                .build();
+                    .putAll(properties)
+                    .put(S3Properties.USE_PATH_STYLE, "true")
+                    .build();
         }
     }
 

fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java

@@ -52,9 +52,7 @@
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.property.storage.StorageProperties;
 import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.remote.AzureFileSystem;
 import org.apache.doris.fs.remote.RemoteFileSystem;
-import org.apache.doris.fs.remote.S3FileSystem;
 import org.apache.doris.persist.BarrierLog;
 import org.apache.doris.task.DirMoveTask;
 import org.apache.doris.task.DownloadTask;
@@ -237,12 +235,10 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException {
      *
      * @param repoName    The name of the repository to alter.
      * @param newProps    The new properties to apply to the repository.
-     * @param strictCheck If true, only allows altering S3 or Azure repositories and validates properties accordingly.
-     *                    TODO: Investigate why only S3 and Azure repositories are supported for alter operation
      * @throws DdlException if the repository does not exist, fails to apply properties, or cannot connect
      * to the updated repository.
      */
-    public void alterRepository(String repoName, Map<String, String> newProps, boolean strictCheck)
+    public void alterRepository(String repoName, Map<String, String> newProps)
             throws DdlException {
         tryLock();
         try {
@@ -251,7 +247,7 @@ public void alterRepository(String repoName, Map<String, String> newProps, boole
                 throw new DdlException("Repository does not exist");
             }
             // Merge new properties with the existing repository's properties
-            Map<String, String> mergedProps = mergeProperties(oldRepo, newProps, strictCheck);
+            Map<String, String> mergedProps = mergeProperties(oldRepo, newProps);
             // Create new remote file system with merged properties
             RemoteFileSystem fileSystem = FileSystemFactory.get(StorageProperties.createPrimary(mergedProps));
             // Create new Repository instance with updated file system
@@ -278,34 +274,16 @@ public void alterRepository(String repoName, Map<String, String> newProps, boole
 
     /**
      * Merges new user-provided properties into the existing repository's configuration.
-     * In strict mode, only supports S3 or Azure repositories and applies internal S3 merge logic.
      *
      * @param repo        The existing repository.
      * @param newProps    New user-specified properties.
-     * @param strictCheck Whether to enforce S3/Azure-only and validate the new properties.
      * @return A complete set of merged properties.
-     * @throws DdlException if the merge fails or the repository type is unsupported.
      */
-    private Map<String, String> mergeProperties(Repository repo, Map<String, String> newProps, boolean strictCheck)
-            throws DdlException {
-        if (strictCheck) {
-            if (!(repo.getRemoteFileSystem() instanceof S3FileSystem
-                    || repo.getRemoteFileSystem() instanceof AzureFileSystem)) {
-                throw new DdlException("Only support altering S3 or Azure repository");
-            }
-            // Let the repository validate and enrich the new S3/Azure properties
-            Map<String, String> propsCopy = new HashMap<>(newProps);
-            Status status = repo.alterRepositoryS3Properties(propsCopy);
-            if (!status.ok()) {
-                throw new DdlException("Failed to merge S3 properties: " + status.getErrMsg());
-            }
-            return propsCopy;
-        } else {
-            // General case: just override old props with new ones
-            Map<String, String> combined = new HashMap<>(repo.getRemoteFileSystem().getProperties());
-            combined.putAll(newProps);
-            return combined;
-        }
+    private Map<String, String> mergeProperties(Repository repo, Map<String, String> newProps) {
+        // General case: just override old props with new ones
+        Map<String, String> combined = new HashMap<>(repo.getRemoteFileSystem().getProperties());
+        combined.putAll(newProps);
+        return combined;
     }
 
     /**

fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java

@@ -30,7 +30,6 @@
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.util.PrintableMap;
 import org.apache.doris.common.util.TimeUtils;
-import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.datasource.property.storage.BrokerProperties;
 import org.apache.doris.datasource.property.storage.StorageProperties;
 import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
@@ -39,7 +38,6 @@
 import org.apache.doris.fs.remote.BrokerFileSystem;
 import org.apache.doris.fs.remote.RemoteFile;
 import org.apache.doris.fs.remote.RemoteFileSystem;
-import org.apache.doris.fs.remote.S3FileSystem;
 import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.system.Backend;
@@ -69,7 +67,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.UUID;
 
 /*
@@ -209,38 +206,6 @@ public static Repository read(DataInput in) throws IOException {
         }
     }
 
-    //todo why only support alter S3 properties
-    public Status alterRepositoryS3Properties(Map<String, String> properties) {
-        if (this.fileSystem instanceof S3FileSystem) {
-            Map<String, String> oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties());
-            oldProperties.remove(S3Properties.ACCESS_KEY);
-            oldProperties.remove(S3Properties.SECRET_KEY);
-            oldProperties.remove(S3Properties.SESSION_TOKEN);
-            oldProperties.remove(S3Properties.Env.ACCESS_KEY);
-            oldProperties.remove(S3Properties.Env.SECRET_KEY);
-            oldProperties.remove(S3Properties.Env.TOKEN);
-            for (Map.Entry<String, String> entry : properties.entrySet()) {
-                if (Objects.equals(entry.getKey(), S3Properties.ACCESS_KEY)
-                        || Objects.equals(entry.getKey(), S3Properties.Env.ACCESS_KEY)) {
-                    oldProperties.putIfAbsent(S3Properties.ACCESS_KEY, entry.getValue());
-                }
-                if (Objects.equals(entry.getKey(), S3Properties.SECRET_KEY)
-                        || Objects.equals(entry.getKey(), S3Properties.Env.SECRET_KEY)) {
-                    oldProperties.putIfAbsent(S3Properties.SECRET_KEY, entry.getValue());
-                }
-                if (Objects.equals(entry.getKey(), S3Properties.SESSION_TOKEN)
-                        || Objects.equals(entry.getKey(), S3Properties.Env.TOKEN)) {
-                    oldProperties.putIfAbsent(S3Properties.SESSION_TOKEN, entry.getValue());
-                }
-            }
-            properties.clear();
-            properties.putAll(oldProperties);
-            return Status.OK;
-        } else {
-            return new Status(ErrCode.COMMON_ERROR, "Only support alter s3 repository");
-        }
-    }
-
     @Override
     public void gsonPostProcess() {
         try {

fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java

@@ -21,8 +21,8 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.proc.BaseProcResult;
 import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.datasource.property.storage.AzureProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
 import org.apache.doris.datasource.property.storage.StorageProperties;
 import org.apache.doris.fs.obj.AzureObjStorage;
 import org.apache.doris.fs.obj.ObjStorage;

fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java

@@ -19,7 +19,7 @@
 
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.proc.BaseProcResult;
-import org.apache.doris.datasource.property.constants.HMSProperties;
+import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -63,10 +63,8 @@ public void modifyProperties(Map<String, String> properties) throws DdlException
 
     @Override
     protected void setProperties(ImmutableMap<String, String> properties) throws DdlException {
-        for (String field : HMSProperties.REQUIRED_FIELDS) {
-            if (!properties.containsKey(field)) {
-                throw new DdlException("Missing [" + field + "] in properties.");
-            }
+        if (!properties.containsKey(HMSBaseProperties.HIVE_METASTORE_URIS)) {
+            throw new DdlException("Missing [" + HMSBaseProperties.HIVE_METASTORE_URIS + "] in properties.");
         }
         this.properties.putAll(properties);
     }

fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java

@@ -22,7 +22,6 @@
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.security.authentication.AuthenticationConfig;
 import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties;
 import org.apache.doris.datasource.property.storage.StorageProperties;
 import org.apache.doris.fs.remote.dfs.DFSFileSystem;
@@ -59,7 +58,7 @@
  */
 public class HdfsStorageVault extends StorageVault {
     private static final Logger LOG = LogManager.getLogger(HdfsStorageVault.class);
-
+    public static final String S3_VALIDITY_CHECK = "s3_validity_check";
     public static final String HADOOP_FS_PREFIX = "dfs.";
     public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit";
     public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path";
@@ -87,7 +86,7 @@ public static class PropertyKey {
      * such as `type`, `path_prefix`, etc.
      */
     private static final Set<String> NON_HDFS_CONF_PROPERTY_KEYS =
-            ImmutableSet.of(StorageVault.PropertyKey.TYPE, PropertyKey.VAULT_PATH_PREFIX, S3Properties.VALIDITY_CHECK)
+            ImmutableSet.of(StorageVault.PropertyKey.TYPE, PropertyKey.VAULT_PATH_PREFIX, S3_VALIDITY_CHECK)
                     .stream().map(String::toLowerCase)
                     .collect(ImmutableSet.toImmutableSet());
 
@@ -113,8 +112,8 @@ public Map<String, String> getCopiedProperties() {
     }
 
     public static void checkConnectivity(Map<String, String> newProperties) throws DdlException {
-        if (newProperties.containsKey(S3Properties.VALIDITY_CHECK)
-                && newProperties.get(S3Properties.VALIDITY_CHECK).equalsIgnoreCase("false")) {
+        if (newProperties.containsKey(S3_VALIDITY_CHECK)
+                && newProperties.get(S3_VALIDITY_CHECK).equalsIgnoreCase("false")) {
             return;
         }
 

fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java

@@ -21,8 +21,8 @@
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.security.authentication.AuthType;
 import org.apache.doris.common.security.authentication.AuthenticationConfig;
-import org.apache.doris.datasource.property.constants.HMSProperties;
-import org.apache.doris.datasource.property.constants.S3Properties;
+import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
 import org.apache.doris.thrift.THiveTable;
 import org.apache.doris.thrift.TTableDescriptor;
 import org.apache.doris.thrift.TTableType;
@@ -49,6 +49,8 @@ public class HiveTable extends Table {
     private static final String PROPERTY_ERROR_MSG = "Hive table properties('%s'='%s')"
             + " is illegal or not supported. Please check it";
 
+    public static final String AWS_PROPERTIES_PREFIX = "AWS";
+
     @SerializedName("hdb")
     private String hiveDb;
     @SerializedName("ht")
@@ -105,18 +107,19 @@ private void validate(Map<String, String> properties) throws DdlException {
 
         // check hive properties
         // hive.metastore.uris
-        String hiveMetaStoreUris = copiedProps.get(HMSProperties.HIVE_METASTORE_URIS);
+        String hiveMetaStoreUris = copiedProps.get(HMSBaseProperties.HIVE_METASTORE_URIS);
         if (Strings.isNullOrEmpty(hiveMetaStoreUris)) {
             throw new DdlException(String.format(
-                    PROPERTY_MISSING_MSG, HMSProperties.HIVE_METASTORE_URIS, HMSProperties.HIVE_METASTORE_URIS));
+                    PROPERTY_MISSING_MSG, HMSBaseProperties.HIVE_METASTORE_URIS,
+                    HMSBaseProperties.HIVE_METASTORE_URIS));
         }
-        copiedProps.remove(HMSProperties.HIVE_METASTORE_URIS);
-        hiveProperties.put(HMSProperties.HIVE_METASTORE_URIS, hiveMetaStoreUris);
+        copiedProps.remove(HMSBaseProperties.HIVE_METASTORE_URIS);
+        hiveProperties.put(HMSBaseProperties.HIVE_METASTORE_URIS, hiveMetaStoreUris);
         // support multi hive version
-        String hiveVersion = copiedProps.get(HMSProperties.HIVE_VERSION);
+        String hiveVersion = copiedProps.get(HMSBaseProperties.HIVE_VERSION);
         if (!Strings.isNullOrEmpty(hiveVersion)) {
-            copiedProps.remove(HMSProperties.HIVE_VERSION);
-            hiveProperties.put(HMSProperties.HIVE_VERSION, hiveVersion);
+            copiedProps.remove(HMSBaseProperties.HIVE_VERSION);
+            hiveProperties.put(HMSBaseProperties.HIVE_VERSION, hiveVersion);
         }
 
         // check auth type
@@ -163,7 +166,7 @@ private void validate(Map<String, String> properties) throws DdlException {
                 String key = entry.getKey();
                 if (key.startsWith(HdfsResource.HADOOP_FS_PREFIX)
                         || key.startsWith(S3Properties.S3_PREFIX)
-                        || key.startsWith(S3Properties.Env.PROPERTIES_PREFIX)) {
+                        || key.startsWith(AWS_PROPERTIES_PREFIX)) {
                     hiveProperties.put(key, entry.getValue());
                     iter.remove();
                 }

fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java

@@ -22,8 +22,8 @@
 import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
 import org.apache.doris.common.proc.BaseProcResult;
 import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
+import org.apache.doris.datasource.property.storage.S3Properties;
 import org.apache.doris.datasource.property.storage.StorageProperties;
 import org.apache.doris.fs.obj.ObjStorage;
 import org.apache.doris.fs.obj.RemoteObjects;