[fix](docker)(case) Fix kerberized-HDFS p2 suites: SASL client property, Paimon fixture, iceberg root-cause reporting

docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh

@@ -136,9 +136,35 @@ kdestroy
 
 report_stage "initialize-hive-metastore"
 schematool -dbType derby -initSchema
+# The Paimon table declares no columns, so the metastore itself resolves its
+# schema through the storage handler. The handler class must therefore be on the
+# metastore service's own classpath, not merely on the DDL client's.
+if [[ -d /opt/doris/auxlib ]]; then
+    export HIVE_AUX_JARS_PATH=/opt/doris/auxlib
+fi
 start_service hive --service metastore -p "${HMS_PORT}"
 wait_for_port "${HOST}" "${HMS_PORT}" "Hive Metastore"
 
+# Register the Paimon fixture consumed by test_paimon_hms_catalog. This runs
+# before the readiness marker below on purpose: run-thirdparties-docker.sh
+# releases the pipeline on DORIS_KERBEROS_READY, so anything published later
+# would race the suites. A failure here aborts the container (set -e) instead of
+# handing out an environment that is silently missing hdfs_db.
+#
+# Both kerberos containers run this entrypoint with the same env switch, but the
+# fixture and its mounts (sql/, paimon_data/, auxlib/) belong to kerberos1 only -
+# its metastore (9583) is the one the suite talks to. Gate on the container role,
+# not on the mounts, so a broken mount on kerberos1 still fails loudly.
+if [[ "${enablePaimonHms:-false}" == "true" && "${HOST:-}" == "hadoop-master" ]]; then
+    report_stage "load-paimon-hms"
+    export KRB5CCNAME=FILE:/tmp/hive-admin.ccache
+    kinit -kt /data/keytabs/hive.keytab "${HIVE_PRINCIPAL}"
+    hdfs dfs -mkdir -p /user/hive/warehouse
+    hdfs dfs -put -f /opt/doris/paimon_data/* /user/hive/warehouse/
+    hive -f /opt/doris/sql/create_paimon_hive_table.hql
+    kdestroy
+fi
+
 touch /tmp/SUCCESS
 echo "Minimal Kerberos HDFS and Hive Metastore environment is ready"
 echo "DORIS_KERBEROS_READY"

docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl

@@ -29,3 +29,7 @@ PRESTO_CLIENT_KEYTAB=${PRESTO_CLIENT_KEYTAB}
 HADOOP_CONF_DIR=/opt/doris/conf
 HIVE_CONF_DIR=/opt/doris/conf
 KRB5_CONFIG=/etc/krb5.conf
+# Paimon-on-kerberized-HMS fixture switch. Read from hive-3x_settings.env by
+# start_kerberos() rather than duplicated into kerberos*_settings.env, so that
+# the pipeline keeps patching exactly one file and the two stacks cannot drift.
+enablePaimonHms=${enablePaimonHms}

docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl

@@ -33,11 +33,17 @@ services:
     <<: *kerberos-service
     container_name: doris-${CONTAINER_UID}-kerberos1
     hostname: hadoop-master
+    # auxlib/sql/paimon_data are mounted on kerberos1 only: test_paimon_hms_catalog
+    # is the sole consumer and it talks to this metastore (9583). Leaving kerberos2
+    # untouched keeps the second container as light as it was.
     volumes:
       - ./conf/kerberos1:/opt/doris/conf:ro
       - ./conf/kerberos1/krb5.conf:/etc/krb5.conf:ro
       - ./data/kerberos1:/data
       - ./two-kerberos-hives:/keytabs
+      - ./auxlib:/opt/doris/auxlib:ro
+      - ./sql:/opt/doris/sql:ro
+      - ./paimon_data:/opt/doris/paimon_data:ro
     env_file:
       - ./hadoop-hive-1.env
 

docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql

@@ -0,0 +1,22 @@
+-- Paimon table registered in the kerberized Hive Metastore, consumed by the
+-- hdfs_kerberos / hdfs_new_kerberos cases of
+-- external_table_p2/paimon/test_paimon_hms_catalog.groovy.
+--
+-- hdfs_db is backed by ../paimon_data, which the entrypoint uploads into HDFS.
+-- (branch-4.0 additionally registers ali_db over OSS here; master has no case
+-- that reads it through this metastore, so it is deliberately not provisioned.)
+--
+-- The table declares no columns, so the metastore derives them through the
+-- Paimon storage handler (metastore.storage.schema.reader.impl is
+-- SerDeStorageSchemaReader) -- the Paimon jar has to be on the metastore's own
+-- classpath, not just on the classpath of whoever runs this script.
+
+CREATE DATABASE IF NOT EXISTS hdfs_db;
+
+USE hdfs_db;
+
+DROP TABLE IF EXISTS external_test_table;
+
+CREATE EXTERNAL TABLE external_test_table
+    STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
+LOCATION 'hdfs:///user/hive/warehouse/hdfs_db.db/external_test_table';

docker/thirdparties/run-thirdparties-docker.sh

@@ -1595,6 +1595,48 @@ validate_kerberos_container() {
     fi
 }
 
+# Aux jar for the Paimon fixture in the kerberized metastore: the Paimon storage
+# handler the metastore loads to derive the fixture table's schema. Same source
+# and same skip-if-present caching the Hive3 stack uses in prepare-hive-data.sh.
+# (branch-4.0 also stages the jindo/aliyun jars for its ali_db-over-OSS case;
+# master has no such case in this metastore.)
+download_kerberos_paimon_jars() {
+    local auxlib_dir="$1"
+    local url_prefix="https://${s3BucketName}.${s3Endpoint}/regression/docker/hive3"
+    local jars=(
+        paimon-hive-connector-3.1-1.3-SNAPSHOT.jar
+    )
+    local jar
+
+    for jar in "${jars[@]}"; do
+        if [[ -f "${auxlib_dir}/${jar}" ]]; then
+            echo "Reuse cached kerberos aux jar ${jar}"
+            continue
+        fi
+        echo "Download kerberos aux jar ${jar}"
+        curl -sSfL -o "${auxlib_dir}/${jar}" "${url_prefix}/${jar}"
+    done
+}
+
+# enablePaimonHms lives in hive-3x_settings.env, which the pipeline already
+# patches with the real value. Read it from there instead of duplicating the
+# switch into kerberos*_settings.env, where the two copies would drift and one
+# stack would silently lose the fixture. Sourced in a subshell so that hive3's
+# FS_PORT / HMS_PORT cannot leak into the kerberos settings sourced a few lines
+# below.
+resolve_kerberos_paimon_env() {
+    local hive3_settings="${ROOT}/docker-compose/hive/hive-3x_settings.env"
+
+    enablePaimonHms="false"
+    if [[ -f "${hive3_settings}" ]]; then
+        read -r enablePaimonHms < <(
+            . "${hive3_settings}" >/dev/null 2>&1
+            printf '%s\n' "${enablePaimonHms:-false}"
+        )
+    fi
+    export enablePaimonHms
+}
+
 start_kerberos() {
     echo "RUN_KERBEROS"
     local KERBEROS_DIR="${ROOT}/docker-compose/kerberos"
@@ -1609,8 +1651,13 @@ start_kerberos() {
 
     export CONTAINER_UID=${CONTAINER_UID}
     envsubst <"${KERBEROS_DIR}/kerberos.yaml.tpl" >"${KERBEROS_DIR}/kerberos.yaml"
+    # auxlib must exist even when the fixture is off: kerberos1 mounts it read-only.
     mkdir -p "${KERBEROS_DIR}/conf/kerberos1" "${KERBEROS_DIR}/conf/kerberos2" \
-        "${KERBEROS_DIR}/two-kerberos-hives"
+        "${KERBEROS_DIR}/two-kerberos-hives" "${KERBEROS_DIR}/auxlib"
+    resolve_kerberos_paimon_env
+    if [[ "${enablePaimonHms}" == "true" && "${STOP}" -ne 1 ]]; then
+        download_kerberos_paimon_jars "${KERBEROS_DIR}/auxlib"
+    fi
     for i in {1..2}; do
         . "${KERBEROS_DIR}/kerberos${i}_settings.env"
         envsubst <"${KERBEROS_DIR}/hadoop-hive.env.tpl" >"${KERBEROS_DIR}/hadoop-hive-${i}.env"

fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java

@@ -52,6 +52,7 @@
 import org.apache.doris.thrift.TTableDescriptor;
 import org.apache.doris.thrift.TTableType;
 
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.iceberg.BaseTable;
 import org.apache.iceberg.MetadataTableType;
 import org.apache.iceberg.MetadataTableUtils;
@@ -965,7 +966,8 @@ public void createTable(ConnectorSession session, ConnectorCreateTableRequest re
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to create Iceberg table "
-                    + request.getDbName() + "." + request.getTableName() + ": " + e.getMessage(), e);
+                    + request.getDbName() + "." + request.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1158,7 +1160,8 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle handle,
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to add column " + column.getName() + " to Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1177,7 +1180,8 @@ public void addColumns(ConnectorSession session, ConnectorTableHandle handle, Li
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to add columns to Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1192,7 +1196,8 @@ public void dropColumn(ConnectorSession session, ConnectorTableHandle handle, St
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to drop column " + columnName + " from Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1209,7 +1214,7 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle handle,
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to rename column " + oldName + " to " + newName
                     + " in Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1255,7 +1260,7 @@ public void modifyColumn(ConnectorSession session, ConnectorTableHandle handle,
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to modify column " + column.getName()
                     + " in Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1299,7 +1304,8 @@ public void reorderColumns(ConnectorSession session, ConnectorTableHandle handle
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to reorder columns in Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1345,7 +1351,7 @@ public void addNestedColumn(ConnectorSession session, ConnectorTableHandle handl
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to add nested column " + path.getFullPath()
                     + " to Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1367,7 +1373,7 @@ public void dropNestedColumn(ConnectorSession session, ConnectorTableHandle hand
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to drop nested column " + path.getFullPath()
                     + " from Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1391,7 +1397,7 @@ public void renameNestedColumn(ConnectorSession session, ConnectorTableHandle ha
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to rename nested column " + path.getFullPath()
                     + " to " + newName + " in Iceberg table " + iceHandle.getDbName() + "."
-                    + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getTableName() + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1440,7 +1446,7 @@ public void modifyNestedColumn(ConnectorSession session, ConnectorTableHandle ha
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to modify nested column " + path.getFullPath()
                     + " in Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1462,7 +1468,7 @@ public void modifyColumnComment(ConnectorSession session, ConnectorTableHandle h
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to modify comment for column " + path.getFullPath()
                     + " in Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1485,7 +1491,7 @@ public void createOrReplaceBranch(ConnectorSession session, ConnectorTableHandle
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to create or replace branch " + branch.getName()
                     + " on Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1501,7 +1507,7 @@ public void createOrReplaceTag(ConnectorSession session, ConnectorTableHandle ha
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to create or replace tag " + tag.getName()
                     + " on Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1517,7 +1523,7 @@ public void dropBranch(ConnectorSession session, ConnectorTableHandle handle, Dr
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to drop branch " + branch.getName()
                     + " from Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1533,7 +1539,7 @@ public void dropTag(ConnectorSession session, ConnectorTableHandle handle, DropR
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to drop tag " + tag.getName()
                     + " from Iceberg table " + iceHandle.getDbName() + "." + iceHandle.getTableName()
-                    + ": " + e.getMessage(), e);
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1556,7 +1562,8 @@ public void addPartitionField(ConnectorSession session, ConnectorTableHandle han
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to add partition field to Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1572,7 +1579,8 @@ public void dropPartitionField(ConnectorSession session, ConnectorTableHandle ha
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to drop partition field from Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 
@@ -1588,7 +1596,8 @@ public void replacePartitionField(ConnectorSession session, ConnectorTableHandle
             });
         } catch (Exception e) {
             throw new DorisConnectorException("Failed to replace partition field in Iceberg table "
-                    + iceHandle.getDbName() + "." + iceHandle.getTableName() + ": " + e.getMessage(), e);
+                    + iceHandle.getDbName() + "." + iceHandle.getTableName()
+                    + ": " + ExceptionUtils.getRootCauseMessage(e), e);
         }
     }
 

regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy

@@ -77,6 +77,7 @@ suite("refactor_params_hdfs_all_test", "p0,external") {
     def hdfsNonXmlParams = "\"fs.defaultFS\" = \"hdfs://${externalEnvIp}:8520\",\n" +
             "\"dfs.namenode.kerberos.principal\" = \"hdfs/hadoop-master@LABS.TERADATA.COM\",\n" +
             "\"dfs.client.use.datanode.hostname\" = \"true\",\n" +
+            "\"dfs.data.transfer.protection\" = \"authentication\",\n" +
             "\"hadoop.security.token.service.use_ip\" = \"false\",\n" +
             "\"hadoop.kerberos.min.seconds.before.relogin\" = \"5\",\n" +
             "\"hadoop.security.authentication\" = \"kerberos\",\n" +

regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy

@@ -124,6 +124,7 @@ suite("test_paimon_hms_catalog", "p2,external") {
                 "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520",
                 "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM",
                 "dfs.client.use.datanode.hostname" = "true",
+                "dfs.data.transfer.protection" = "authentication",
                 "hadoop.security.token.service.use_ip" = "false",
                 "hadoop.security.authentication" = "kerberos",           
                 "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM",
@@ -134,6 +135,7 @@ suite("test_paimon_hms_catalog", "p2,external") {
                 "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520",
                 "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM",
                 "dfs.client.use.datanode.hostname" = "true",
+                "dfs.data.transfer.protection" = "authentication",
                 "hadoop.security.token.service.use_ip" = "false",
                 "hdfs.authentication.type" = "kerberos",
                 "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM",

regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy

@@ -483,6 +483,7 @@ suite("hive_on_hms_and_dlf", "p2,external") {
                 "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
                 "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM",
                 "dfs.client.use.datanode.hostname" = "true",
+                "dfs.data.transfer.protection" = "authentication",
                 "hadoop.security.token.service.use_ip" = "false",
                 "hadoop.security.authentication" = "kerberos",
 
@@ -493,6 +494,7 @@ suite("hive_on_hms_and_dlf", "p2,external") {
                 "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
                 "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM",
                 "dfs.client.use.datanode.hostname" = "true",
+                "dfs.data.transfer.protection" = "authentication",
                 "hadoop.security.token.service.use_ip" = "false",
                 "hdfs.authentication.type" = "kerberos",
                 "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM",