Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/ast/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1693,6 +1693,8 @@ pub enum Statement {
partitions: Option<Vec<Expr>>,
/// TABLE - optional keyword;
table: bool,
/// IF EXISTS - optional clause (Snowflake)
if_exists: bool,
},
/// Msck (Hive)
Msck {
Expand Down Expand Up @@ -3371,9 +3373,11 @@ impl fmt::Display for Statement {
table_name,
partitions,
table,
if_exists,
} => {
let table = if *table { "TABLE " } else { "" };
write!(f, "TRUNCATE {table}{table_name}")?;
let if_exists = if *if_exists { "IF EXISTS " } else { "" };
write!(f, "TRUNCATE {table}{if_exists}{table_name}")?;
if let Some(ref parts) = partitions {
if !parts.is_empty() {
write!(f, " PARTITION ({})", display_comma_separated(parts))?;
Expand Down
7 changes: 7 additions & 0 deletions src/ast/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2018,13 +2018,20 @@ impl fmt::Display for TableAlias {
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub enum TableVersion {
/// BigQuery / MSSQL: `FOR SYSTEM_TIME AS OF <expr>`
ForSystemTimeAsOf(Expr),
/// Delta Lake / Databricks: `VERSION AS OF <version>` (also the `@v<version>` shorthand)
VersionAsOf(Expr),
/// Delta Lake / Databricks: `TIMESTAMP AS OF <expr>` (also the `@<yyyyMMddHHmmssSSS>` shorthand)
TimestampAsOf(Expr),
}

impl Display for TableVersion {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
TableVersion::ForSystemTimeAsOf(e) => write!(f, " FOR SYSTEM_TIME AS OF {e}")?,
TableVersion::VersionAsOf(e) => write!(f, " VERSION AS OF {e}")?,
TableVersion::TimestampAsOf(e) => write!(f, " TIMESTAMP AS OF {e}")?,
}
Ok(())
}
Expand Down
84 changes: 84 additions & 0 deletions src/parser/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1087,6 +1087,7 @@ impl<'a> Parser<'a> {

pub fn parse_truncate(&mut self) -> Result<Statement, ParserError> {
let table = self.parse_keyword(Keyword::TABLE);
let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]);
let table_name = self.parse_object_name(false)?;
let mut partitions = None;
if self.parse_keyword(Keyword::PARTITION) {
Expand All @@ -1098,6 +1099,7 @@ impl<'a> Parser<'a> {
table_name,
partitions,
table,
if_exists,
})
}

Expand Down Expand Up @@ -3246,6 +3248,13 @@ impl<'a> Parser<'a> {
};

let (field_type, trailing_bracket) = self.parse_data_type_helper()?;
// Databricks allows a per-field COLLATE clause on string types inside a
// STRUCT, e.g. `STRUCT<first: STRING COLLATE UTF8_LCASE>`. The collation
// name carries no lineage, so it is consumed but not retained in the AST.
// Only valid when the field type did not itself consume the closing `>`.
if !trailing_bracket.0 && self.parse_keyword(Keyword::COLLATE) {
let _ = self.parse_collation_name()?;
}
let not_null = if !trailing_bracket.0 {
self.parse_keywords(&[Keyword::NOT, Keyword::NULL])
} else {
Expand Down Expand Up @@ -3278,6 +3287,13 @@ impl<'a> Parser<'a> {
self.expect_token(&Token::Colon)?;

let (field_type, trailing_bracket) = self.parse_data_type_helper()?;
// Databricks allows a per-field COLLATE clause on string types inside a
// STRUCT, e.g. `STRUCT<first: STRING COLLATE UTF8_LCASE>`. The collation
// name carries no lineage, so it is consumed but not retained in the AST.
// Only valid when the field type did not itself consume the closing `>`.
if !trailing_bracket.0 && self.parse_keyword(Keyword::COLLATE) {
let _ = self.parse_collation_name()?;
}
let not_null = if !trailing_bracket.0 {
self.parse_keywords(&[Keyword::NOT, Keyword::NULL])
} else {
Expand Down Expand Up @@ -16133,6 +16149,17 @@ impl<'a> Parser<'a> {
///
/// For now it only supports timestamp versioning for BigQuery and MSSQL dialects.
pub fn parse_table_version(&mut self) -> Result<Option<TableVersion>, ParserError> {
// Delta Lake / Databricks (and Spark) time travel — may appear before the alias:
// table VERSION AS OF <version>
// table TIMESTAMP AS OF <expr>
// table@v<version> (version shorthand)
// table@<yyyyMMddHHmmssSSS> (timestamp shorthand)
// https://docs.databricks.com/aws/en/delta/history
if dialect_of!(self is DatabricksDialect | GenericDialect) {
if let Some(version) = self.maybe_parse_delta_time_travel()? {
return Ok(Some(version));
}
}
if dialect_of!(self is BigQueryDialect | MsSqlDialect)
&& self.parse_keywords(&[Keyword::FOR, Keyword::SYSTEM_TIME, Keyword::AS, Keyword::OF])
{
Expand Down Expand Up @@ -16221,6 +16248,63 @@ impl<'a> Parser<'a> {
}
}

/// Parse a Delta Lake / Databricks time-travel qualifier that follows a table
/// reference: `VERSION AS OF <version>`, `TIMESTAMP AS OF <expr>`, or the
/// `@v<version>` / `@<timestamp>` shorthands. Returns `None` (consuming nothing)
/// when no such qualifier is present.
fn maybe_parse_delta_time_travel(&mut self) -> Result<Option<TableVersion>, ParserError> {
// VERSION AS OF <version> (VERSION is not a reserved keyword — match by value)
if matches!(
self.peek_tokens(),
[Token::Word(w), Token::Word(a), Token::Word(o)]
if w.value.eq_ignore_ascii_case("VERSION")
&& a.keyword == Keyword::AS
&& o.keyword == Keyword::OF
) {
self.next_token(); // VERSION
self.next_token(); // AS
self.next_token(); // OF
return Ok(Some(TableVersion::VersionAsOf(self.parse_expr()?)));
}
// TIMESTAMP AS OF <expr>
if matches!(
self.peek_tokens(),
[Token::Word(w), Token::Word(a), Token::Word(o)]
if w.keyword == Keyword::TIMESTAMP
&& a.keyword == Keyword::AS
&& o.keyword == Keyword::OF
) {
self.next_token(); // TIMESTAMP
self.next_token(); // AS
self.next_token(); // OF
return Ok(Some(TableVersion::TimestampAsOf(self.parse_expr()?)));
}
// `@v<version>` / `@<yyyyMMddHHmmssSSS>` shorthand. The `@` abuts the table
// name, so it surfaces as a standalone AtSign followed by the qualifier token.
let shorthand = match self.peek_tokens() {
[Token::AtSign, Token::Word(w)]
if w.value.len() > 1
&& (w.value.starts_with('v') || w.value.starts_with('V'))
&& w.value[1..].bytes().all(|b| b.is_ascii_digit()) =>
{
Some(TableVersion::VersionAsOf(Expr::Value(Value::Number(
w.value[1..].parse().unwrap(),
false,
))))
}
[Token::AtSign, Token::Number(n, long)] => Some(TableVersion::TimestampAsOf(
Expr::Value(Value::Number(n.parse().unwrap(), long)),
)),
_ => None,
};
if let Some(version) = shorthand {
self.next_token(); // @
self.next_token(); // version / timestamp token
return Ok(Some(version));
}
Ok(None)
}

pub fn parse_array_join_table_factor(&mut self) -> Result<TableFactor, ParserError> {
{
let expr = self.parse_expr()?;
Expand Down
86 changes: 86 additions & 0 deletions tests/sqlparser_databricks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -662,3 +662,89 @@ fn parse_create_materialized_view_skips_schedule_and_trigger() {
}
}
}

#[test]
fn parse_struct_field_collate() {
// Databricks allows a per-field COLLATE on string types inside STRUCT / nested
// ARRAY<STRUCT<...>>. The collation is consumed but not retained in the AST,
// so the canonical form drops it (lineage-irrelevant).
// https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-collation
databricks().one_statement_parses_to(
"CREATE TABLE users (id STRING COLLATE UTF8_LCASE, name STRUCT<first: STRING COLLATE UTF8_LCASE, last: STRING COLLATE UTF8_LCASE>)",
"CREATE TABLE users (id STRING COLLATE UTF8_LCASE, name STRUCT<first: STRING, last: STRING>)",
);
databricks().one_statement_parses_to(
"CREATE TABLE events (id STRING, tags ARRAY<STRUCT<key: STRING COLLATE UNICODE_CI, value: STRING COLLATE UNICODE_CI>>)",
"CREATE TABLE events (id STRING, tags ARRAY<STRUCT<key: STRING, value: STRING>>)",
);
}

#[test]
fn parse_delta_time_travel_version_as_of() {
// https://docs.databricks.com/aws/en/delta/history
let select = databricks().verified_only_select("SELECT * FROM people10m VERSION AS OF 123");
match &select.from[0].relation {
TableFactor::Table { name, version, .. } => {
assert_eq!(name, &ObjectName(vec![Ident::new("people10m")]));
assert_eq!(
version,
&Some(TableVersion::VersionAsOf(Expr::Value(number("123"))))
);
}
other => panic!("expected Table, got {other:?}"),
}
}

#[test]
fn parse_delta_time_travel_timestamp_as_of() {
// String literal, cast, and function-call timestamp expressions are all valid.
let select = databricks()
.verified_only_select("SELECT * FROM people10m TIMESTAMP AS OF '2018-10-18T22:15:12.013Z'");
match &select.from[0].relation {
TableFactor::Table { name, version, .. } => {
assert_eq!(name, &ObjectName(vec![Ident::new("people10m")]));
assert_eq!(
version,
&Some(TableVersion::TimestampAsOf(Expr::Value(
Value::SingleQuotedString("2018-10-18T22:15:12.013Z".to_string())
)))
);
}
other => panic!("expected Table, got {other:?}"),
}
databricks().verified_stmt(
"SELECT * FROM people10m TIMESTAMP AS OF CAST('2018-10-18 13:36:32 CEST' AS TIMESTAMP)",
);
databricks().verified_stmt(
"SELECT * FROM people10m TIMESTAMP AS OF current_timestamp() - INTERVAL '12' HOUR",
);
}

#[test]
fn parse_delta_time_travel_with_alias_and_merge() {
// Time travel may be written before the alias; it canonicalizes to alias-first
// (matching the existing FOR SYSTEM_TIME ordering).
databricks().one_statement_parses_to(
"SELECT * FROM t VERSION AS OF 5 AS snap",
"SELECT * FROM t AS snap VERSION AS OF 5",
);
databricks().verified_stmt("SELECT * FROM t AS snap VERSION AS OF 5");
// Inside MERGE ... USING (the canonical alias-first form round-trips).
databricks().verified_stmt(
"MERGE INTO my_table AS target USING my_table AS source TIMESTAMP AS OF date_sub(current_date(), 1) ON source.userId = target.userId WHEN MATCHED THEN DELETE",
);
}

#[test]
fn parse_delta_time_travel_at_shorthand() {
// `@v<version>` and `@<yyyyMMddHHmmssSSS>` canonicalize to the AS OF forms.
let select = databricks().one_statement_parses_to(
"SELECT * FROM people10m@v123",
"SELECT * FROM people10m VERSION AS OF 123",
);
assert!(matches!(select, Statement::Query(_)));
databricks().one_statement_parses_to(
"SELECT * FROM people10m@20190101000000000",
"SELECT * FROM people10m TIMESTAMP AS OF 20190101000000000",
);
}
3 changes: 2 additions & 1 deletion tests/sqlparser_postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4157,7 +4157,8 @@ fn parse_truncate() {
Statement::Truncate {
table_name: ObjectName(vec![Ident::new("db"), Ident::new("table_name")]),
partitions: None,
table: false
table: false,
if_exists: false
},
truncate
);
Expand Down
23 changes: 23 additions & 0 deletions tests/sqlparser_snowflake.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4602,3 +4602,26 @@ fn parse_create_unsupported_is_not_a_comment() {
Err(e) => panic!("parse failed: {e}"),
}
}

#[test]
fn parse_truncate_if_exists() {
// Snowflake: TRUNCATE [TABLE] [IF EXISTS] <name>
// https://docs.snowflake.com/en/sql-reference/sql/truncate-table
let truncate =
snowflake().verified_stmt(r#"TRUNCATE IF EXISTS "MY_DB"."MY_SCHEMA"."MY_TABLE""#);
assert_eq!(
Statement::Truncate {
table_name: ObjectName(vec![
Ident::with_quote('"', "MY_DB"),
Ident::with_quote('"', "MY_SCHEMA"),
Ident::with_quote('"', "MY_TABLE"),
]),
partitions: None,
table: false,
if_exists: true,
},
truncate
);
// TABLE keyword may precede IF EXISTS.
snowflake().verified_stmt("TRUNCATE TABLE IF EXISTS db.table_name");
}
Loading