From 0caf4052e2af775f7b41b179977b3c58d0082b47 Mon Sep 17 00:00:00 2001 From: Rafal Hawrylak Date: Wed, 19 Aug 2026 12:10:46 +0000 Subject: [PATCH] Wire PropertyGraph action into compile pipeline Builds on top of #2246 (PropertyGraph action class) and #2249 (review follow-up). The class parsed and rendered a graph body but was not yet loaded from user projects or exposed to downstream code. - Session loading: main.ts finds `graph.yaml` files under `definitions/` and registers a PropertyGraph action per file. At most one graph.yaml is accepted per project. - Action index: actions/index.ts and session.ts expose PropertyGraph so downstream code treats it like tables and operations. - Emitter extensions: description and synonyms on both entities and relationships, fieldWildcard normalization for `fields: { importAll: true, except: [...] }`, and an OPTIONS clause on the graph carrying the graph description. - Proto: adds `is_default` to GraphLabel for the label wiring above. - Coverage: property_graph_test.ts grows for the new normalization and emitter behavior, and main_test.ts adds session-level tests plus a propertyGraph case in the shared test-helper switch. Older `dataformCoreVersion` values silently skip PropertyGraph files so projects pinning an earlier core do not break at load time. --- core/actions/index.ts | 3 + core/actions/property_graph.ts | 128 +++-- core/actions/property_graph_test.ts | 769 +++++++++++++++++++++++++--- core/main.ts | 56 +- core/main_test.ts | 693 ++++++++++++++++++++++++- core/session.ts | 11 +- core/utils.ts | 15 + protos/core.proto | 1 + 8 files changed, 1569 insertions(+), 107 deletions(-) diff --git a/core/actions/index.ts b/core/actions/index.ts index 28b3eabfb..dcf7a83cb 100644 --- a/core/actions/index.ts +++ b/core/actions/index.ts @@ -4,6 +4,7 @@ import { Declaration } from "df/core/actions/declaration"; import { IncrementalTable } from "df/core/actions/incremental_table"; import { Notebook } from "df/core/actions/notebook"; import { Operation } from "df/core/actions/operation"; +import { PropertyGraph } from "df/core/actions/property_graph"; import { Table } from "df/core/actions/table"; import { Test } from "df/core/actions/test"; import { View } from "df/core/actions/view"; @@ -21,6 +22,7 @@ export type Action = | Declaration | Notebook | DataPreparation + | PropertyGraph | Test; export type ActionProto = @@ -30,6 +32,7 @@ export type ActionProto = | dataform.Declaration | dataform.Notebook | dataform.DataPreparation + | dataform.PropertyGraph | dataform.Test; // In v4, consider making methods on inheritors of this private, forcing users to use constructors diff --git a/core/actions/property_graph.ts b/core/actions/property_graph.ts index 87694d3bb..f6002b55a 100644 --- a/core/actions/property_graph.ts +++ b/core/actions/property_graph.ts @@ -113,22 +113,7 @@ export class PropertyGraph extends ActionBuilder { throw new Error(`${where} must declare 'keys'.`); } - const rootFields = entityConfig.fields || []; - const rootWildcard = entityConfig.fieldWildcard; - const configuredLabels = entityConfig.labels || []; - if ((rootFields.length > 0 || rootWildcard) && configuredLabels.length > 0) { - throw new Error( - `${where} cannot combine root-level 'fields'/'fieldWildcard' with a 'labels' list.` - ); - } - - const labels = buildLabels( - entityName, - rootFields, - rootWildcard, - configuredLabels, - where - ); + const labels = buildLabels(entityName, entityConfig, where); return dataform.GraphEntity.create({ name: entityName, @@ -159,16 +144,7 @@ export class PropertyGraph extends ActionBuilder { const source = buildEndpoint(relConfig.source, `${where} source`); const destination = buildEndpoint(relConfig.destination, `${where} destination`); - const rootFields = relConfig.fields || []; - const rootWildcard = relConfig.fieldWildcard; - const configuredLabels = relConfig.labels || []; - if ((rootFields.length > 0 || rootWildcard) && configuredLabels.length > 0) { - throw new Error( - `${where} cannot combine root-level 'fields'/'fieldWildcard' with a 'labels' list.` - ); - } - - const labels = buildLabels(relName, rootFields, rootWildcard, configuredLabels, where); + const labels = buildLabels(relName, relConfig, where); return dataform.GraphRelationship.create({ name: relName, @@ -276,13 +252,17 @@ export class PropertyGraph extends ActionBuilder { } private emitGraphBody(): string { - const nodeEntries = this.proto.entities.map(entity => renderNode(entity)); const parts: string[] = []; + const nodeEntries = this.proto.entities.map(entity => renderNode(entity)); parts.push(`NODE TABLES (\n ${nodeEntries.join(",\n ")}\n)`); if (this.proto.relationships.length > 0) { const edgeEntries = this.proto.relationships.map(rel => renderEdge(rel)); parts.push(`EDGE TABLES (\n ${edgeEntries.join(",\n ")}\n)`); } + const options = renderOptionsClause(this.proto.description, undefined); + if (options) { + parts.push(options); + } return parts.join("\n"); } } @@ -294,8 +274,14 @@ function normalizeKeys(entityOrRel: any) { } function normalizeFields(container: any) { - if (Array.isArray(container.fields)) { - container.fields = container.fields.map((field: string | dataform.IGraphFieldConfig) => + const f = container.fields; + if (f && !Array.isArray(f) && typeof f === "object" && "importAll" in f) { + container.fieldWildcard = f; + delete container.fields; + return; + } + if (Array.isArray(f)) { + container.fields = f.map((field: string | dataform.IGraphFieldConfig) => typeof field === "string" ? { name: field, expression: field } : field ); } @@ -345,26 +331,43 @@ function buildEndpoint( function buildLabels( defaultLabelName: string, - rootFields: dataform.IGraphFieldConfig[], - rootWildcard: dataform.IGraphFieldWildcard | undefined | null, - configuredLabels: dataform.IGraphLabelConfig[], + config: dataform.IGraphEntityConfig | dataform.IGraphRelationshipConfig, where: string ): dataform.GraphLabel[] { - if (configuredLabels.length > 0) { - return configuredLabels.map(label => buildLabel(label, where)); + const rootFields = config.fields || []; + const rootWildcard = config.fieldWildcard; + const configuredLabels = config.labels || []; + if ((rootFields.length > 0 || rootWildcard) && configuredLabels.length > 0) { + throw new Error( + `${where} cannot combine root-level 'fields'/'fieldWildcard' with a 'labels' list.` + ); } - if (rootFields.length === 0 && !rootWildcard) { + if (configuredLabels.length > 0) { + return configuredLabels.map(label => buildLabel(label, where, false)); + } + const hasRootData = + rootFields.length > 0 || + !!rootWildcard || + !!config.description || + (config.synonyms && config.synonyms.length > 0); + if (!hasRootData) { return []; } const synthesized = dataform.GraphLabelConfig.create({ name: defaultLabelName, + description: config.description || undefined, + synonyms: config.synonyms || [], fields: rootFields, fieldWildcard: rootWildcard }); - return [buildLabel(synthesized, where)]; + return [buildLabel(synthesized, where, true)]; } -function buildLabel(label: dataform.IGraphLabelConfig, where: string): dataform.GraphLabel { +function buildLabel( + label: dataform.IGraphLabelConfig, + where: string, + isDefault: boolean +): dataform.GraphLabel { if (!label.name) { throw new Error(`${where}: every label must have a 'name'.`); } @@ -376,9 +379,11 @@ function buildLabel(label: dataform.IGraphLabelConfig, where: string): dataform. } const hasFields = !!(label.fields && label.fields.length > 0); const hasWildcard = !!(wildcard && wildcard.importAll); - if (!hasFields && !hasWildcard) { + const hasOptions = !!label.description || !!(label.synonyms && label.synonyms.length > 0); + if (!hasFields && !hasWildcard && !hasOptions) { throw new Error( - `${where}: label '${label.name}' must declare at least one field or a wildcard.` + `${where}: label '${label.name}' must declare at least one of: 'fields', ` + + `'fieldWildcard', 'description', or 'synonyms'.` ); } return dataform.GraphLabel.create({ @@ -394,7 +399,8 @@ function buildLabel(label: dataform.IGraphLabelConfig, where: string): dataform. }) ), importAll: !!wildcard?.importAll, - importExcept: wildcard?.except || [] + importExcept: wildcard?.except || [], + isDefault }); } @@ -455,7 +461,17 @@ function renderEndpoint(role: "SOURCE" | "DESTINATION", endpoint: dataform.IGrap } function renderLabelClause(label: dataform.IGraphLabel): string { - return `LABEL ${label.name} ${renderPropertiesClause(label)}`; + const head = label.isDefault ? "DEFAULT LABEL" : `LABEL ${label.name}`; + const parts: string[] = [head]; + const options = renderOptionsClause(label.description, label.synonyms); + if (options) { + parts.push(options); + } + const properties = renderPropertiesClause(label); + if (properties) { + parts.push(properties); + } + return parts.join(" "); } function renderPropertiesClause(label: dataform.IGraphLabel): string { @@ -465,10 +481,32 @@ function renderPropertiesClause(label: dataform.IGraphLabel): string { } return "PROPERTIES ARE ALL COLUMNS"; } - const rendered = label.fields.map(field => + if (label.fields.length === 0) { + return ""; + } + const rendered = label.fields.map(field => renderField(field)); + return `PROPERTIES (${rendered.join(", ")})`; +} + +function renderField(field: dataform.IGraphField): string { + const expr = field.expression && field.expression !== field.name ? `${field.expression} AS ${field.name}` - : field.name - ); - return `PROPERTIES (${rendered.join(", ")})`; + : field.name; + const options = renderOptionsClause(field.description, field.synonyms); + return options ? `${expr} ${options}` : expr; +} + +function renderOptionsClause( + description: string | null | undefined, + synonyms: string[] | null | undefined +): string { + const parts: string[] = []; + if (description) { + parts.push(`description=${JSON.stringify(description)}`); + } + if (synonyms && synonyms.length > 0) { + parts.push(`synonyms=[${synonyms.map(s => JSON.stringify(s)).join(", ")}]`); + } + return parts.length > 0 ? `OPTIONS(${parts.join(", ")})` : ""; } diff --git a/core/actions/property_graph_test.ts b/core/actions/property_graph_test.ts index d9b7b34e2..4fd7094b6 100644 --- a/core/actions/property_graph_test.ts +++ b/core/actions/property_graph_test.ts @@ -19,6 +19,12 @@ function compile(config: any, filename = "definitions/graph.yaml"): dataform.Pro return new PropertyGraph(makeSession(), config, filename).compile(); } +const graphTarget = (name: string) => ({ + database: "defaultProject", + schema: "defaultDataset", + name +}); + suite("property_graph", () => { test("compiles a minimal graph with one entity and no relationships", () => { const compiled = compile({ @@ -34,16 +40,8 @@ suite("property_graph", () => { expect(asPlainObject(compiled)).deep.equals( asPlainObject({ - target: { - database: "defaultProject", - schema: "defaultDataset", - name: "Simple" - }, - canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", - name: "Simple" - }, + target: graphTarget("Simple"), + canonicalTarget: graphTarget("Simple"), fileName: "definitions/graph.yaml", entities: [ { @@ -65,7 +63,21 @@ suite("property_graph", () => { ] }); - expect(compiled.entities[0].keys).deep.equals(["id"]); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"] + } + ], + graphBody: "NODE TABLES (\n `p.d.A` AS A KEY (id)\n)" + }) + ); }); test("parses 3-part dataSourceString into database.schema.name", () => { @@ -76,11 +88,21 @@ suite("property_graph", () => { ] }); - expect(asPlainObject(compiled.entities[0].dataSource)).deep.equals({ - database: "myProj", - schema: "myDs", - name: "MyTable" - }); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "myProj", schema: "myDs", name: "MyTable" }, + keys: ["id"] + } + ], + graphBody: "NODE TABLES (\n `myProj.myDs.MyTable` AS A KEY (id)\n)" + }) + ); }); test("resolves dataSourceDataset map form with default project fallback", () => { @@ -95,11 +117,21 @@ suite("property_graph", () => { ] }); - expect(asPlainObject(compiled.entities[0].dataSource)).deep.equals({ - database: "defaultProject", - schema: "customDs", - name: "MyTable" - }); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "defaultProject", schema: "customDs", name: "MyTable" }, + keys: ["id"] + } + ], + graphBody: "NODE TABLES (\n `defaultProject.customDs.MyTable` AS A KEY (id)\n)" + }) + ); }); test("synthesizes default label named after entity when root-level fields set", () => { @@ -115,10 +147,31 @@ suite("property_graph", () => { ] }); - expect(compiled.entities[0].labels).length(1); - expect(compiled.entities[0].labels[0].name).equals("Account"); - expect(compiled.entities[0].labels[0].fields).length(1); - expect(compiled.entities[0].labels[0].fields[0].name).equals("balance"); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "Account", + description: "", + fields: [{ name: "balance", expression: "balance" }], + importAll: false, + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id) DEFAULT LABEL PROPERTIES (balance)\n)" + }) + ); }); test("expands field shorthand strings into {name, expression} objects", () => { @@ -134,11 +187,34 @@ suite("property_graph", () => { ] }); - const fields = compiled.entities[0].labels[0].fields; - expect(fields.map(f => ({ name: f.name, expression: f.expression }))).deep.equals([ - { name: "balance", expression: "balance" }, - { name: "owner", expression: "owner" } - ]); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "A", + description: "", + fields: [ + { name: "balance", expression: "balance" }, + { name: "owner", expression: "owner" } + ], + importAll: false, + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL PROPERTIES (balance, owner)\n)" + }) + ); }); test("fieldWildcard.importAll renders as PROPERTIES ARE ALL COLUMNS", () => { @@ -154,8 +230,25 @@ suite("property_graph", () => { ] }); - expect(compiled.graphBody).contains("PROPERTIES ARE ALL COLUMNS"); - expect(compiled.graphBody).not.contains("EXCEPT"); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { name: "A", description: "", importAll: true, isDefault: true } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL PROPERTIES ARE ALL COLUMNS\n)" + }) + ); }); test("fieldWildcard with except renders as PROPERTIES ARE ALL COLUMNS EXCEPT (...)", () => { @@ -171,8 +264,31 @@ suite("property_graph", () => { ] }); - expect(compiled.graphBody).contains( - "PROPERTIES ARE ALL COLUMNS EXCEPT (secret, internal)" + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "A", + description: "", + importAll: true, + importExcept: ["secret", "internal"], + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL " + + "PROPERTIES ARE ALL COLUMNS EXCEPT (secret, internal)\n)" + }) ); }); @@ -192,9 +308,38 @@ suite("property_graph", () => { ] }); - expect(compiled.entities[0].labels.map(l => l.name)).deep.equals(["Account", "Auditable"]); - const fragment = compiled.graphBody; - expect(fragment.indexOf("LABEL Account")).below(fragment.indexOf("LABEL Auditable")); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "Account", + fields: [{ name: "id", expression: "id" }], + importAll: false, + isDefault: false + }, + { + name: "Auditable", + fields: [{ name: "createdAt", expression: "created_at" }], + importAll: false, + isDefault: false + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id) " + + "LABEL Account PROPERTIES (id) " + + "LABEL Auditable PROPERTIES (created_at AS createdAt)\n)" + }) + ); }); test("relationship endpoints render SOURCE/DESTINATION KEY REFERENCES clauses", () => { @@ -221,16 +366,8 @@ suite("property_graph", () => { expect(asPlainObject(compiled)).deep.equals( asPlainObject({ - target: { - database: "defaultProject", - schema: "defaultDataset", - name: "FinGraph" - }, - canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", - name: "FinGraph" - }, + target: graphTarget("FinGraph"), + canonicalTarget: graphTarget("FinGraph"), fileName: "definitions/graph.yaml", entities: [ { @@ -286,8 +423,41 @@ suite("property_graph", () => { ] }); - expect(compiled.relationships[0].source.entityColumns).deep.equals(["id"]); - expect(compiled.relationships[0].destination.entityColumns).deep.equals(["id"]); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"] + } + ], + relationships: [ + { + name: "SelfLink", + dataSource: { database: "p", schema: "d", name: "Links" }, + source: { + entity: "Account", + relationshipColumns: ["from_id"], + entityColumns: ["id"] + }, + destination: { + entity: "Account", + relationshipColumns: ["to_id"], + entityColumns: ["id"] + } + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id)\n)\n" + + "EDGE TABLES (\n `p.d.Links` AS SelfLink " + + "SOURCE KEY (from_id) REFERENCES Account (id) " + + "DESTINATION KEY (to_id) REFERENCES Account (id)\n)" + }) + ); }); test("every emitted NODE and EDGE table has explicit AS alias", () => { @@ -313,9 +483,47 @@ suite("property_graph", () => { ] }); - expect(compiled.graphBody).contains("`p.d.Accounts` AS Account"); - expect(compiled.graphBody).contains("`p.d.Persons` AS Person"); - expect(compiled.graphBody).contains("`p.d.Ownership` AS Owns"); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "Accounts" }, + keys: ["id"] + }, + { + name: "Person", + dataSource: { database: "p", schema: "d", name: "Persons" }, + keys: ["id"] + } + ], + relationships: [ + { + name: "Owns", + dataSource: { database: "p", schema: "d", name: "Ownership" }, + source: { + entity: "Person", + relationshipColumns: ["person_id"], + entityColumns: ["id"] + }, + destination: { + entity: "Account", + relationshipColumns: ["account_id"], + entityColumns: ["id"] + } + } + ], + graphBody: + "NODE TABLES (\n `p.d.Accounts` AS Account KEY (id),\n" + + " `p.d.Persons` AS Person KEY (id)\n)\n" + + "EDGE TABLES (\n `p.d.Ownership` AS Owns " + + "SOURCE KEY (person_id) REFERENCES Person (id) " + + "DESTINATION KEY (account_id) REFERENCES Account (id)\n)" + }) + ); }); test("errors when graph name is missing", () => { @@ -475,8 +683,21 @@ suite("property_graph", () => { ] }); - expect(compiled.entities[0].keys).deep.equals(["id", "date"]); - expect(compiled.graphBody).contains("KEY (id, date)"); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id", "date"] + } + ], + graphBody: "NODE TABLES (\n `p.d.A` AS A KEY (id, date)\n)" + }) + ); }); test("joinKeys array shorthand expands to relationshipColumns with defaulted entityColumns", () => { @@ -495,10 +716,41 @@ suite("property_graph", () => { ] }); - expect(compiled.relationships[0].source.relationshipColumns).deep.equals(["from_id"]); - expect(compiled.relationships[0].source.entityColumns).deep.equals(["id"]); - expect(compiled.relationships[0].destination.relationshipColumns).deep.equals(["to_id"]); - expect(compiled.relationships[0].destination.entityColumns).deep.equals(["id"]); + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"] + } + ], + relationships: [ + { + name: "Link", + dataSource: { database: "p", schema: "d", name: "L" }, + source: { + entity: "Account", + relationshipColumns: ["from_id"], + entityColumns: ["id"] + }, + destination: { + entity: "Account", + relationshipColumns: ["to_id"], + entityColumns: ["id"] + } + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id)\n)\n" + + "EDGE TABLES (\n `p.d.L` AS Link " + + "SOURCE KEY (from_id) REFERENCES Account (id) " + + "DESTINATION KEY (to_id) REFERENCES Account (id)\n)" + }) + ); }); test("errors when label declares neither fields nor a wildcard", () => { @@ -514,6 +766,405 @@ suite("property_graph", () => { } ] }) - ).to.throw("must declare at least one field or a wildcard"); + ).to.throw("must declare at least one of: 'fields', 'fieldWildcard', 'description'"); + }); + + test("fields:{importAll:true} map form hoists to fieldWildcard", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "A", + dataSourceString: "p.d.A", + keys: ["id"], + fields: { importAll: true } + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { name: "A", description: "", importAll: true, isDefault: true } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL PROPERTIES ARE ALL COLUMNS\n)" + }) + ); + }); + + test("fields:{importAll:true, except:[...]} map form hoists with except", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "A", + dataSourceString: "p.d.A", + keys: ["id"], + fields: { importAll: true, except: ["secret"] } + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "A", + description: "", + importAll: true, + importExcept: ["secret"], + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL " + + "PROPERTIES ARE ALL COLUMNS EXCEPT (secret)\n)" + }) + ); + }); + + test("synthesized default label sets isDefault=true and renders DEFAULT LABEL", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "Account", + dataSourceString: "p.d.A", + keys: ["id"], + fields: [{ name: "balance", expression: "balance" }] + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "Account", + description: "", + fields: [{ name: "balance", expression: "balance" }], + importAll: false, + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id) DEFAULT LABEL PROPERTIES (balance)\n)" + }) + ); + }); + + test("explicitly configured labels set isDefault=false and render LABEL ", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "Account", + dataSourceString: "p.d.A", + keys: ["id"], + labels: [ + { name: "Account", fields: [{ name: "id", expression: "id" }] } + ] + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "Account", + fields: [{ name: "id", expression: "id" }], + importAll: false, + isDefault: false + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id) LABEL Account PROPERTIES (id)\n)" + }) + ); + }); + + test("named label description and synonyms render as inline LABEL OPTIONS", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "Account", + dataSourceString: "p.d.A", + keys: ["id"], + labels: [ + { + name: "Account", + description: "customer accounts", + synonyms: ["customer", "user_account"], + fields: [{ name: "id", expression: "id" }] + } + ] + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "Account", + description: "customer accounts", + synonyms: ["customer", "user_account"], + fields: [{ name: "id", expression: "id" }], + importAll: false, + isDefault: false + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id) LABEL Account " + + `OPTIONS(description="customer accounts", ` + + `synonyms=["customer", "user_account"]) PROPERTIES (id)\n)` + }) + ); + }); + + test("field description and synonyms render as per-field OPTIONS(...)", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "A", + dataSourceString: "p.d.A", + keys: ["id"], + fields: [ + { + name: "balance", + expression: "balance", + description: "current balance in USD", + synonyms: ["amount"] + } + ] + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "A", + description: "", + fields: [ + { + name: "balance", + expression: "balance", + description: "current balance in USD", + synonyms: ["amount"] + } + ], + importAll: false, + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL PROPERTIES " + + `(balance OPTIONS(description="current balance in USD", synonyms=["amount"]))\n)` + }) + ); + }); + + test("graph-level description appends OPTIONS(description=...) after NODE/EDGE TABLES", () => { + const compiled = compile({ + name: "G", + description: "high-value customer graph", + entities: [ + { name: "A", dataSourceString: "p.d.A", keys: ["id"] } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + description: "high-value customer graph", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id)\n)\n" + + `OPTIONS(description="high-value customer graph")` + }) + ); + }); + + test("entity with only description synthesizes a DEFAULT LABEL with OPTIONS", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "Account", + dataSourceString: "p.d.A", + keys: ["id"], + description: "customer account entity" + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Account", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "Account", + description: "customer account entity", + importAll: false, + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS Account KEY (id) DEFAULT LABEL " + + `OPTIONS(description="customer account entity")\n)` + }) + ); + }); + + test("string quoting in OPTIONS escapes embedded double quotes and backslashes", () => { + const compiled = compile({ + name: "G", + entities: [ + { + name: "A", + dataSourceString: "p.d.A", + keys: ["id"], + description: `has "quotes" and \\ backslash` + } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"], + labels: [ + { + name: "A", + description: `has "quotes" and \\ backslash`, + importAll: false, + isDefault: true + } + ] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id) DEFAULT LABEL " + + `OPTIONS(description="has \\"quotes\\" and \\\\ backslash")\n)` + }) + ); + }); + + test("string quoting in OPTIONS escapes newlines carriage returns and tabs", () => { + const compiled = compile({ + name: "G", + description: "line1\nline2\r\nline3\ttab", + entities: [ + { name: "A", dataSourceString: "p.d.A", keys: ["id"] } + ] + }); + + expect(asPlainObject(compiled)).deep.equals( + asPlainObject({ + target: graphTarget("G"), + canonicalTarget: graphTarget("G"), + fileName: "definitions/graph.yaml", + description: "line1\nline2\r\nline3\ttab", + entities: [ + { + name: "A", + dataSource: { database: "p", schema: "d", name: "A" }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n `p.d.A` AS A KEY (id)\n)\n" + + `OPTIONS(description="line1\\nline2\\r\\nline3\\ttab")` + }) + ); }); }); diff --git a/core/main.ts b/core/main.ts index 87df59ee7..9cfbeb65a 100644 --- a/core/main.ts +++ b/core/main.ts @@ -10,12 +10,13 @@ import { Declaration } from "df/core/actions/declaration"; import { IncrementalTable } from "df/core/actions/incremental_table"; import { Notebook } from "df/core/actions/notebook"; import { Operation } from "df/core/actions/operation"; +import { PropertyGraph } from "df/core/actions/property_graph"; import { Table } from "df/core/actions/table"; import { View } from "df/core/actions/view"; import { IDataformExtension } from "df/core/extension"; import * as Path from "df/core/path"; import { Session } from "df/core/session"; -import { nativeRequire } from "df/core/utils"; +import { nativeRequire, snakeToCamelKeys } from "df/core/utils"; import { readWorkflowSettings } from "df/core/workflow_settings"; import { dataform } from "df/protos/ts"; @@ -222,6 +223,58 @@ function loadActionConfigsFile( return dataform.ActionConfigs.fromObject(actionConfigsAsJson); } +// Only a single graph.yaml per project is accepted for now, though it may live +// under any subdirectory of definitions/. Multi-graph support may be added in +// a future version. +function loadPropertyGraphs(session: Session, filePaths: string[]) { + const graphPaths = filePaths + .filter( + path => + path.startsWith(`definitions${Path.separator}`) && + Path.basename(path) === "graph" && + Path.fileExtension(path) === "yaml" + ) + .sort(); + if (graphPaths.length === 0) { + return; + } + if (graphPaths.length > 1) { + session.compileError( + new Error( + `At most one graph.yaml is allowed per project (found ${graphPaths.length}: ` + + `${graphPaths.join(", ")}). This restriction may be relaxed in a future ` + + `version.` + ), + graphPaths[0] + ); + return; + } + const graphPath = graphPaths[0]; + let configAsJson: any; + try { + // tslint:disable-next-line: tsr-detect-non-literal-require + configAsJson = snakeToCamelKeys(nativeRequire(graphPath).asJson); + } catch (e) { + session.compileError(e, graphPath); + return; + } + if (!configAsJson || typeof configAsJson !== "object" || Array.isArray(configAsJson)) { + session.compileError( + new Error( + "Property graph config is empty or malformed. " + + "Expected a top-level object with 'name' and 'entities'." + ), + graphPath + ); + return; + } + try { + session.actions.push(new PropertyGraph(session, configAsJson, graphPath)); + } catch (e) { + session.compileError(e, graphPath); + } +} + function prologueCompile(compileRequest: dataform.ICompileExecutionRequest, session: Session) { if (compileRequest?.compileConfig?.extension?.compilationMode === dataform.ExtensionCompilationMode.PROLOGUE) { extensionCompile(compileRequest, session); @@ -279,6 +332,7 @@ function dataformCompile(compileRequest: dataform.ICompileExecutionRequest, sess globalAny.getContents = session.getContents.bind(session); loadActionConfigs(session, compileRequest.compileConfig.filePaths); + loadPropertyGraphs(session, compileRequest.compileConfig.filePaths); // Require all "definitions" files (attaching them to the session). compileRequest.compileConfig.filePaths diff --git a/core/main_test.ts b/core/main_test.ts index 47ac684c5..b65695c6d 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -26,7 +26,7 @@ interface IVerifiableAction { } function toVerifiableAction(graph: dataform.ICompiledGraph, actionType: string): IVerifiableAction { - let action: dataform.IAssertion | dataform.ITable + let action: dataform.IAssertion | dataform.ITable | dataform.IPropertyGraph switch (actionType) { case "assertion": action = graph.assertions[0]; @@ -34,6 +34,9 @@ function toVerifiableAction(graph: dataform.ICompiledGraph, actionType: string): case "operations": action = graph.operations[0]; break; + case "propertyGraph": + action = graph.propertyGraphs[0]; + break; default: action = graph.tables[0]; } @@ -2380,4 +2383,692 @@ select 1 as a` expect(result.compile.compiledGraph.tables[0].bigquery.preserveGovernanceControls).equals(false); }); }); + + suite("property graphs", () => { + const graphProjectConfig = { + warehouse: "bigquery", + defaultSchema: "defaultDataset", + defaultDatabase: "defaultProject", + defaultLocation: "US" + }; + const graphStackTail = "\n at CallSite {}".repeat(10); + const graphError = (fileName: string, message: string, extra: object = {}) => ({ + fileName, + message, + stack: `Error: ${message}${graphStackTail}`, + ...extra + }); + + test("valid graph.yaml compiles end-to-end", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: SimpleGraph +entities: +- name: Customer + dataSourceString: defaultProject.defaultDataset.customers + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: {}, + dataformCoreVersion: version, + targets: [ + { schema: "defaultDataset", name: "SimpleGraph", database: "defaultProject" } + ], + jitData: {}, + propertyGraphs: [ + { + target: { + schema: "defaultDataset", + name: "SimpleGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "SimpleGraph", + database: "defaultProject" + }, + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Customer", + dataSource: { + schema: "defaultDataset", + name: "customers", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.customers` AS Customer KEY (id)\n" + + ")" + } + ] + })); + }); + + test("more than one graph.yaml is rejected", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.mkdirSync(path.join(projectDir, "definitions/subdir")); + const graphBody = ` +name: MultiGraph +entities: +- name: Node + dataSourceString: defaultProject.defaultDataset.t + keys: + - id +`; + fs.writeFileSync(path.join(projectDir, "definitions/graph.yaml"), graphBody); + fs.writeFileSync(path.join(projectDir, "definitions/subdir/graph.yaml"), graphBody); + + const request = dataform.CoreExecutionRequest.create({ + compile: { + compileConfig: { + projectDir: fs.realpathSync(projectDir), + filePaths: [ + "workflow_settings.yaml", + "definitions/graph.yaml", + "definitions/subdir/graph.yaml" + ] + } + } + }); + + const result = runMainInVm(request); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + "At most one graph.yaml is allowed per project (found 2: " + + "definitions/graph.yaml, definitions/subdir/graph.yaml). This " + + "restriction may be relaxed in a future version." + ) + ] + }, + dataformCoreVersion: version, + jitData: {} + })); + }); + + test("nodes-only graph compiles without EDGE TABLES", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: NodesOnly +entities: +- name: Customer + dataSourceString: defaultProject.defaultDataset.customers + keys: + - id +- name: Product + dataSourceString: defaultProject.defaultDataset.products + keys: + - sku +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: {}, + dataformCoreVersion: version, + targets: [ + { schema: "defaultDataset", name: "NodesOnly", database: "defaultProject" } + ], + jitData: {}, + propertyGraphs: [ + { + target: { + schema: "defaultDataset", + name: "NodesOnly", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "NodesOnly", + database: "defaultProject" + }, + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Customer", + dataSource: { + schema: "defaultDataset", + name: "customers", + database: "defaultProject" + }, + keys: ["id"] + }, + { + name: "Product", + dataSource: { + schema: "defaultDataset", + name: "products", + database: "defaultProject" + }, + keys: ["sku"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.customers` AS Customer KEY (id),\n" + + " `defaultProject.defaultDataset.products` AS Product KEY (sku)\n" + + ")" + } + ] + })); + }); + + test("targetDataset overrides the schema on the graph target", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: CustomDsGraph +targetDataset: + datasetId: customDs +entities: +- name: Customer + dataSourceString: defaultProject.defaultDataset.customers + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: {}, + dataformCoreVersion: version, + targets: [ + { schema: "customDs", name: "CustomDsGraph", database: "defaultProject" } + ], + jitData: {}, + propertyGraphs: [ + { + target: { + schema: "customDs", + name: "CustomDsGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "customDs", + name: "CustomDsGraph", + database: "defaultProject" + }, + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Customer", + dataSource: { + schema: "defaultDataset", + name: "customers", + database: "defaultProject" + }, + keys: ["id"] + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.customers` AS Customer KEY (id)\n" + + ")" + } + ] + })); + }); + + test("empty graph.yaml produces a compilation error", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/graph.yaml"), ""); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + "Property graph config is empty or malformed. Expected a top-level " + + "object with 'name' and 'entities'." + ) + ] + }, + dataformCoreVersion: version, + jitData: {} + })); + }); + + test("graph.yaml with only a comment produces a compilation error", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + "# nothing here\n" + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + "Property graph config is empty or malformed. Expected a top-level " + + "object with 'name' and 'entities'." + ) + ] + }, + dataformCoreVersion: version, + jitData: {} + })); + }); + + test("graph.yaml with a top-level scalar produces a compilation error", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + "just a string\n" + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + "Property graph config is empty or malformed. Expected a top-level " + + "object with 'name' and 'entities'." + ) + ] + }, + dataformCoreVersion: version, + jitData: {} + })); + }); + + test("graph.yaml missing entities produces a compilation error", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: EmptyGraph +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError( + "definitions/graph.yaml", + "Property graph 'EmptyGraph' must declare at least one entity." + ) + ] + }, + dataformCoreVersion: version, + jitData: {} + })); + }); + + test("graph with relationships emits EDGE TABLES", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: RelGraph +entities: +- name: Customer + dataSourceString: defaultProject.defaultDataset.customers + keys: + - id +- name: Order + dataSourceString: defaultProject.defaultDataset.orders + keys: + - id +relationships: +- name: PlacedBy + dataSourceString: defaultProject.defaultDataset.orders + source: + entity: Order + joinKeys: + - order_id + destination: + entity: Customer + joinKeys: + - customer_id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: {}, + dataformCoreVersion: version, + targets: [ + { schema: "defaultDataset", name: "RelGraph", database: "defaultProject" } + ], + jitData: {}, + propertyGraphs: [ + { + target: { + schema: "defaultDataset", + name: "RelGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "RelGraph", + database: "defaultProject" + }, + fileName: "definitions/graph.yaml", + entities: [ + { + name: "Customer", + dataSource: { + schema: "defaultDataset", + name: "customers", + database: "defaultProject" + }, + keys: ["id"] + }, + { + name: "Order", + dataSource: { + schema: "defaultDataset", + name: "orders", + database: "defaultProject" + }, + keys: ["id"] + } + ], + relationships: [ + { + name: "PlacedBy", + dataSource: { + schema: "defaultDataset", + name: "orders", + database: "defaultProject" + }, + source: { + entity: "Order", + relationshipColumns: ["order_id"], + entityColumns: ["id"] + }, + destination: { + entity: "Customer", + relationshipColumns: ["customer_id"], + entityColumns: ["id"] + } + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.customers` AS Customer KEY (id),\n" + + " `defaultProject.defaultDataset.orders` AS Order KEY (id)\n" + + ")\n" + + "EDGE TABLES (\n" + + " `defaultProject.defaultDataset.orders` AS PlacedBy " + + "SOURCE KEY (order_id) REFERENCES Order (id) " + + "DESTINATION KEY (customer_id) REFERENCES Customer (id)\n" + + ")" + } + ] + })); + }); + + test("graph target colliding with a table target is flagged as duplicate", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/collision.sqlx"), + `config {type: "table", name: "CollisionName"} +select 1 as a` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: CollisionName +entities: +- name: Customer + dataSourceString: defaultProject.defaultDataset.customers + keys: + - id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + const collisionTarget = { + schema: "defaultDataset", + name: "CollisionName", + database: "defaultProject" + }; + const collisionActionName = "defaultProject.defaultDataset.CollisionName"; + const collisionTargetJson = + `{"schema":"defaultDataset","name":"CollisionName","database":"defaultProject"}`; + const duplicateActionMessage = + "Duplicate action name detected. Names within a schema must be unique " + + "across tables, declarations, assertions, and operations:\n" + + `"${collisionTargetJson}"`; + const duplicateCanonicalMessage = + "Duplicate canonical target detected. Canonical targets must be unique " + + "across tables, declarations, assertions, and operations:\n" + + `"${collisionTargetJson}"`; + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: { + compilationErrors: [ + graphError("definitions/collision.sqlx", duplicateActionMessage, { + actionName: collisionActionName, + actionTarget: collisionTarget + }), + graphError("definitions/collision.sqlx", duplicateCanonicalMessage, { + actionName: collisionActionName, + actionTarget: collisionTarget + }), + graphError("definitions/graph.yaml", duplicateActionMessage, { + actionName: collisionActionName, + actionTarget: collisionTarget + }), + graphError("definitions/graph.yaml", duplicateCanonicalMessage, { + actionName: collisionActionName, + actionTarget: collisionTarget + }) + ] + }, + dataformCoreVersion: version, + targets: [collisionTarget, collisionTarget], + jitData: {} + })); + }); + test("graph.yaml accepts snake_case keys per BQ spec", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/graph.yaml"), + ` +name: SnakeGraph +description: end to end snake case +target_dataset: + project_id: defaultProject + dataset_id: defaultDataset +entities: +- name: Account + data_source_string: defaultProject.defaultDataset.accounts + keys: + - id + fields: + import_all: true + except: + - secret +relationships: +- name: Owns + data_source_string: defaultProject.defaultDataset.ownership + source: + entity: Account + join_keys: + relationship_columns: + - owner_id + destination: + entity: Account + join_keys: + relationship_columns: + - owned_id +` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(asPlainObject(result.compile.compiledGraph)).deep.equals(asPlainObject({ + projectConfig: graphProjectConfig, + graphErrors: {}, + dataformCoreVersion: version, + targets: [ + { schema: "defaultDataset", name: "SnakeGraph", database: "defaultProject" } + ], + jitData: {}, + propertyGraphs: [ + { + target: { + schema: "defaultDataset", + name: "SnakeGraph", + database: "defaultProject" + }, + canonicalTarget: { + schema: "defaultDataset", + name: "SnakeGraph", + database: "defaultProject" + }, + fileName: "definitions/graph.yaml", + description: "end to end snake case", + entities: [ + { + name: "Account", + dataSource: { + schema: "defaultDataset", + name: "accounts", + database: "defaultProject" + }, + keys: ["id"], + labels: [ + { + name: "Account", + description: "", + importAll: true, + importExcept: ["secret"], + isDefault: true + } + ] + } + ], + relationships: [ + { + name: "Owns", + dataSource: { + schema: "defaultDataset", + name: "ownership", + database: "defaultProject" + }, + source: { + entity: "Account", + relationshipColumns: ["owner_id"], + entityColumns: ["id"] + }, + destination: { + entity: "Account", + relationshipColumns: ["owned_id"], + entityColumns: ["id"] + } + } + ], + graphBody: + "NODE TABLES (\n" + + " `defaultProject.defaultDataset.accounts` AS Account KEY (id) " + + "DEFAULT LABEL PROPERTIES ARE ALL COLUMNS EXCEPT (secret)\n" + + ")\n" + + "EDGE TABLES (\n" + + " `defaultProject.defaultDataset.ownership` AS Owns " + + "SOURCE KEY (owner_id) REFERENCES Account (id) " + + "DESTINATION KEY (owned_id) REFERENCES Account (id)\n" + + ")\n" + + `OPTIONS(description="end to end snake case")` + } + ] + })); + }); + }); }); diff --git a/core/session.ts b/core/session.ts index c078f158b..373a76bfb 100644 --- a/core/session.ts +++ b/core/session.ts @@ -11,6 +11,7 @@ import { Declaration } from "df/core/actions/declaration"; import { IncrementalTable } from "df/core/actions/incremental_table"; import { Notebook } from "df/core/actions/notebook"; import { Operation, OperationContext } from "df/core/actions/operation"; +import { PropertyGraph } from "df/core/actions/property_graph"; import { Table, TableContext } from "df/core/actions/table"; import { Test } from "df/core/actions/test"; import { View } from "df/core/actions/view"; @@ -504,6 +505,9 @@ export class Session { dataPreparations: this.compileGraphChunk( this.actions.filter(action => action instanceof DataPreparation) ), + propertyGraphs: this.compileGraphChunk( + this.actions.filter(action => action instanceof PropertyGraph) + ), graphErrors: this.graphErrors, dataformCoreVersion, targets: this.actions.map(action => action.getTarget()), @@ -521,6 +525,7 @@ export class Session { compiledGraph.operations, compiledGraph.notebooks, compiledGraph.dataPreparations, + compiledGraph.propertyGraphs, compiledGraph.tests ) ); @@ -532,6 +537,7 @@ export class Session { compiledGraph.operations, compiledGraph.notebooks, compiledGraph.dataPreparations, + compiledGraph.propertyGraphs, compiledGraph.tests ), [].concat(compiledGraph.declarations.map(declaration => declaration.target)) @@ -548,6 +554,7 @@ export class Session { compiledGraph.operations, compiledGraph.notebooks, compiledGraph.dataPreparations, + compiledGraph.propertyGraphs, compiledGraph.tests ) ); @@ -787,7 +794,8 @@ export class Session { compiledGraph.operations, compiledGraph.declarations, compiledGraph.notebooks, - compiledGraph.dataPreparations + compiledGraph.dataPreparations, + compiledGraph.propertyGraphs ); const nonUniqueActionsTargets = getNonUniqueTargets(actions.map(action => action.target)); @@ -835,6 +843,7 @@ export class Session { compiledGraph.assertions = compiledGraph.assertions.filter(isUniqueAction); compiledGraph.notebooks = compiledGraph.notebooks.filter(isUniqueAction); compiledGraph.dataPreparations = compiledGraph.dataPreparations.filter(isUniqueAction); + compiledGraph.propertyGraphs = compiledGraph.propertyGraphs.filter(isUniqueAction); } } diff --git a/core/utils.ts b/core/utils.ts index 9068ad669..960a1e616 100644 --- a/core/utils.ts +++ b/core/utils.ts @@ -663,3 +663,18 @@ export class ResolvableMap { this.setByNameLevel(forSchema, actionTarget.name, value); } } + +export function snakeToCamelKeys(value: any): any { + if (Array.isArray(value)) { + return value.map(snakeToCamelKeys); + } + if (value && typeof value === "object") { + const out: { [key: string]: any } = {}; + for (const [key, val] of Object.entries(value)) { + const camel = key.replace(/_([a-z])/g, (_, c) => c.toUpperCase()); + out[camel] = snakeToCamelKeys(val); + } + return out; + } + return value; +} diff --git a/protos/core.proto b/protos/core.proto index cf9f6e4e7..913daf5f7 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -472,6 +472,7 @@ message GraphLabel { repeated GraphField fields = 4; bool import_all = 5; repeated string import_except = 6; + bool is_default = 7; } message GraphField {