{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.7.json", "title": "FLUID 0.7.7 — Preview: recursive streaming types", "description": "Opt-in preview. Adds recursive struct fields, array elements and map key/value types. Stable 0.7.6 is unchanged.", "type": "object", "additionalProperties": false, "required": [ "fluidVersion", "kind", "id", "name", "metadata", "exposes" ], "examples": [ { "fluidVersion": "0.5.7", "kind": "DataProduct", "id": "gold.customer.analytics_360_v1", "name": "Customer 360 Analytics", "description": "Unified customer profiles with ML-driven insights", "domain": "Customer Experience", "tags": [ "customer-data", "analytics", "gold-layer" ], "labels": { "team": "customer-analytics", "criticality": "high", "cost-center": "engineering" }, "metadata": { "layer": "Gold", "owner": { "team": "customer-analytics", "email": "customer-analytics@company.com" }, "businessContext": { "domain": "Customer Experience", "subdomain": "Customer Intelligence" } }, "exposes": [ { "exposeId": "customer_profiles", "title": "Unified Customer Profiles", "version": "2.1.0", "kind": "table", "tags": [ "pii", "customer-facing", "real-time" ], "labels": { "sensitivity": "high", "retention": "7-years", "region": "us-central1" }, "contract": { "schema": [ { "name": "customer_id", "type": "STRING", "required": true, "description": "Unique customer identifier", "sensitivity": "cleartext", "tags": [ "identifier", "primary-key" ], "labels": { "business-name": "Customer ID", "data-category": "identifier" } } ] }, "binding": { "platform": "gcp", "format": "bigquery_table", "location": { "project": "company-data", "dataset": "gold_customer", "table": "profiles_v1" } } } ] } ], "properties": { "extensions": { "type": "object", "additionalProperties": true, "description": "Vendor / plugin-namespaced configuration. Each plugin claims a single sub-key (e.g. customScaffold, customCatalog) and validates its own schema there via the fluid_build.extension_validators entry-point." }, "fluidVersion": { "type": "string", "pattern": "^\\d+\\.\\d+(\\.\\d+)?$", "enum": [ "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7" ], "description": "Contract schema version. Accepts '0.7.3' (source-aligned data products + acquisition pattern + ingestion engines) or '0.7.4' (adds the MCP output-port gateway + runtime agentPolicy enforcement). 0.7.4 is fully backward-compatible with 0.7.3 — every 0.7.3 contract validates as 0.7.4 unchanged. '0.7.7' is the current preview.", "examples": [ "0.7.4", "0.7.3" ] }, "kind": { "type": "string", "enum": [ "DataProduct", "MLPipeline" ], "description": "Kind of contract. 'MLPipeline' added for basic ML support." }, "id": { "$ref": "#/$defs/identifier", "description": "Stable product identifier, e.g. 'gold.hr.employee_360_v1'." }, "name": { "type": "string", "minLength": 1 }, "description": { "description": "NEW in v0.7.6: Either a plain string — unchanged meaning, it becomes `purpose` on export — or the structured object. While this was a string, an export of a valid contract could only ever carry `description.purpose`: `usage` and `limitations` had no home a contract could declare, so every ODCS and ODPS export left them empty. `fluid odps import` restores the object to the root `description` when the schema version the imported contract declares accepts an object, and also keeps a verbatim copy in `extensions.odps.metadata.description` so the ODPS leg stays an exact fixed point. All three fields therefore survive a FLUID → ODPS → FLUID cycle and reach both the ODCS and the ODPS export afterwards. Importing against a version whose root `description` is a string still roots `purpose` alone, with the object recoverable from that bucket.", "oneOf": [ { "type": "string", "description": "Prose description. Exported as `description.purpose`." }, { "$ref": "#/$defs/descriptionObject" } ] }, "domain": { "type": "string", "description": "Business domain/mesh domain (e.g., 'HR', 'Finance')." }, "tags": { "$ref": "#/$defs/tags", "description": "Product-level tags for categorization and discovery." }, "labels": { "$ref": "#/$defs/labels", "description": "Product-level labels for metadata and automation." }, "metadata": { "type": "object", "additionalProperties": false, "required": [ "owner" ], "properties": { "provenance": { "type": "object", "description": "Generation envelope injected by `fluid forge` / `fluid init` — records how, when and by which tool/command the contract was generated. Machine-written (not user-authored); additive metadata that does not affect the data product's semantic contract.", "properties": { "schema_version": { "type": "integer", "description": "Envelope schema version." }, "kind": { "type": "string", "description": "Envelope discriminator, e.g. 'ContractMetadata'." }, "generated_at": { "type": "string", "format": "date-time", "description": "ISO 8601 UTC timestamp of generation." }, "generated_by": { "type": "object", "description": "Tool, version and command that generated the contract.", "properties": { "tool": { "type": "string" }, "version": { "type": "string" }, "command": { "type": "string" } } } } }, "layer": { "type": "string", "description": "Data layer label (e.g., Bronze/Silver/Gold). Free-form. When set together with `productType`, the two MUST be consistent under the canonical mapping (Bronze↔SDP, Silver↔ADP, Gold↔CDP)." }, "productType": { "type": "string", "enum": [ "SDP", "ADP", "CDP" ], "description": "NEW in v0.7.3: Data Mesh data-product type. SDP = Source-Aligned Data Product (raw ingestion from external systems; corresponds to Bronze). ADP = Aggregated Data Product (cross-source joined / domain-modelled; corresponds to Silver). CDP = Consumption-Aligned Data Product (analytics- or product-shaped; corresponds to Gold). Either `metadata.layer` or `metadata.productType` is sufficient — the validator infers the missing one. When both are present they MUST agree." }, "classification": { "type": "string", "enum": [ "public", "internal", "confidential", "restricted" ], "description": "NEW in v0.7.3: Data classification label propagated to catalog + access policy enforcement." }, "experimental": { "type": "array", "description": "NEW in v0.7.3: Feature gates the contract opts into (e.g., 'acquisition' while Bronze acquisition is in beta).", "items": { "type": "string" }, "uniqueItems": true }, "owner": { "type": "object", "additionalProperties": false, "properties": { "team": { "type": "string" }, "email": { "type": "string", "format": "email" }, "slack": { "type": "string" }, "oncall": { "type": "string", "description": "Oncall rotation or contact for operational issues." } } }, "team": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: The team behind this product, as a roster rather than a single contact. `metadata.owner` holds one team name and one mailbox and carries no role, so a product with a data owner, a steward and two engineers had nowhere to say so and the exported ODCS/ODPS team block came out empty. The field names are Bitol's, so an imported team round-trips without a translation. Optional: a contract without it exports exactly as it does today.", "properties": { "name": { "type": "string", "description": "Team name, e.g. 'Sales Analytics'." }, "members": { "type": "array", "description": "The people on the team, each with the role they hold on this product.", "items": { "$ref": "#/$defs/teamMember" } } } }, "createdAt": { "type": "string", "format": "date-time" }, "businessContext": { "type": "object", "description": "Business alignment information for data mesh organization.", "additionalProperties": false, "properties": { "domain": { "type": "string", "description": "Business domain this product belongs to." }, "subdomain": { "type": "string", "description": "Business subdomain for more granular organization." }, "businessCapability": { "type": "string", "description": "Business capability this product supports." }, "valueStream": { "type": "string", "description": "Value stream this product participates in." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Product-level tags on metadata for discovery and categorization." } } }, "exposes": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/expose" } }, "consumes": { "type": "array", "items": { "$ref": "#/$defs/consumeRef" } }, "builds": { "type": "array", "description": "NEW in v0.5.5: Multiple build configurations for multi-modal data products (e.g., batch + streaming, SQL + ML).", "items": { "$ref": "#/$defs/build" } }, "build": { "$ref": "#/$defs/build", "description": "Single build configuration (legacy). Use 'builds' array for multi-modal." }, "lineage": { "$ref": "#/$defs/lineage", "description": "Optional lineage tracking for better data governance." }, "schemaEvolution": { "$ref": "#/$defs/schemaEvolution", "description": "Optional schema evolution strategy for managing changes." }, "machineLearning": { "$ref": "#/$defs/machineLearning", "description": "Optional ML model specifications for MLPipeline kind." }, "environments": { "type": "object", "description": "Optional environment-specific overrides (dev, staging, prod).", "additionalProperties": { "$ref": "#/$defs/environmentConfig" } }, "lifecycle": { "$ref": "#/$defs/lifecycle" }, "docs": { "$ref": "#/$defs/docs" }, "sovereignty": { "$ref": "#/$defs/sovereignty", "description": "NEW in v0.7.1: Jurisdiction and data residency requirements for compliance enforcement." }, "accessPolicy": { "$ref": "#/$defs/accessPolicy", "description": "Root-level access policy for automated IAM binding generation. Defines principals, permissions, and resources for data product access control." }, "orchestration": { "$ref": "#/$defs/orchestration", "description": "Top-level orchestration configuration for scheduling and workflow engines." }, "retention": { "$ref": "#/$defs/retention", "description": "NEW in v0.7.3: Retention durations for run state, run logs, lineage events, and DLQ records. Honored by the retention sweeper invoked from policy-apply." }, "consumers": { "type": "array", "description": "NEW in v0.7.6: Declared downstream consumers of this data product — the business artifacts built on it (dashboards, notebooks, analyses, ML apps). Shape follows dbt exposures so existing dbt exposure definitions map field-for-field. Purely declarative and additive: consumers never affect plan/apply. RESERVED, not yet consumed — no code reads this key today; it is carried verbatim into plan.json and the contract COMMENT so it survives round-trip, and is intended to feed lineage terminal nodes and impact statements ('Affects: Revenue dashboard') in a later release.", "items": { "$ref": "#/$defs/consumerRef" } }, "governance": { "$ref": "#/$defs/governance", "description": "Contract-level governance: account/project-wide settings (e.g. AWS Lake Formation admins and LF-tag definitions). Per-resource grants and tag associations live under binding.governance." }, "packaging": { "$ref": "#/$defs/packaging", "description": "NEW in v0.7.6: Contract-wide packaging default — declarative container ownership (isolated = this product creates and owns its infrastructure containers; shared = containers are pre-existing, platform-owned pools the product writes into but does not own). Overridable per exposure via binding.packaging. An ABSENT block means legacy behavior: the IaC emit is byte-identical to pre-packaging releases." } }, "$defs": { "descriptionObject": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: The structured form of the root `description`. The three field names are Bitol's — ODPS v1.1.0 `$defs.Description` and the ODCS v3.1.0 root `description` both use exactly purpose/usage/limitations — so import and export are a pass-through rather than a translation.", "properties": { "purpose": { "type": "string", "description": "What this data product is for." }, "usage": { "type": "string", "description": "How a consumer is expected to use it." }, "limitations": { "type": "string", "description": "What it must not be used for, and what it does not cover." } } }, "teamMember": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: One member of `metadata.team.members`. The field names and the username-only required list are Bitol's ODCS v3.1.0 / ODPS v1.1.0 `$defs.TeamMember`, copied verbatim so an imported team exports back unchanged.", "required": [ "username" ], "properties": { "username": { "type": "string", "description": "The member's identifier in the organisation's directory — the one field Bitol requires." }, "name": { "type": "string", "description": "Display name." }, "role": { "type": "string", "description": "The role this member holds on this product, e.g. 'owner', 'steward', 'data engineer'. Free-form: Bitol does not constrain it." }, "email": { "type": "string", "format": "email" } } }, "packaging": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: Declarative packaging mode — per-container ownership for the OpenTofu IaC emit (RFC-packaging-modes.md). 'isolated' emits owned resources (today's exact emit); 'shared' emits a data-source reference to a platform-owned pool plus leaf-only owned resources (tables, prefixed objects, scoped grants). Hybrid tiers (e.g. Snowflake shared-database / owned-schema / owned-warehouse) come from the per-kind containers override map. Prior art: Terraform's resource-vs-data-source ownership split, Unity Catalog isolation-mode OPEN|ISOLATED, Kubernetes shared-vs-dedicated node pools.", "properties": { "mode": { "type": "string", "enum": [ "isolated", "shared" ], "description": "NEW in v0.7.6: Blanket ownership mode for every container kind. Default when the block is present: 'isolated'. Per-kind exceptions go in `containers`." }, "pool": { "type": "string", "minLength": 1, "description": "NEW in v0.7.6: Pool id of the platform-owned tenant pool this product writes into. REQUIRED (enforced by the resolver) whenever any container resolves 'shared' — a pool must be addressable. Propagated as the `fluid_pool` label/tag for cost attribution." }, "poolManifest": { "type": "string", "minLength": 1, "description": "NEW in v0.7.6: Optional path to the platform team's pool manifest file; snapshotted into the bundle at stage 1 so bundleDigest covers it (any pool-file edit invalidates downstream plans)." }, "containers": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: Per-container-kind ownership overrides — each key wins over `mode` for that kind, yielding hybrid tiers. Kind↔platform mapping: bucket → aws_s3_bucket / google_storage_bucket; database → snowflake_database AND aws_glue_catalog_database; dataset → google_bigquery_dataset; schema → snowflake_schema; warehouse → snowflake_warehouse; cluster → Confluent environment/cluster (v1: shared only).", "properties": { "bucket": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the object-store bucket (aws_s3_bucket / google_storage_bucket)." }, "database": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the database container — covers Snowflake `database` AND AWS `glue_database`." }, "dataset": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the BigQuery dataset (google_bigquery_dataset)." }, "schema": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the Snowflake schema (snowflake_schema)." }, "warehouse": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the Snowflake warehouse (snowflake_warehouse) — isolated gives per-product cost attribution." }, "cluster": { "$ref": "#/$defs/packagingContainerMode", "description": "NEW in v0.7.6: Ownership of the Confluent environment/cluster. v1 accepts 'shared' only — dedicated-cluster provisioning is not yet supported (the resolver rejects 'isolated')." } } } } }, "packagingContainerMode": { "type": "string", "enum": [ "isolated", "shared" ], "description": "NEW in v0.7.6: Per-container ownership — 'isolated' (this product creates and owns it) or 'shared' (pre-existing platform-owned pool, referenced not created)." }, "retention": { "type": "object", "description": "NEW in v0.7.3: Retention durations (ISO-8601). Defaults: runState=P30D, runLogs=P90D, lineage=P365D, dlq=P180D.", "additionalProperties": false, "properties": { "runState": { "$ref": "#/$defs/isoDuration" }, "runLogs": { "$ref": "#/$defs/isoDuration" }, "lineage": { "$ref": "#/$defs/isoDuration" }, "dlq": { "$ref": "#/$defs/isoDuration" } } }, "identifier": { "type": "string", "pattern": "^[A-Za-z0-9_][A-Za-z0-9_.-]*[A-Za-z0-9_]$|^[A-Za-z0-9_]$", "description": "Unified identifier pattern: starts/ends with alphanumeric or underscore, allows dots and hyphens in middle. Both casing conventions accepted (snake_case / dotted lowercase / PascalCase / UPPER) so existing catalog ids round-trip unchanged." }, "semver": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?$", "description": "Semantic version (semver.org)." }, "semverRange": { "type": "string", "description": "Semver range (e.g., ^1.2, ~1.4, >=1.0.0). Not strictly validated by regex; tooling enforces." }, "isoDuration": { "type": "string", "pattern": "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(\\d+H)?(\\d+M)?(\\d+S)?)?$", "description": "ISO-8601 duration (e.g., P1D, PT15M, P2Y6M)." }, "availabilityPct": { "type": "string", "pattern": "^(100(\\.0+)?|\\d{2}(\\.\\d+)?|\\d{1}\\d(\\.\\d+)?)%$", "description": "Availability percentage (e.g., 99.9%)." }, "lifecycleState": { "type": "string", "enum": [ "preview", "active", "deprecated", "retired" ], "description": "Unified lifecycle state vocabulary." }, "sensitivityLevel": { "type": "string", "enum": [ "none", "internal", "confidential", "restricted", "pii", "phi", "cleartext", "treated", "anonymized", "pseudonymized", "tokenized", "encrypted" ], "description": "Unified data sensitivity classification." }, "tags": { "type": "array", "description": "Simple string tags for categorization, discovery, and automation. Consistent pattern used throughout FLUID objects.", "items": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$", "description": "Tag format: lowercase alphanumeric with hyphens, e.g., 'customer-data', 'pii', 'real-time'" }, "uniqueItems": true }, "labels": { "type": "object", "description": "Key-value labels for structured metadata and automation. Consistent pattern used throughout FLUID objects.", "additionalProperties": { "type": "string", "description": "Label values are always strings for consistency and tooling compatibility." }, "examples": [ { "team": "customer-analytics", "criticality": "high", "cost-center": "engineering", "retention": "7-years", "region": "us-central1", "business-name": "Customer Profiles", "data-category": "analytical" } ] }, "expose": { "type": "object", "additionalProperties": false, "required": [ "exposeId", "kind", "binding", "contract" ], "examples": [ { "exposeId": "customer_profiles", "title": "Customer Profiles Table", "kind": "table", "tags": [ "pii", "customer-facing", "analytical" ], "labels": { "sensitivity": "high", "retention": "7-years", "team": "customer-analytics" }, "contract": { "schema": [ { "name": "customer_id", "type": "STRING", "required": true, "sensitivity": "cleartext", "tags": [ "identifier", "primary-key" ], "labels": { "business-name": "Customer ID", "data-category": "identifier" } } ] }, "binding": { "platform": "gcp", "format": "bigquery_table", "location": { "project": "company-data", "dataset": "customer", "table": "profiles" } } } ], "properties": { "exposeId": { "$ref": "#/$defs/identifier", "description": "Stable interface handle (used by consumers & build.outputs)." }, "title": { "type": "string" }, "description": { "type": "string", "description": "Detailed description of the exposed resource." }, "version": { "$ref": "#/$defs/semver" }, "kind": { "type": "string", "enum": [ "table", "view", "api", "file", "stream", "topic", "feature_store", "model", "vector", "graph", "time_series", "other" ] }, "tags": { "$ref": "#/$defs/tags", "description": "Expose-level tags for categorization and routing." }, "labels": { "$ref": "#/$defs/labels", "description": "Expose-level labels for metadata and automation." }, "contract": { "$ref": "#/$defs/exposeContract" }, "qos": { "$ref": "#/$defs/exposeQoS" }, "policy": { "$ref": "#/$defs/exposePolicy" }, "binding": { "$ref": "#/$defs/binding" }, "lifecycle": { "$ref": "#/$defs/exposeLifecycle" }, "observability": { "$ref": "#/$defs/observability" }, "docs": { "$ref": "#/$defs/docs" }, "semantics": { "$ref": "#/$defs/semanticModel", "description": "NEW in v0.7.2: Semantic model definition mapping physical columns to business concepts. Defines entities, measures, dimensions, and metrics for agent-consumable business logic. Eliminates semantic hallucination by providing machine-readable KPI definitions. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats." }, "crawler": { "type": "object", "description": "AWS Glue crawler configuration for auto-discovery of schema from S3 paths.", "properties": { "name": { "type": "string", "description": "Glue crawler name." }, "role": { "type": "string", "description": "IAM role ARN for the crawler." }, "schedule": { "type": "string", "description": "Cron schedule expression (e.g., 'cron(0 6 * * ? *)')." }, "classifiers": { "type": "array", "description": "Custom classifier names to apply.", "items": { "type": "string" } }, "schemaChangePolicy": { "type": "object", "description": "Policy for schema changes detected by the crawler.", "properties": { "updateBehavior": { "type": "string", "enum": [ "UPDATE_IN_DATABASE", "LOG" ] }, "deleteBehavior": { "type": "string", "enum": [ "LOG", "DELETE_FROM_DATABASE", "DEPRECATE_IN_DATABASE" ] } } } } }, "iceberg": { "type": "object", "description": "AWS Glue Iceberg table management configuration (snapshots, compaction).", "properties": { "writeFormat": { "type": "string", "enum": [ "parquet", "orc", "avro" ], "description": "Underlying file format for Iceberg data files." }, "snapshotRetention": { "type": "object", "description": "Snapshot retention policy.", "properties": { "maxSnapshotAgeMs": { "type": "integer", "description": "Maximum age of snapshots in milliseconds." }, "minSnapshotsToKeep": { "type": "integer", "description": "Minimum number of snapshots to retain." } } }, "compaction": { "type": "object", "description": "Iceberg compaction settings.", "properties": { "enabled": { "type": "boolean" }, "targetFileSizeMb": { "type": "integer", "description": "Target file size in MB after compaction." } } } } }, "mcp": { "$ref": "#/$defs/exposeMcp", "description": "NEW in v0.7.4: Declares this expose as agent-consumable over MCP, with per-expose overrides for the consumer-side output-port server (`fluid mcp output-port serve`). Its presence opts the expose into the Fluid MCP gateway, where `policy.agentPolicy.allowedModels` / `allowedUseCases` are enforced at every read." } } }, "exposeContract": { "type": "object", "additionalProperties": false, "properties": { "id": { "type": "string", "description": "NEW in v0.7.6: An explicit, stable contract id, used verbatim as the ODCS `id` and as the ODPS output port `contractId`. Without it the id is derived as `.`, so renaming the product or the expose silently changes the contract id and breaks every consumer reference to it, and a catalog that requires an id format of its own (a UUID, say) cannot be served at all. Each expose needs its own: two exposes sharing one id is refused by `fluid validate`.", "minLength": 1 }, "schemaSignature": { "type": "string", "pattern": "^sha256:[0-9a-fA-F]{64}$", "description": "Hash of canonical schema/OpenAPI. Used by contract-tests gates." }, "schemaPolicy": { "type": "string", "enum": [ "strict", "discover_and_freeze", "evolve_safe", "evolve_all" ], "default": "strict", "description": "NEW in v0.7.3: Schema evolution policy. strict = fail on any schema change; discover_and_freeze = capture from source on first run, fail thereafter; evolve_safe = accept additive changes; evolve_all = accept all changes (cast on type narrow, route failures to DLQ)." }, "schema": { "type": "array", "items": { "$ref": "#/$defs/column" }, "description": "Tabular schema (for table/view/file)." }, "openapiRef": { "type": "string", "format": "uri", "description": "OpenAPI document URL (for APIs)." }, "guarantees": { "type": "object", "additionalProperties": false, "properties": { "compatibility": { "type": "string", "enum": [ "none", "patch", "minor", "backward", "forward", "strict" ], "description": "Compatibility promise for non-breaking evolution." }, "evolution": { "type": "array", "items": { "type": "string", "enum": [ "additive_columns", "additive_fields", "additive_endpoints", "nullable_to_required_disallowed", "field_removal_with_notice", "type_widening_allowed" ] } } } }, "dq": { "$ref": "#/$defs/dqSpec", "description": "Enhanced data quality rules including anomaly detection." }, "quality": { "type": "array", "description": "Simplified SQL-expression data quality rules (alternate form of dq.rules for inline assertions).", "items": { "type": "object", "required": [ "rule", "expression", "severity" ], "properties": { "rule": { "type": "string", "description": "Rule identifier." }, "expression": { "type": "string", "description": "SQL or boolean expression that must evaluate to true." }, "severity": { "type": "string", "enum": [ "error", "warning", "info" ], "description": "Severity when the rule fails." } } } }, "tags": { "$ref": "#/$defs/tags", "description": "Contract-level tags for data quality and governance." }, "labels": { "$ref": "#/$defs/labels", "description": "Contract-level labels for automation and metadata." } }, "anyOf": [ { "required": [ "schema" ] }, { "required": [ "openapiRef" ] } ] }, "column": { "type": "object", "additionalProperties": false, "required": [ "name", "type" ], "properties": { "fields": { "type": "array", "minItems": 1, "items": {"$ref": "#/$defs/column"}, "description": "Typed child columns of a struct/row/record." }, "element": {"$ref": "#/$defs/nestedType"}, "key": {"$ref": "#/$defs/nestedType"}, "value": {"$ref": "#/$defs/nestedType"}, "name": { "type": "string", "minLength": 1 }, "type": { "description": "Column data type. Accepts a canonical FLUID/SQL type name (case-insensitive) or a parameterized form such as decimal(18,4) or varchar(255). The enum lists the canonical bare types; the pattern additionally permits case variants and parameter suffixes. Intentionally generous — it rejects obvious garbage (e.g. MYSTERY_TYPE) while admitting the cross-dialect superset.", "anyOf": [ { "type": "string", "enum": [ "string", "text", "varchar", "varchar2", "nvarchar", "char", "nchar", "character", "clob", "int", "integer", "int2", "int4", "int8", "int16", "int32", "int64", "tinyint", "smallint", "mediumint", "bigint", "long", "longint", "serial", "bigserial", "float", "float4", "float8", "float32", "float64", "double", "real", "decimal", "dec", "numeric", "number", "bignumeric", "money", "boolean", "bool", "bit", "date", "time", "datetime", "datetime2", "smalldatetime", "timestamp", "timestamptz", "timestamp_tz", "timestamp_ntz", "timestamp_ltz", "timestampntz", "interval", "year", "variant", "object", "array", "struct", "map", "record", "row", "json", "jsonb", "super", "binary", "varbinary", "bytes", "blob", "bytea", "raw", "geography", "geometry", "geom", "point", "uuid", "uniqueidentifier", "guid", "enum", "hll" ] }, { "type": "string", "pattern": "(?i)^\\s*(string|text|varchar|varchar2|nvarchar|char|nchar|character|clob|int|integer|int2|int4|int8|int16|int32|int64|tinyint|smallint|mediumint|bigint|long|longint|serial|bigserial|float|float4|float8|float32|float64|double|real|double\\s+precision|decimal|dec|numeric|number|bignumeric|money|boolean|bool|bit|date|time|datetime|datetime2|smalldatetime|timestamp|timestamptz|timestamp_tz|timestamp_ntz|timestamp_ltz|timestampntz|timestamp\\s+with(out)?\\s+time\\s+zone|interval|year|variant|object|array|struct|map|record|row|json|jsonb|super|binary|varbinary|bytes|blob|bytea|raw|geography|geometry|geom|point|uuid|uniqueidentifier|guid|enum|hll)\\s*(\\(\\s*[0-9A-Za-z_,\\s'\\\"-]*\\s*\\))?\\s*$" } ] }, "required": { "type": "boolean" }, "description": { "type": "string" }, "sensitivity": { "$ref": "#/$defs/sensitivityLevel", "description": "Field-level sensitivity classification for improved policy ergonomics." }, "semanticType": { "type": "string", "description": "Semantic meaning (e.g., 'email', 'phone', 'identifier')." }, "businessName": { "type": "string", "description": "Business-friendly name for this field." }, "businessDefinition": { "type": "string", "description": "Business definition and meaning of this field." }, "validationRules": { "type": "array", "description": "Field-level validation constraints.", "items": { "type": "object", "additionalProperties": false, "properties": { "type": { "type": "string", "enum": [ "range", "regex", "enum", "custom" ] }, "constraint": { "type": "string" }, "message": { "type": "string" } } } }, "quality": { "type": "array", "description": "NEW in v0.7.6: Column-level data quality rules, in ODCS's own shape — the array that is written straight into `schema[].properties[].quality[]` of the exported ODCS v3.1.0 contract, and the array an imported ODCS column's rules are read back into, inline on the column rather than in the importer's pass-through bucket: the ODPS importer merges a resolved port's `contract` block into its output-port stub and discards the sub-import's root `extensions`, so rules left in that bucket were dropped outright by a FLUID → ODPS → FLUID cycle. The ODCS exporter has always replayed this key verbatim and always read it back on import, but `$defs.column` is `additionalProperties: false` and did not define it, so a column written the way the exporter supports was REFUSED by `fluid validate`: the only way to get a hand-written quality rule into a published contract was to write an invalid contract. Optional and additive — a column without it exports exactly as it does today, with the rules forge derives from `required`, the primary-key tag and `validationRules`. Setting it REPLACES those derived rules for that column rather than adding to them, which is what makes an imported ODCS column round-trip byte-for-byte. The authority for an item's contents is ODCS v3.1.0 `$defs.DataQuality`, deliberately not re-declared here (a copy would drift from the vendored schema, and `fluid odcs validate` already checks an emitted document against it); the keys below are the common ones. Every rule should carry an `id`: it is optional in ODCS, but a DQ engine that labels its results by rule id rejects a contract whose rules have none.", "items": { "type": "object", "minProperties": 1, "properties": { "id": { "type": "string", "pattern": "^[A-Za-z0-9_-]+$", "description": "Stable rule identifier, unique within this column's rules. The pattern is ODCS `$defs.StableId`. Rules forge derives itself are named `-`." }, "type": { "type": "string", "enum": [ "text", "library", "sql", "custom" ], "description": "ODCS quality check type. text = prose; library = one of ODCS's predefined metrics; sql = a query whose result is compared; custom = a named engine's own check." }, "metric": { "type": "string", "enum": [ "nullValues", "missingValues", "invalidValues", "duplicateValues", "rowCount" ], "description": "type: library only. The five predefined metrics ODCS v3.1.0 defines; a name outside this list is not a library rule." }, "arguments": { "type": "object", "description": "type: library only. Extra arguments for the metric — `pattern` (a regex) or `validValues` (a list) for invalidValues, as DataHub's ODCS ingestion source reads them." }, "query": { "type": "string", "description": "type: sql only. The query whose result the comparison operator is applied to." }, "engine": { "type": "string", "description": "type: custom only. The engine that runs the check, e.g. soda, great-expectations, dbt." }, "implementation": { "description": "type: custom only. The check as that engine expresses it." }, "dimension": { "type": "string", "enum": [ "accuracy", "completeness", "conformity", "consistency", "coverage", "timeliness", "uniqueness" ], "description": "The data quality dimension the rule measures." }, "description": { "type": "string" }, "severity": { "type": "string", "description": "Severity of a failure. ODCS types this as a free string and documents info / warning / error by example." }, "unit": { "type": "string", "description": "Unit the comparison is in, usually rows (the default) or percent." } } } }, "tags": { "$ref": "#/$defs/tags", "description": "Field-level tags for categorization and data governance." }, "labels": { "$ref": "#/$defs/labels", "description": "Field-level labels for metadata and automation." } } }, "nestedType": { "type": "object", "additionalProperties": false, "required": ["type"], "properties": { "type": {"$ref": "#/$defs/column/properties/type"}, "required": {"type": "boolean"}, "fields": { "type": "array", "minItems": 1, "items": {"$ref": "#/$defs/column"} }, "element": {"$ref": "#/$defs/nestedType"}, "key": {"$ref": "#/$defs/nestedType"}, "value": {"$ref": "#/$defs/nestedType"} } }, "streamSource": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.7 (preview): The Kafka records a streaming build reads, described apart from the engine that reads them: one topic, how its values are encoded, the value's fields, an optional key, and Kafka record metadata read as columns. The value and key schemas are lists of columns, the same shape as an expose's contract.schema, so one fragment can describe a topic and a table that lands it. Brokers, consumer groups and registries are the engine's parameters, not part of the stream.", "required": ["kind", "topic", "format", "schema"], "properties": { "kind": { "const": "kafka", "description": "The streaming system. Only kafka in this preview." }, "topic": { "type": "string", "pattern": "^(?!\\.\\.?(?![\\s\\S]))[A-Za-z0-9._-]{1,249}(?![\\s\\S])", "description": "One Kafka topic, by name: Kafka's legal characters (not . or ..), no pattern and no list." }, "format": { "enum": ["json", "avro-confluent"], "description": "How record values are encoded. json: one JSON object per record; a field the schema does not declare is ignored, and a declared field that is missing reads as null. avro-confluent: Avro in the Confluent Schema Registry wire format. The declared schema is the reader schema, so the producers' subject must stay readable by it: FORWARD or FULL compatibility, transitive." }, "schema": { "type": "array", "minItems": 1, "items": {"$ref": "#/$defs/column"}, "description": "The record value's fields, as columns. Nested data uses fields (struct), element (array) and key/value (map)." }, "key": {"$ref": "#/$defs/streamKey"}, "metadata": { "type": "array", "minItems": 1, "items": {"$ref": "#/$defs/streamMetadataColumn"}, "description": "Kafka record metadata, each read as a column." } } }, "streamKey": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.7 (preview): The record key, read as columns next to the value's. A record without a key reads its key columns as null.", "required": ["format", "schema"], "properties": { "format": { "enum": ["raw", "json", "avro-confluent"], "description": "raw: the key's bytes as one column, of type string (UTF-8) or binary. json and avro-confluent: a key record whose fields are the schema's columns." }, "schema": { "type": "array", "minItems": 1, "items": {"$ref": "#/$defs/column"}, "description": "The key's fields, as columns: exactly one for raw." }, "prefix": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", "description": "Put in front of every key column's name, so a key field can share its name with a value field (a Debezium key repeats the value's id). A mapping names the prefixed column." } } }, "streamMetadataColumn": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.7 (preview): One piece of Kafka record metadata, read as a column. Its type is fixed by what it reads: topic string, partition int, offset long, timestamp timestamp_ltz(3), timestamp-type string, headers map and leader-epoch int. leader-epoch is null when the broker does not report it; the others are never null.", "required": ["name", "from"], "properties": { "name": { "type": "string", "minLength": 1, "description": "The column's name, unique among the stream's value, key and metadata columns." }, "from": { "enum": ["topic", "partition", "offset", "timestamp", "timestamp-type", "headers", "leader-epoch"], "description": "The metadata the column reads." }, "description": {"type": "string"} } }, "recordMapping": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.7 (preview): How the stream's records become the output's rows. An output column the mapping does not list, nested fields included, is read from the stream column or field at the same path, and keeps its type: so a stream landed as it is, flat or nested, needs no mapping. forge-cli writes the query from the mapping and checks every output column's type and nullability against it before anything deploys; no contract text reaches the job's SQL.", "properties": { "explode": {"$ref": "#/$defs/recordExplode"}, "columns": { "type": "array", "minItems": 1, "items": {"$ref": "#/$defs/recordMappingColumn"}, "description": "Output columns, or nested fields of them, that are read from another path, renamed or converted." } } }, "recordExplode": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or empty produces no row.", "required": ["path", "as"], "properties": { "path": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", "description": "The array: a stream column or a nested field, such as items or order.lines." }, "as": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", "description": "The element's name in mapping paths: item.sku for a field of a struct element, item for a scalar element." } } }, "recordMappingColumn": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.7 (preview): Where one output column, or one nested field of one, is read from.", "required": ["name", "from"], "properties": { "name": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*(?![\\s\\S])", "description": "The output column, or a field of a struct output column as a dotted path (customer.id). Mapping a nested field builds that struct from the paths given, so a nested output can be written from fields that are flat or elsewhere in the record." }, "from": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\['[^'\\\\\\u0000-\\u001f\\u007f]{1,256}'\\])*(?![\\s\\S])", "description": "The source: a stream column (value, key or metadata), a nested field (customer.address.country), a map entry (attributes['channel'], null when the key is absent) or the exploded element (item.sku)." }, "cast": { "$ref": "#/$defs/column/properties/type", "description": "Convert the source to this type: binary to string (UTF-8), or a widening (int to long, float to double, a decimal to a wider precision at the same scale, a timestamp to a finer precision)." }, "description": {"type": "string"} } }, "dqSpec": { "type": "object", "additionalProperties": false, "properties": { "rules": { "type": "array", "items": { "$ref": "#/$defs/dqRule" } }, "monitoring": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "window": { "$ref": "#/$defs/isoDuration" }, "owner": { "type": "string" } } }, "tags": { "$ref": "#/$defs/tags", "description": "Data quality specification tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Data quality specification labels." } } }, "dqRule": { "type": "object", "additionalProperties": false, "required": [ "id", "type", "severity" ], "properties": { "id": { "type": "string" }, "type": { "type": "string", "enum": [ "freshness", "completeness", "uniqueness", "valid_values", "accuracy", "schema", "anomaly_detection", "drift_detection" ] }, "selector": { "type": "string" }, "threshold": { "type": "number" }, "operator": { "type": "string", "enum": [ ">=", ">", "<=", "<", "==", "!=" ] }, "window": { "$ref": "#/$defs/isoDuration" }, "severity": { "type": "string", "enum": [ "info", "warn", "error", "critical" ] }, "description": { "type": "string" }, "tags": { "$ref": "#/$defs/tags", "description": "DQ rule tags for categorization and automation." }, "labels": { "$ref": "#/$defs/labels", "description": "DQ rule labels for metadata and routing." } } }, "exposeQoS": { "type": "object", "additionalProperties": false, "properties": { "availability": { "$ref": "#/$defs/availabilityPct" }, "freshnessSLO": { "$ref": "#/$defs/isoDuration" }, "dataLossSLO": { "type": "string", "description": "e.g., '0 rows'" }, "latencyP95": { "$ref": "#/$defs/isoDuration" }, "completenessTarget": { "type": "number", "minimum": 0, "maximum": 1, "description": "Target completeness ratio (0.0 to 1.0)." }, "errorBudget": { "type": "number", "minimum": 0, "maximum": 1, "description": "Acceptable error rate (0.0 to 1.0)." }, "tags": { "$ref": "#/$defs/tags", "description": "QoS-level tags for SLA management." }, "labels": { "$ref": "#/$defs/labels", "description": "QoS-level labels for automation and reporting." } } }, "exposePolicy": { "type": "object", "additionalProperties": false, "properties": { "authn": { "type": "string", "enum": [ "oidc", "oauth2", "api_key", "none", "custom", "iam", "jwt" ] }, "authz": { "type": "object", "additionalProperties": false, "properties": { "readers": { "type": "array", "items": { "type": "string" } }, "writers": { "type": "array", "items": { "type": "string" } }, "columnRestrictions": { "type": "array", "description": "Column-level access control. A column named in any restriction is restricted. 'deny': the principal may not read the columns. 'allow': the columns are readable only by the principals an allow names. A deny beats an allow, and a restriction never grants access: the readers are the expose's readers. Principals are logical and resolve through binding.principals. NEW in v0.7.6 (the field itself is older): `fluid apply` enforces it. GCP BigQuery: a Data Catalog taxonomy per product and dataset with fine-grained access control, a policy tag per set of restricted columns that share their readers, attached through the table schema's policyTags, and roles/datacatalog.categoryFineGrainedReader for exactly the expose's readers allowed to read them: the accessPolicy read grantees and the expose's own policy.authz.readers (a denied principal gets an access error on the columns). A restriction on an expose with no reader on the binding is refused, as it would lock the columns for everyone. A restriction's tags and labels are written into the policy tag's description; AWS has no place for them. AWS: each governance.lakeFormation grant with SELECT excludes the restricted columns its principal may not read (table_with_columns.excluded_column_names); a grant's hand-written excludedColumns must agree, and a binding with no Lake Formation grants is refused. `fluid verify` checks the policy tags and their readers on GCP, and the principals' Lake Formation permissions on AWS. NEW in v0.7.6: access: mask lets the principal read the columns masked by the platform (see mask): a deny beats a mask, and a mask beats an allow for the principal it names.", "items": { "type": "object", "additionalProperties": false, "properties": { "principal": { "type": "string", "description": "Logical principal the restriction applies to, written as in accessPolicy (e.g. group:analysts@company.example); binding.principals maps it per cloud." }, "columns": { "type": "array", "items": { "type": "string" }, "description": "Columns of this expose's schema." }, "access": { "type": "string", "enum": [ "allow", "deny", "mask" ], "description": "deny: the principal may not read the columns. allow: only the principals an allow names may read them. mask: the principal reads the columns masked by the platform, by the rule in mask, and never their raw values (NEW in v0.7.6)." }, "tags": { "$ref": "#/$defs/tags", "description": "Access control rule tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Access control rule labels." }, "mask": { "type": "string", "enum": [ "last_four", "first_four", "nullify" ], "description": "The masking rule an access: mask restriction applies, required with it and refused without it. last_four: 'XXXXX' and the value's last four characters; first_four: its first four characters and 'XXXXX' (a value of four characters or fewer is hashed instead, by each platform); nullify: NULL. GCP BigQuery: a data policy (DATA_MASKING_POLICY, LAST_FOUR_CHARACTERS, FIRST_FOUR_CHARACTERS or ALWAYS_NULL) on the columns' policy tag, and roles/bigquerydatapolicy.maskedReader for the masked principals, who read the same column masked. AWS Glue: Lake Formation cannot mask, so the masked principals' grant on the table excludes the columns and they read a protected Glue Data Catalog view (SECURITY DEFINER, binding.governance.lakeFormation.maskedViews) that returns them masked by the same rule (NEW in v0.7.6)." } }, "allOf": [ { "if": { "properties": { "access": { "const": "mask" } }, "required": [ "access" ] }, "then": { "required": [ "mask" ] } }, { "if": { "required": [ "mask" ] }, "then": { "properties": { "access": { "const": "mask" } }, "required": [ "access" ] } } ] } }, "tags": { "$ref": "#/$defs/tags", "description": "Authorization policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Authorization policy labels." }, "rowFilters": { "type": "array", "description": "Row-level access control (NEW in v0.7.6). Each filter lets its principal read only the rows its where predicate selects; a reader no filter names reads every row, and a principal may have one filter per expose. The predicate is one SQL boolean expression over this expose's columns, with no statement separator or comment. Principals are logical and resolve through binding.principals. GCP BigQuery: a row access policy (filter_predicate, grantees) per filter, plus one that selects every row for every other reader and writer, since a principal no row access policy names reads no row at all. AWS Glue: a Lake Formation data cells filter (row_filter and the principal's excluded columns), and the principal's SELECT is granted on the filter instead of the table.", "items": { "type": "object", "additionalProperties": false, "required": [ "principal", "where" ], "properties": { "principal": { "type": "string", "description": "Logical principal the filter applies to, written as in accessPolicy; binding.principals maps it per cloud." }, "name": { "type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$", "description": "The filter's name on the platform (the row access policy or data cells filter); derived from the principal when omitted." }, "where": { "type": "string", "minLength": 1, "description": "One SQL boolean expression over the expose's columns, e.g. consent_data_analytics = true AND is_suppressed = false." }, "description": { "type": "string" } } } } } }, "privacy": { "type": "object", "additionalProperties": false, "properties": { "masking": { "type": "array", "description": "Columns to treat before they land. The DuckDB acquisition runner rewrites each named column inside the write, so the file, S3 object or BigQuery load and the DLQ receive the treated value; quality gates judge the source value first. A treated column lands as a string, so declare it with a string type (string, or VARCHAR under a schemaPolicy that compares source type names). A rule the build cannot apply (an unset salt or key, k_anonymity, a column the stream lacks, a non-string declared type, the incremental cursor) refuses the build rather than landing the column untreated. fluid verify checks every non-null value of a masked column for its strategy's shape, on a local file and, through Athena, on an S3+Glue table. The MCP output-port gateway drops masked columns from what it serves.", "items": { "type": "object", "additionalProperties": false, "required": [ "column", "strategy" ], "properties": { "column": { "type": "string" }, "strategy": { "type": "string", "description": "hash: lowercase hex SHA-256 of salt || value, deterministic so joins work (64 hex characters). mask: every character but the first params.keepFirst and the last params.keepLast replaced by '*', length preserved. tokenize: HMAC-SHA256 token, 32 hex characters, the token the tokenize_pii preLand hook makes. encrypt: AES-GCM, written as 'aesgcm:v1:' + base64url(nonce || ciphertext || tag) with the column name as associated data; reversible with the key, not deterministic. k_anonymity: accepted by the schema but refused at landing: it is a property of a whole table over its quasi-identifiers, not of one value.", "enum": [ "mask", "hash", "tokenize", "encrypt", "k_anonymity" ] }, "params": { "type": "object", "additionalProperties": true, "description": "Strategy parameters. The runner accepts exactly these keys and refuses any other, including a literal salt or key: secrets come only from the environment, never the contract, which is published with the plan, the bundle and the Glue table's fluid_contract parameter. Contracts on 0.7.5 and earlier may use the same keys; their schemas leave params untyped, and the runner checks them at build time.", "properties": { "saltEnv": { "type": "string", "pattern": "^[A-Z_][A-Z0-9_]{0,127}$", "default": "FLUID_PII_HASH_SECRET", "description": "NEW in v0.7.6: hash only. The environment variable holding the salt, at least 16 bytes. Unset or empty, the build is refused: an unsalted SHA-256 of a phone number is reversed by enumerating the numbering plan." }, "keyEnv": { "type": "string", "pattern": "^[A-Z_][A-Z0-9_]{0,127}$", "description": "NEW in v0.7.6: tokenize and encrypt. The environment variable holding the key. tokenize defaults to FLUID_PII_TOKENIZATION_KEY and needs at least 32 bytes; encrypt defaults to FLUID_PII_ENCRYPTION_SECRET_KEY and needs the base64 of a 16, 24 or 32-byte AES key (openssl rand -base64 32). Unset or empty, the build is refused." }, "keepFirst": { "type": "integer", "minimum": 0, "maximum": 64, "default": 0, "description": "NEW in v0.7.6: mask only. Leading characters left as they are." }, "keepLast": { "type": "integer", "minimum": 0, "maximum": 64, "default": 4, "description": "NEW in v0.7.6: mask only. Trailing characters left as they are, so +46701234567 lands as ********4567. A value no longer than keepFirst + keepLast is masked entirely." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Privacy masking rule tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Privacy masking rule labels." } } } }, "rowLevelPolicy": { "type": "object", "additionalProperties": false, "required": [ "expression" ], "properties": { "expression": { "type": "string", "description": "Provider-specific predicate" }, "tags": { "$ref": "#/$defs/tags", "description": "Row-level policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Row-level policy labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Privacy policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Privacy policy labels." } } }, "classification": { "type": "string", "enum": [ "Public", "Internal", "Confidential", "Restricted" ] }, "agentPolicy": { "$ref": "#/$defs/agentPolicy", "description": "NEW in v0.7.1: AI/LLM usage governance - controls which AI models can consume this data and for what purposes." }, "tags": { "$ref": "#/$defs/tags", "description": "Policy-level tags for governance." }, "labels": { "$ref": "#/$defs/labels", "description": "Policy-level labels for automation." } } }, "binding": { "type": "object", "additionalProperties": false, "required": [ "platform", "format", "location" ], "properties": { "platform": { "type": "string", "enum": [ "gcp", "aws", "azure", "snowflake", "databricks", "kafka", "confluent", "local", "kubernetes", "postgres", "pgvector", "other" ], "description": "Target platform. NEW in v0.7.5: 'pgvector' for the vector/embeddings output port (pgvector-on-PostgreSQL RAG target — emits the embeddings table + ANN index DDL for the ai-embeddable columns). NEW in v0.7.5: 'confluent' for the Confluent Cloud Tableflow managed Kafka→Iceberg emitter. NEW in v0.7.4: 'postgres' for the MCP output-port PostgreSQL driver (AWS Athena uses platform 'aws')." }, "format": { "type": "string", "enum": [ "bigquery_table", "snowflake_table", "snowflake_view", "gcs_file", "s3_file", "http_api", "grpc_api", "pubsub_topic", "kafka_topic", "delta_table", "iceberg", "parquet", "csv", "json", "redshift_table", "redshift_serverless", "redshift_external_schema", "postgres_table", "athena_table", "glue_table", "pgvector_table", "other" ], "description": "Binding format. NEW in v0.7.5: 'pgvector_table' — a pgvector embeddings table target for the vector output port. NEW in v0.7.4: 'postgres_table' (PostgreSQL driver), 'athena_table' / 'glue_table' (AWS Athena driver) for the MCP output port." }, "icebergConfig": { "type": "object", "description": "Apache Iceberg table format configuration", "additionalProperties": false, "properties": { "writeVersion": { "type": "integer", "enum": [ 1, 2 ], "description": "Iceberg table format version" }, "fileFormat": { "type": "string", "enum": [ "parquet", "orc", "avro" ], "description": "Underlying file format for data files" }, "partitionSpec": { "type": "array", "description": "Partition transformation specifications", "items": { "type": "object", "required": [ "sourceColumn", "transform" ], "properties": { "name": { "type": "string", "description": "Partition field name" }, "sourceColumn": { "type": "string", "description": "Source column for partitioning" }, "transform": { "type": "string", "enum": [ "identity", "year", "month", "day", "hour", "bucket", "truncate" ], "description": "Partition transform function" }, "numBuckets": { "type": "integer", "description": "Number of buckets (for bucket transform)" }, "width": { "type": "integer", "description": "Truncate width (for truncate transform)" } } } }, "sortOrder": { "type": "array", "description": "Sort order specifications for data files", "items": { "type": "object", "required": [ "column" ], "properties": { "column": { "type": "string", "description": "Column to sort by" }, "direction": { "type": "string", "enum": [ "asc", "desc" ], "description": "Sort direction" }, "nullOrder": { "type": "string", "enum": [ "nulls-first", "nulls-last" ], "description": "NULL value ordering" } } } }, "properties": { "type": "object", "description": "Iceberg table properties", "additionalProperties": { "type": "string" } } } }, "vectorConfig": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.5: vector / embeddings output-port configuration (platform 'pgvector', format 'pgvector_table'). Consumed by fluid_build.output_ports.vector to emit the pgvector embeddings-table + ANN-index DDL and a RAG provenance manifest for the schema's ai-embeddable columns.", "required": [ "dimensions" ], "properties": { "dimensions": { "type": "integer", "minimum": 1, "maximum": 16000, "description": "Embedding vector width (e.g. 1536 for OpenAI text-embedding-3-small, 1024 for Cohere embed-english-light). Maps to the pgvector column type vector()." }, "embeddingModel": { "type": "string", "description": "Provenance: the embedding model that produces the vectors (e.g. 'text-embedding-3-small'). Recorded in the RAG manifest and the embeddings-table's embedding_model column so a retriever can verify model parity." }, "vectorType": { "type": "string", "enum": [ "vector", "halfvec" ], "default": "vector", "description": "pgvector column type: 'vector' (float32, up to 2000 dims) or 'halfvec' (float16, half the storage, up to 4000 dims)." }, "indexType": { "type": "string", "enum": [ "hnsw", "ivfflat", "none" ], "default": "hnsw", "description": "ANN index built on the embedding column. 'hnsw' (higher recall / lower query latency, buildable before data lands), 'ivfflat' (lower build cost, needs data first), or 'none' (no index — exact search only)." }, "distanceMetric": { "type": "string", "enum": [ "cosine", "l2", "inner_product", "l1" ], "default": "cosine", "description": "Distance function the index optimises for → pgvector operator class: cosine→vector_cosine_ops, l2→vector_l2_ops, inner_product→vector_ip_ops, l1→vector_l1_ops." }, "table": { "type": "string", "description": "Target embeddings table name. Defaults to '_embeddings' when omitted." }, "sourceKeyColumn": { "type": "string", "description": "Provenance: the source primary-key column that ties each embedding row back to its origin row (written to the embeddings-table's source_key column). Enables update/delete propagation from the source product to the vector store." }, "hnsw": { "type": "object", "additionalProperties": false, "description": "HNSW index tuning (used when indexType is 'hnsw').", "properties": { "m": { "type": "integer", "minimum": 1, "description": "Max bi-directional links per node (pgvector default 16)." }, "efConstruction": { "type": "integer", "minimum": 1, "description": "Build-time dynamic candidate-list size (pgvector default 64)." } } }, "ivfflat": { "type": "object", "additionalProperties": false, "description": "IVFFlat index tuning (used when indexType is 'ivfflat').", "properties": { "lists": { "type": "integer", "minimum": 1, "description": "Number of inverted lists (pgvector guidance: rows/1000 for < 1M rows)." } } } } }, "location": { "$ref": "#/$defs/bindingLocation" }, "tags": { "$ref": "#/$defs/tags", "description": "Binding configuration tags for infrastructure." }, "labels": { "$ref": "#/$defs/labels", "description": "Binding configuration labels for automation." }, "properties": { "type": "object", "description": "Provider-specific binding properties (e.g., cluster_by, comment for Snowflake).", "additionalProperties": true }, "governance": { "$ref": "#/$defs/bindingGovernance", "description": "Per-resource governance: registerLocation, principal grants, tag associations, row/column filters. Account-wide governance (admins, tag definitions) lives at the contract top-level governance block." }, "packaging": { "$ref": "#/$defs/packaging", "description": "NEW in v0.7.6: Per-exposure packaging override — key-wise precedence over the contract-wide top-level `packaging` block (binding.packaging > packaging > absent-legacy)." }, "encryption": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: Encryption at rest for an AWS S3 binding or a GCP BigQuery binding. Absent: nothing is emitted, and the cloud's default applies: SSE-S3 for new S3 objects, Google-managed keys for BigQuery.", "properties": { "kms": { "type": "string", "default": "product", "anyOf": [ { "enum": [ "product", "none" ] }, { "pattern": "^alias/[A-Za-z0-9/_-]{1,250}$" }, { "pattern": "^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:(key|alias)/[A-Za-z0-9/_-]{1,250}$" }, { "pattern": "^projects/[a-z0-9.:-]{1,100}/locations/[a-z0-9-]{1,63}/keyRings/[A-Za-z0-9_-]{1,63}/cryptoKeys/[A-Za-z0-9_-]{1,63}$" } ], "description": "NEW in v0.7.6: The KMS key the bucket's objects are encrypted with (SSE-KMS with an S3 Bucket Key). 'product' (the default): `fluid apply` creates a customer managed key for each bucket this product owns (aws_kms_key, rotation on, alias alias/fluid//) and makes it the bucket's default encryption (aws_s3_bucket_server_side_encryption_configuration). Its key policy lets the account's IAM policies decide who may use it (the KMS default statement for the account root) and, when the binding carries governance.lakeFormation, lets the Lake Formation service-linked role AWSServiceRoleForLakeFormationDataAccess encrypt and decrypt, matched by aws:PrincipalArn so the role need not exist yet. A principal querying through Athena with credentials Lake Formation vends then needs no KMS permission of its own; a grantee in another account that the bucket policy lets read the objects directly (governance.lakeFormation.bucketPolicy) is also allowed kms:Decrypt, through S3 only. The writer (the identity the build runs as) needs kms:GenerateDataKey and kms:Decrypt from IAM. `tofu destroy` deletes the alias and schedules the key for deletion after 7 days, the minimum KMS allows; kms:CancelKeyDeletion recovers it until then. Dropping the block, or naming another key, removes the product key while the bucket keeps the objects encrypted under it, which become unreadable once the deletion window ends unless they are rewritten or the deletion is cancelled; `fluid apply` refuses that plan without --allow-data-loss. On a shared (pool) bucket the bucket's default encryption is the pool owner's, so 'product' is refused there. 'alias/' or a key or alias ARN: an existing key, looked up at plan time (data aws_kms_key, so the identity running `fluid apply` needs kms:DescribeKey on it) and made the default encryption of a bucket this product owns by its key ARN, since S3 resolves an alias in the account of whoever writes; the plan fails unless the key is Enabled and a symmetric encryption key (SYMMETRIC_DEFAULT). On a pool bucket it is only checked. Its key policy is its owner's, and must let the Lake Formation role use it when the location is registered: an AWS managed key cannot be used with that role, so alias/aws/s3 is refused with registerLocation, and the plan fails for any key that is not customer managed. 'none': nothing is emitted. `fluid verify` checks that the key is Enabled and that the objects under the binding's prefix are SSE-KMS with it. On a gcp binding (fluid-schema 0.7.6, BigQuery tables): 'product' creates a Cloud KMS key ring and crypto key for each dataset this product writes (google_kms_key_ring fluid-- and google_kms_crypto_key 'bigquery', in the dataset's location, EU and US mapping to the europe and us key locations, rotated every 90 days), grants the project's BigQuery service agent roles/cloudkms.cryptoKeyEncrypterDecrypter on it, and makes it the dataset's default_encryption_configuration and the table's encryption_configuration. The project must have the Cloud KMS API (cloudkms.googleapis.com) enabled. A key ring and key cannot be deleted on GCP: `tofu destroy` removes them from state and schedules the key's versions for destruction (30 days by default), and a later apply adopts the same names. 'projects/

/locations//keyRings//cryptoKeys/': an existing key in the dataset's location, used as given; its owner grants the BigQuery service agent. 'none': Google-managed keys. An alias/... or arn:... value is refused on a gcp binding, and a Cloud KMS name on an aws binding. Adding a key to a table that exists replaces the table (BigQuery cannot re-key it in place), so `fluid apply` refuses that plan without --allow-data-loss; the next build lands the data again. `fluid verify` checks kmsKeyName on the dataset and the table." } } }, "principals": { "type": "object", "description": "NEW in v0.7.6: Maps the contract's LOGICAL principals (accessPolicy.grants[].principal and exposes[].policy.authz.columnRestrictions[].principal, keyed exactly as the contract writes them) to the identities they stand for on this binding's platform, so the base contract stays cloud-neutral and each environment's overlay binds it. On gcp an identity is an IAM member (user:, group:, serviceAccount: or domain:); on aws an IAM principal ARN. A value is one identity or a list; [] says the principal has no identity on this cloud, and nothing is granted to it there. When the block is present, every principal the contract names for this expose must be a key: an unmapped one is refused, never emitted as written. Without the block the principals are used as written, except that a GCP principal in a reserved top-level domain (.example, .test, .invalid, .localhost) is refused as a placeholder. Modelled on ODCS v3's roles[] bound per server (servers[].roles) and on dbt grants resolved per target.", "propertyNames": { "minLength": 1 }, "additionalProperties": { "anyOf": [ { "type": "string", "minLength": 1 }, { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true } ] } }, "serialization": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.6: How message VALUES (and keys) are serialized on the wire. A Kafka binding today declares only `format: kafka_topic`, which tells a consumer nothing about whether the values are Avro or JSON — the one thing it has to know to read the topic at all. The first four values are the message formats ODCS v3.1.0 documents for its own Kafka server `format` field; the two `-confluent` values name those same wire formats carried through a Confluent schema registry, spelled as Apache Flink spells them.", "properties": { "value": { "type": "string", "enum": [ "json", "avro", "protobuf", "xml", "avro-confluent", "json-confluent" ], "description": "How message values are serialized." }, "key": { "type": "string", "enum": [ "json", "avro", "protobuf", "xml", "avro-confluent", "json-confluent" ], "description": "How message keys are serialized, for a topic whose keys are not opaque bytes." } } } }, "allOf": [ { "if": { "properties": { "platform": { "const": "gcp" } }, "required": [ "platform" ] }, "then": { "properties": { "encryption": { "properties": { "kms": { "not": { "pattern": "^(alias/|arn:)" } } } }, "principals": { "additionalProperties": { "anyOf": [ { "type": "string", "pattern": "^(user|group|serviceAccount|domain):.+$" }, { "type": "array", "items": { "type": "string", "pattern": "^(user|group|serviceAccount|domain):.+$" } } ] } } } } }, { "if": { "properties": { "platform": { "const": "aws" } }, "required": [ "platform" ] }, "then": { "properties": { "encryption": { "properties": { "kms": { "not": { "pattern": "^projects/" } } } }, "principals": { "additionalProperties": { "anyOf": [ { "type": "string", "pattern": "^arn:aws[a-z0-9-]*:iam::" }, { "type": "array", "items": { "type": "string", "pattern": "^arn:aws[a-z0-9-]*:iam::" } } ] } } } } } ] }, "bindingLocation": { "type": "object", "description": "Provider-native addressing with tightened structure for better interop.", "additionalProperties": false, "properties": { "account": { "type": "string", "description": "Cloud account ID" }, "project": { "type": "string", "description": "GCP project ID" }, "dataset": { "type": "string", "description": "BigQuery dataset or database name" }, "database": { "type": "string", "description": "Database name" }, "schema": { "type": "string", "description": "Schema name" }, "table": { "type": "string", "description": "Table name" }, "bucket": { "type": "string", "description": "Storage bucket name" }, "path": { "type": "string", "description": "File or object path" }, "gateway": { "type": "string", "description": "API gateway endpoint" }, "host": { "type": "string", "description": "NEW in v0.7.6: Host or endpoint; for a Kafka binding, the bootstrap servers. The ODCS server mapper already reads `host` and falls back to `account`, so until now the only valid way to set the bootstrap servers was a field named for something else — and with neither set, the exported ODCS server is the placeholder `${KAFKA_BOOTSTRAP_SERVERS}`. Usually set per environment in an overlay rather than in the base contract." }, "baseUrl": { "type": "string", "format": "uri", "description": "Base URL for API endpoints" }, "topic": { "type": "string", "description": "Pub/Sub or Kafka topic name" }, "subscription": { "type": "string", "description": "Pub/Sub subscription name" }, "region": { "type": "string", "description": "Cloud region" }, "zone": { "type": "string", "description": "Cloud zone" }, "environment_id": { "type": "string", "description": "NEW in v0.7.5: Confluent Cloud environment id (env-xxxxx) for the Tableflow emitter." }, "kafka_cluster_id": { "type": "string", "description": "NEW in v0.7.5: Confluent Cloud Kafka cluster id (lkc-xxxxx) the Tableflow topic belongs to." }, "stream": { "type": "string", "description": "NEW in v0.7.5: AWS Kinesis Data Stream name — maps to aws_kinesis_stream in the streaming IaC emitter." }, "namespace": { "type": "string", "description": "NEW in v0.7.5: Amazon Redshift Serverless namespace name — maps to aws_redshiftserverless_namespace." }, "workgroup": { "type": "string", "description": "NEW in v0.7.5: Amazon Redshift Serverless workgroup name — maps to aws_redshiftserverless_workgroup." }, "iam_role_arn": { "type": "string", "description": "NEW in v0.7.5: ARN of the IAM role attached to the Redshift Serverless namespace / used by the redshift-data CREATE EXTERNAL SCHEMA bridge." }, "external_schema": { "type": "string", "description": "NEW in v0.7.5: Redshift external (Spectrum) schema name created over a Glue database via the redshift-data CREATE EXTERNAL SCHEMA bridge." }, "glue_database": { "type": "string", "description": "NEW in v0.7.5: Glue Data Catalog database that backs the Redshift external schema." }, "confluent_role_arn": { "type": "string", "description": "NEW in v0.7.5: ARN of the pre-created AWS IAM role Confluent Tableflow assumes (byob_aws). Its trust policy is updated post-apply with the provider integration's external_id (two-phase IAM)." }, "catalog": { "type": "string", "description": "NEW in v0.7.5: The Iceberg catalog that owns this table: glue, rest, lakekeeper, polaris, unity, nessie, bigquery, hive, jdbc, hadoop, dynamodb or snowflake-managed. Spellings fold case and '-'/'_', so iceberg_rest == rest and snowflake == snowflake-managed; fluid validate refuses any other value on an Iceberg expose. When absent: glue on platform aws, snowflake-managed on snowflake, rest elsewhere. The REST catalogs (rest, lakekeeper, polaris, unity) need location.uri plus location.warehouse as the catalog's warehouse NAME, not an object-store URI; Lakekeeper serves its Iceberg REST API under /catalog (uri: http://lakekeeper:8181/catalog). On platform aws, a kind other than glue gets no Glue table and no Glue grant: that catalog creates and governs the table. Every emitter (streaming sinks, dbt catalogs.yml, the IaC, policy compile) reads one table, providers/_iceberg_catalog.py." }, "warehouse": { "type": "string", "description": "NEW in v0.7.5: Iceberg warehouse — the canonical Iceberg REST 'warehouse' property (maps to iceberg.catalog.warehouse): an object-store URI (s3://|gs://|abfss://) for object-store catalogs, or the catalog/warehouse name for a REST catalog." }, "uri": { "type": "string", "description": "NEW in v0.7.5: REST Iceberg catalog endpoint URL — the canonical Iceberg REST 'uri' property (maps to iceberg.catalog.uri)." }, "partitionBy": { "type": "array", "items": { "type": "string" }, "description": "NEW in v0.7.5: Iceberg partition columns — a FLUID abstraction over the Iceberg PARTITIONED BY / PartitionSpec (maps to iceberg.tables.default-partition-by). On a gcp BigQuery table with exposes[].lifecycle.expire: true (fluid-schema 0.7.6), one DATE, TIMESTAMP or DATETIME column to partition by (time_partitioning.field); without it the table is partitioned by ingestion time. With a column, retention counts from the date in that column, not from landing: BigQuery deletes a partition `retention` after its date, so a backfill of older rows is deleted at once. Leave it out for the S3 rule's semantics (age since written)." } } }, "observability": { "type": "object", "additionalProperties": false, "properties": { "metrics": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "name", "source" ], "properties": { "name": { "type": "string" }, "source": { "type": "string" }, "sli": { "type": "string", "description": "Human/machine-readable SLI check description" } } } }, "onBreach": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "slack", "email", "webhook", "pagerduty", "custom" ] }, "channel": { "type": "string" }, "target": { "type": "string" }, "url": { "type": "string", "format": "uri" }, "severity": { "type": "string", "enum": [ "info", "warning", "critical" ] }, "tags": { "$ref": "#/$defs/tags", "description": "Alert configuration tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Alert configuration labels." } } } }, "defaultSLIs": { "$ref": "#/$defs/defaultSLIs", "description": "Default SLI trio (freshness, completeness, latency) by kind." }, "alert": { "type": "object", "additionalProperties": false, "description": "Alert dispatcher config consumed by the acquisition Alerter (DLQ overflow, schema-fingerprint changes, anomaly signals).", "properties": { "channels": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "kind" ], "properties": { "kind": { "type": "string", "enum": [ "log", "file", "webhook" ] }, "path": { "type": "string", "description": "For kind=file: NDJSON sink path (env-vars expanded)." }, "url": { "type": "string", "format": "uri", "description": "For kind=webhook: Slack-compatible incoming webhook URL. Validated against the SSRF guard at construction time." } } } } } }, "tags": { "$ref": "#/$defs/tags", "description": "Observability configuration tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Observability configuration labels." } } }, "defaultSLIs": { "type": "object", "description": "Default SLI trio that CLI can auto-generate based on expose kind.", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true, "description": "Enable auto-generation of default SLIs." }, "freshness": { "type": "object", "description": "Default freshness SLI configuration.", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "threshold": { "$ref": "#/$defs/isoDuration", "description": "Freshness threshold (auto-set by kind: table=PT6H, stream=PT1M, api=PT1S)" }, "severity": { "type": "string", "enum": [ "info", "warning", "critical" ], "default": "warning" }, "tags": { "$ref": "#/$defs/tags", "description": "Freshness SLI tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Freshness SLI labels." } } }, "completeness": { "type": "object", "description": "Default completeness SLI configuration.", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "threshold": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.95, "description": "Completeness threshold (auto-set by kind: table=0.95, stream=0.99)" }, "severity": { "type": "string", "enum": [ "info", "warning", "critical" ], "default": "warning" }, "tags": { "$ref": "#/$defs/tags", "description": "Completeness SLI tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Completeness SLI labels." } } }, "latency": { "type": "object", "description": "Default latency SLI configuration.", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "threshold": { "$ref": "#/$defs/isoDuration", "description": "P95 latency threshold (auto-set by kind: api=PT200MS, table=PT5S, stream=PT100MS)" }, "percentile": { "type": "number", "minimum": 0, "maximum": 100, "default": 95, "description": "Latency percentile to measure." }, "severity": { "type": "string", "enum": [ "info", "warning", "critical" ], "default": "warning" }, "tags": { "$ref": "#/$defs/tags", "description": "Latency SLI tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Latency SLI labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Default SLI configuration tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Default SLI configuration labels." } } }, "lifecycle": { "type": "object", "additionalProperties": false, "properties": { "state": { "$ref": "#/$defs/lifecycleState", "description": "Unified lifecycle state vocabulary." }, "retention": { "$ref": "#/$defs/isoDuration" }, "deprecationPolicy": { "type": "object", "additionalProperties": false, "properties": { "noticePeriod": { "$ref": "#/$defs/isoDuration" }, "contact": { "type": "string" }, "replacement": { "type": "string", "description": "Reference to replacement data product or expose." }, "tags": { "$ref": "#/$defs/tags", "description": "Deprecation policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Deprecation policy labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Lifecycle tags for automation." }, "labels": { "$ref": "#/$defs/labels", "description": "Lifecycle labels for governance." } } }, "exposeLifecycle": { "type": "object", "additionalProperties": false, "properties": { "state": { "$ref": "#/$defs/lifecycleState", "description": "Unified lifecycle state vocabulary." }, "retention": { "$ref": "#/$defs/isoDuration", "description": "How long this expose's data is kept (ISO-8601 duration, e.g. P30D, P7Y). A declaration unless `expire` is true. With `expire: true` on an AWS S3 binding, `fluid apply` expires the objects under the binding's prefix this long after they are written; on a GCP BigQuery table it deletes each daily partition this long after its day ends. Years and months are counted as 365 and 30 days, and a part of a day rounds up to a whole one." }, "expire": { "type": "boolean", "default": false, "description": "NEW in v0.7.6: Delete stored data once it is older than `retention`. Default false, so a contract that declares a retention period does not start deleting data when forge-cli is upgraded. Honoured by the AWS emitter on an S3 binding and by the GCP emitter on a BigQuery table; on a gcp binding that is not a BigQuery table (GCS, Pub/Sub, Iceberg storage) it is refused rather than dropped. AWS: `fluid apply` writes one rule per expose into the bucket's aws_s3_bucket_lifecycle_configuration, filtered to the binding's prefix (location.path, else {database}/{table}/; a binding with neither is refused, so no rule ever expires a whole bucket). The rule expires the current version `retention` after it is written, removes a noncurrent version (on a versioning-enabled bucket) one day after it stops being current, and aborts an incomplete multipart upload after 7 days or the retention period if shorter. The lifecycle configuration is AUTHORITATIVE for the whole bucket: S3 keeps one per bucket, so rules that `fluid apply` did not write are replaced. It is therefore written only for a bucket this product owns; on a shared (pool) bucket (packaging) nothing is written, the pool's owner must hold the rule, and `fluid verify` checks that an enabled rule covering the prefix expires objects after exactly this period. Objects already older than the period when the rule is first applied are expired on S3's next lifecycle run, typically within a day. Removing `expire` later deletes the lifecycle configuration, which `fluid apply` refuses without --allow-data-loss (it refuses every plan that removes a resource holding data or policy; removing an access grant or a policy tag is a revocation and is not gated); `tofu destroy` deletes it. GCP (fluid-schema 0.7.6): the BigQuery table is partitioned by day (time_partitioning type DAY, on binding.location.partitionBy when it names one date or timestamp column, else by ingestion time) with expiration_ms = `retention` in days, so BigQuery deletes each partition that long after its day ends. BigQuery counts from the partition's date, not from when its rows were written: by ingestion time (no partitionBy, the default and the S3 rule's semantics) no row is deleted sooner than `retention` after it landed; with partitionBy, retention is the age of the date in that column, so a backfill of rows whose date is already older than `retention` lands in expired partitions and is deleted at once. It is never a whole-table expiration, which would delete the product. BigQuery cannot partition a table that exists, so adding `expire` to a live table replaces it: `fluid apply` refuses that plan without --allow-data-loss, and the next build lands the data again. A later change of `retention` is an in-place update. Removing `expire` from a live table removes the partitioning's replacement trigger, which `fluid apply` also refuses without --allow-data-loss; BigQuery cannot un-partition a table, so to keep data longer set a longer `retention` instead. `fluid verify` checks the live table's partition expiration." }, "deprecationPolicy": { "type": "object", "additionalProperties": false, "properties": { "noticePeriod": { "$ref": "#/$defs/isoDuration" }, "contact": { "type": "string" }, "replacement": { "type": "string", "description": "Reference to replacement data product or expose." }, "tags": { "$ref": "#/$defs/tags", "description": "Deprecation policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Deprecation policy labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Lifecycle tags for automation." }, "labels": { "$ref": "#/$defs/labels", "description": "Lifecycle labels for governance." } }, "description": "NEW in v0.7.6: An expose's lifecycle. The same fields as the contract-root lifecycle, plus `expire`, which is only meaningful where there is storage to expire." }, "docs": { "type": "object", "additionalProperties": false, "properties": { "homepage": { "type": "string", "format": "uri" }, "runbook": { "type": "string", "format": "uri" }, "dictionary": { "type": "string", "format": "uri" }, "changeLog": { "type": "string", "format": "uri" }, "tags": { "$ref": "#/$defs/tags", "description": "Documentation tags for organization." }, "labels": { "$ref": "#/$defs/labels", "description": "Documentation labels for automation." } } }, "agentPolicy": { "type": "object", "description": "NEW in v0.7.1: AI/LLM usage governance. Controls which AI models can consume data and defines usage boundaries. Enables data owners to enforce 'reporting yes, training no' policies.", "additionalProperties": false, "properties": { "allowedModels": { "type": "array", "description": "Whitelist of AI models permitted to consume this data. Examples: ['gpt-4', 'claude-3-opus', 'gemini-pro']. Empty array = no AI access.", "items": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-_.]*[a-z0-9]$|^[a-z0-9]$", "examples": [ "gpt-4", "claude-3-opus", "claude-3-sonnet", "gemini-pro", "gemini-ultra", "llama-3", "mixtral-8x7b" ] }, "uniqueItems": true }, "deniedModels": { "type": "array", "description": "Blacklist of AI models explicitly prohibited from consuming this data. Takes precedence over allowedModels.", "items": { "type": "string" }, "uniqueItems": true }, "maxTokensPerRequest": { "type": "integer", "minimum": 1, "description": "Maximum tokens per AI request. Prevents excessive data exposure in single queries.", "examples": [ 4096, 8192, 16384 ] }, "maxTokensPerDay": { "type": "integer", "minimum": 1, "description": "Daily token limit for AI consumption. Enforces usage quotas." }, "allowedUseCases": { "type": "array", "description": "Permitted AI use cases. Controls whether data can be used for training, inference, reasoning, etc.", "items": { "type": "string", "enum": [ "inference", "reasoning", "analysis", "summarization", "classification", "embedding", "search", "qa", "code_generation", "fine_tuning", "training", "rag" ] }, "uniqueItems": true, "examples": [ [ "inference", "reasoning", "summarization" ], [ "search", "qa", "rag" ] ] }, "deniedUseCases": { "type": "array", "description": "Prohibited AI use cases. Example: Allow reporting but deny model training.", "items": { "type": "string", "enum": [ "inference", "reasoning", "analysis", "summarization", "classification", "embedding", "search", "qa", "code_generation", "fine_tuning", "training", "rag" ] }, "uniqueItems": true, "examples": [ [ "training", "fine_tuning" ] ] }, "canReason": { "type": "boolean", "default": false, "description": "Whether AI agents can use this data for multi-step reasoning/chain-of-thought. False = simple retrieval only." }, "canStore": { "type": "boolean", "default": false, "description": "Whether AI systems can cache/store this data. False = ephemeral processing only." }, "retentionPolicy": { "type": "object", "description": "Data retention requirements for AI systems consuming this data.", "properties": { "maxRetentionDays": { "type": "integer", "minimum": 0, "description": "Maximum days AI system can retain this data. 0 = no retention allowed." }, "requireDeletion": { "type": "boolean", "default": true, "description": "Whether AI system must delete data after use." } } }, "auditRequired": { "type": "boolean", "default": true, "description": "Whether AI consumption must be logged/audited." }, "purposeLimitation": { "type": "string", "description": "Free-text description of allowed purposes. Example: 'Customer service chatbot only, not for marketing'." }, "tags": { "$ref": "#/$defs/tags", "description": "Agent policy tags for categorization." }, "labels": { "$ref": "#/$defs/labels", "description": "Agent policy labels for automation." } } }, "sovereignty": { "type": "object", "description": "NEW in v0.7.1: Data sovereignty and jurisdiction requirements. Declares WHERE data must reside and enables compliance enforcement before deployment.", "additionalProperties": false, "properties": { "jurisdiction": { "type": "string", "description": "Required legal jurisdiction for data storage and processing. Used to validate binding.location matches sovereignty intent.", "enum": [ "EU", "US", "UK", "CA", "AU", "JP", "CN", "IN", "BR", "Global", "Multi-Region" ], "examples": [ "EU", "US", "Global" ] }, "allowedRegions": { "type": "array", "description": "Explicit list of allowed cloud regions. Example: ['eu-west-1', 'eu-central-1'] for EU-only.", "items": { "type": "string" }, "uniqueItems": true, "examples": [ [ "eu-west-1", "eu-central-1" ], [ "us-east-1", "us-west-2" ] ] }, "deniedRegions": { "type": "array", "description": "Explicit list of prohibited cloud regions. Takes precedence over allowedRegions.", "items": { "type": "string" }, "uniqueItems": true }, "dataResidency": { "type": "boolean", "default": true, "description": "Whether data must remain within jurisdiction boundaries at rest and in transit. True = strict residency." }, "crossBorderTransfer": { "type": "boolean", "default": false, "description": "Whether cross-border data transfer is permitted. False = data never leaves jurisdiction." }, "transferMechanisms": { "type": "array", "description": "Allowed legal mechanisms for cross-border transfer (if permitted). Example: ['SCCs', 'BCRs', 'Adequacy'].", "items": { "type": "string", "enum": [ "SCCs", "BCRs", "Adequacy", "DPF", "Consent", "Derogation" ] }, "uniqueItems": true }, "regulatoryFramework": { "type": "array", "description": "Applicable regulatory frameworks. Example: ['GDPR', 'CCPA', 'HIPAA'].", "items": { "type": "string", "enum": [ "GDPR", "CCPA", "CPRA", "HIPAA", "PIPEDA", "LGPD", "PDPA", "POPIA", "DPA", "APPI" ] }, "uniqueItems": true, "examples": [ [ "GDPR" ], [ "GDPR", "CCPA" ], [ "HIPAA" ] ] }, "enforcementMode": { "type": "string", "enum": [ "strict", "advisory", "audit" ], "default": "strict", "description": "How sovereignty violations are handled. strict = block deployment, advisory = warn, audit = log only." }, "validationRequired": { "type": "boolean", "default": true, "description": "Whether binding.location must be validated against sovereignty rules before deployment." }, "tags": { "$ref": "#/$defs/tags", "description": "Sovereignty policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Sovereignty policy labels." } }, "examples": [ { "jurisdiction": "EU", "allowedRegions": [ "eu-west-1", "eu-central-1" ], "dataResidency": true, "crossBorderTransfer": false, "regulatoryFramework": [ "GDPR" ], "enforcementMode": "strict" }, { "jurisdiction": "US", "allowedRegions": [ "us-east-1", "us-west-2" ], "dataResidency": true, "regulatoryFramework": [ "CCPA", "HIPAA" ], "enforcementMode": "strict" } ] }, "accessPolicy": { "type": "object", "description": "Root-level access control policy for automated IAM binding generation. Defines who can access this data product and with what permissions.", "additionalProperties": false, "properties": { "grants": { "type": "array", "description": "List of access grants to principals (users, groups, service accounts).", "items": { "type": "object", "required": [ "principal" ], "additionalProperties": false, "properties": { "principal": { "type": "string", "description": "Identity receiving access. Format: 'user:email@domain.com', 'group:name@domain.com', 'serviceAccount:name@project.iam.gserviceaccount.com'", "examples": [ "group:data-analytics@company.com", "serviceAccount:pipeline@project.iam.gserviceaccount.com", "user:analyst@company.com" ] }, "permissions": { "type": "array", "description": "Permissions to grant. Will be mapped to provider-specific roles (e.g., BigQuery roles).", "items": { "type": "string", "enum": [ "read", "select", "query", "write", "insert", "update", "delete", "create", "admin", "manage" ] }, "uniqueItems": true, "examples": [ [ "read", "select" ], [ "write", "insert", "update" ] ] }, "resources": { "type": "array", "description": "Resources this grant applies to. JSONPath expressions referencing exposes[] items. If omitted, applies to all exposed resources.", "items": { "type": "string" }, "examples": [ "$.exposes[?(@.exposeId=='customer_table')]", "$.exposes[?(@.kind=='table')]" ] }, "conditions": { "type": "object", "description": "Optional conditions for conditional access (e.g., IP restrictions, time windows).", "additionalProperties": true } } } } }, "examples": [ { "grants": [ { "principal": "group:data-analytics@company.com", "permissions": [ "read", "select", "query" ] }, { "principal": "serviceAccount:pipeline@project.iam.gserviceaccount.com", "permissions": [ "write", "insert", "update", "delete" ] } ] } ] }, "consumeRef": { "type": "object", "additionalProperties": false, "required": [ "productId", "exposeId" ], "$comment": "productId identifies data product (renamed from 'provider' for clarity)", "properties": { "productId": { "$ref": "#/$defs/identifier", "description": "Upstream data product identifier (e.g., 'silver.hr.people_v2'). Renamed from 'provider' for clarity." }, "exposeId": { "$ref": "#/$defs/identifier", "description": "The upstream exposeId being consumed." }, "contractId": { "type": "string", "description": "NEW in v0.7.6: The upstream contract's OWN id, for an upstream that has one of its own — a product not built with FLUID, registered in a catalog under an id FLUID did not derive. The canonical-port builder already prefers an explicit contractId over the derived `.`; until now `consumes[]` was closed, so there was no way to supply it and the derived id was exported as if the upstream answered to it." }, "versionConstraint": { "$ref": "#/$defs/semverRange" }, "contractVersion": { "$ref": "#/$defs/semver", "description": "NEW in v0.7.6: The exact upstream contract version this product was built and tested against, within `versionConstraint`. Exported as the ODPS `inputPorts[].version`, and as the version of the matching `outputPorts[].inputContracts[]` entry. Without it the export derives the range's lower bound; for a range it cannot read it falls back to 1.0.0, because ODPS v1.0.0 requires a version on every input port and the field cannot simply be omitted. `versionConstraint` stays the range this product accepts; this is the single version its exported lineage claims. `fluid validate` does not check that it falls inside `versionConstraint`.", "examples": [ "2.1.0" ] }, "qosExpectations": { "type": "object", "additionalProperties": false, "properties": { "freshnessMax": { "$ref": "#/$defs/isoDuration" }, "maxStaleness": { "$ref": "#/$defs/isoDuration", "description": "Maximum acceptable staleness for consumed data." }, "minCompleteness": { "type": "number", "minimum": 0, "maximum": 1, "description": "Minimum completeness expectation." }, "tags": { "$ref": "#/$defs/tags", "description": "QoS expectation tags." }, "labels": { "$ref": "#/$defs/labels", "description": "QoS expectation labels." } } }, "requiredPolicies": { "type": "array", "items": { "type": "string" } }, "purpose": { "type": "string", "description": "Business purpose for consuming this data." }, "tags": { "$ref": "#/$defs/tags", "description": "Consumption relationship tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Consumption relationship labels." }, "upstreamWorkspace": { "type": "string", "minLength": 1, "description": "Cross-mesh only. Id of the federated workspace that owns this upstream, as declared in federation/upstreams.yaml. Set it when the upstream lives in another mesh rather than this workspace; omit it for a local upstream, which is resolved by walking the workspace." }, "upstreamDigest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$", "description": "Cross-mesh only. The contract digest this product was composed against, as sha256:<64 hex>. `fluid apply` re-fetches the upstream's live digest and compares: a mismatch means the upstream changed since composition. Required whenever upstreamWorkspace is set." } }, "dependentRequired": { "upstreamWorkspace": [ "upstreamDigest" ] } }, "build": { "type": "object", "description": "Enhanced build configuration supporting v0.4.0 patterns with v0.5.5 improvements.", "additionalProperties": false, "properties": { "id": { "$ref": "#/$defs/identifier", "description": "Build identifier for multi-build scenarios (e.g., 'batch-processing', 'ml-training')." }, "description": { "type": "string", "description": "Human-readable description of this build configuration." }, "pattern": { "type": "string", "enum": [ "hybrid-reference", "embedded-logic", "multi-stage", "acquisition" ], "default": "hybrid-reference", "description": "Build pattern: hybrid-reference (dbt-style), embedded-logic (raw SQL/code), multi-stage (complex pipelines), or acquisition (source-aligned ingestion from external systems)." }, "engine": { "type": "string", "anyOf": [ { "enum": [ "dbt", "sql", "python", "spark", "glue", "custom", "duckdb", "airbyte", "meltano", "dlt", "kafka-connect", "debezium", "flink" ] }, { "pattern": "^dbt-[a-z0-9]+([_-][a-z0-9]+)*$" } ], "default": "dbt", "description": "Build/ingestion engine. Transformation engines: dbt, sql, python, spark, glue, custom. Adapter-qualified dbt engines (dbt-, e.g. dbt-glue, dbt-snowflake, dbt-bigquery) are accepted too and route through the dbt build runner, which derives the adapter from the suffix. Ingestion engines (acquisition pattern only): duckdb (zero-infra files/JDBC), airbyte (350+ SaaS), meltano (Singer 600+ taps), dlt (Python-native), kafka-connect (streaming), debezium (CDC). AWS Glue: glue (Spark ETL jobs managed by Glue). NEW in v0.7.6: flink (Apache Flink jobs, usually long-running streaming SQL with properties.language: flink_sql). forge-cli validates and plans a flink build but ships no runner for it: the platform that runs the job, such as the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, executes it. Put deployment settings under properties.parameters, for example parallelism and upgradeMode, as the Flink Kubernetes Operator's job spec names them, and flinkConfiguration." }, "capabilities": { "type": "array", "description": "Capabilities the build asks for (acquisition pattern). Validation enforces capabilities ⊆ runner.declared_capabilities.", "items": { "type": "string", "enum": [ "full_refresh", "incremental_append", "incremental_dedup", "incremental_merge", "cdc", "streaming", "schema_discovery", "schema_evolution", "dlp_scan", "at_most_once", "at_least_once", "exactly_once" ] }, "uniqueItems": true }, "repository": { "type": "string" }, "properties": { "$ref": "#/$defs/buildProperties", "description": "Pattern-specific build properties with conditional validation." }, "execution": { "$ref": "#/$defs/buildExecution", "description": "Build execution configuration (triggers, runtime, retries)." }, "outputs": { "type": "array", "description": "List of exposeIds produced by this build.", "items": { "$ref": "#/$defs/identifier" } }, "dependencies": { "type": "array", "description": "Dependencies on other builds (for multi-build orchestration).", "items": { "type": "object", "additionalProperties": false, "properties": { "buildId": { "$ref": "#/$defs/identifier" }, "condition": { "type": "string", "enum": [ "success", "completion", "always" ] } } } }, "transformations": { "type": "array", "description": "Legacy transformations array for backward compatibility.", "items": { "$ref": "#/$defs/transformation" } }, "tags": { "$ref": "#/$defs/tags" }, "labels": { "$ref": "#/$defs/labels" } }, "allOf": [ { "if": { "properties": { "pattern": { "const": "hybrid-reference" } } }, "then": { "properties": { "properties": { "$ref": "#/$defs/hybridReferencePattern" } } } }, { "if": { "properties": { "pattern": { "const": "embedded-logic" } } }, "then": { "properties": { "properties": { "$ref": "#/$defs/embeddedLogicPattern" } } } }, { "if": { "properties": { "pattern": { "const": "multi-stage" } } }, "then": { "properties": { "properties": { "$ref": "#/$defs/multiStagePattern" } } } }, { "if": { "properties": { "pattern": { "const": "acquisition" } } }, "then": { "properties": { "properties": { "$ref": "#/$defs/acquisitionPattern" } } } } ] }, "buildProperties": { "type": "object", "description": "Base properties for all build patterns." }, "hybridReferencePattern": { "type": "object", "description": "Hybrid-reference pattern from v0.4.0 (dbt-style).", "additionalProperties": false, "properties": { "model": { "type": "string" }, "target": { "type": "string", "description": "dbt profile target to run against. Forwarded to dbt as `--target ` by the build runner (allowlisted against the profile's real targets; falls back to the profile default when the requested target is unavailable)." }, "select": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "dbt node selector(s) forwarded to `dbt --select`. Accepts a single selector string or an array of selector strings; the build runner drops flag-like values (those starting with '-')." }, "models": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "Legacy alias for `select` (dbt's older `--models` flag). Same string-or-array shape; `select` takes precedence when both are set." }, "vars": { "type": "object", "additionalProperties": true }, "materializations": { "type": "object", "additionalProperties": { "type": "string", "enum": [ "table", "view", "incremental", "ephemeral" ] } }, "tags": { "$ref": "#/$defs/tags" }, "labels": { "$ref": "#/$defs/labels" } }, "required": [ "model" ] }, "embeddedLogicPattern": { "type": "object", "description": "Embedded logic pattern from v0.4.0.", "additionalProperties": false, "properties": { "sql": { "type": "string" }, "language": { "type": "string", "enum": [ "sql", "flink_sql", "pyspark", "scala", "python", "r" ] }, "parameters": { "type": "object", "additionalProperties": true }, "stream": { "$ref": "#/$defs/streamSource", "description": "NEW in v0.7.7 (preview): The stream the build reads. On a managed Flink build (parameters.flink.management) sql is optional: the lane writes the query from the mapping instead. Elsewhere the stream describes what the build's own sql reads." }, "mapping": { "$ref": "#/$defs/recordMapping", "description": "NEW in v0.7.7 (preview): How the stream's records become the output's rows. Requires stream, and replaces sql." }, "tags": { "$ref": "#/$defs/tags" }, "labels": { "$ref": "#/$defs/labels" } }, "if": { "not": { "required": [ "stream", "parameters" ], "properties": { "parameters": { "required": [ "flink" ], "properties": { "flink": { "required": [ "management" ] } } } } } }, "then": { "required": [ "sql" ] }, "dependentRequired": { "mapping": [ "stream" ] } }, "multiStagePattern": { "type": "object", "description": "NEW in v0.5.5: Multi-stage orchestration pattern.", "additionalProperties": false, "properties": { "stages": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string" }, "pattern": { "type": "string", "enum": [ "hybrid-reference", "embedded-logic" ], "default": "hybrid-reference" }, "properties": { "$ref": "#/$defs/buildProperties" }, "dependsOn": { "type": "array", "items": { "type": "string" } }, "outputs": { "type": "array", "items": { "$ref": "#/$defs/identifier" } }, "tags": { "$ref": "#/$defs/tags" }, "labels": { "$ref": "#/$defs/labels" } } } }, "orchestration": { "type": "object", "properties": { "parallelism": { "type": "integer", "minimum": 1 }, "retryPolicy": { "$ref": "#/$defs/retryPolicy" } } }, "tags": { "$ref": "#/$defs/tags" }, "labels": { "$ref": "#/$defs/labels" } } }, "acquisitionPattern": { "type": "object", "description": "NEW in v0.7.3: Source-aligned acquisition pattern. The build ingests data from an external system (no transformation). Engine ∈ {duckdb, airbyte, meltano, dlt, kafka-connect, debezium}.", "additionalProperties": false, "required": [ "source" ], "properties": { "source": { "$ref": "#/$defs/acquisitionSource" }, "sink": { "$ref": "#/$defs/acquisitionSink" }, "delivery": { "$ref": "#/$defs/acquisitionDelivery" }, "schemaEvolution": { "$ref": "#/$defs/acquisitionSchemaEvolution" }, "preLand": { "type": "array", "description": "Pre-land hook chain - runs on each batch before destination write.", "items": { "type": "string", "enum": [ "dlp_scan", "tokenize_pii", "quality_gate", "emit_lineage_input" ] } }, "quality": { "$ref": "#/$defs/acquisitionQuality" }, "cost": { "$ref": "#/$defs/acquisitionCost" }, "catalog": { "$ref": "#/$defs/acquisitionCatalog" }, "concurrency": { "$ref": "#/$defs/acquisitionConcurrency" }, "lineage": { "type": "object", "additionalProperties": false, "properties": { "emit": { "type": "boolean", "default": true } } }, "duckdb": { "type": "object", "additionalProperties": true, "properties": { "extensions": { "type": "array", "items": { "type": "string" } } } }, "airbyte": { "type": "object", "additionalProperties": true, "properties": { "connector_image": { "type": "string" }, "version": { "type": "string" }, "normalization": { "type": "boolean" }, "image_signature": { "$ref": "#/$defs/acquisitionImageSignature" }, "deployment": { "$ref": "#/$defs/acquisitionDeployment" } } }, "meltano": { "type": "object", "additionalProperties": true, "properties": { "tap": { "type": "string" }, "project_dir": { "type": "string" }, "deployment": { "$ref": "#/$defs/acquisitionDeployment" } } }, "dlt": { "type": "object", "additionalProperties": true, "properties": { "source_module": { "type": "string" }, "pipeline_name": { "type": "string" } } }, "kafka-connect": { "type": "object", "additionalProperties": true, "properties": { "connector_class": { "type": "string" }, "connector_name": { "type": "string" }, "tasks_max": { "type": "integer", "minimum": 1, "default": 1 }, "schema_registry": { "type": "object", "additionalProperties": false, "description": "Confluent-compatible Schema Registry config. When ``url`` is set, the runner injects ``key.converter``/``value.converter`` for Avro into the emitted connector config.", "properties": { "url": { "type": "string", "format": "uri", "description": "Schema Registry HTTP endpoint, e.g. http://schema-registry:8081" }, "auth": { "type": "object", "additionalProperties": false, "properties": { "secretRef": { "type": "string" } } } } }, "sink_connector_name": { "type": "string" }, "sink_connector_config": { "type": "object", "additionalProperties": true }, "image_signature": { "$ref": "#/$defs/acquisitionImageSignature" }, "deployment": { "$ref": "#/$defs/acquisitionDeployment" }, "iceberg_sink_enabled": { "type": "boolean", "description": "Opt-in: derive the Iceberg sink connector config. Defaults OFF when a hand-written sink_connector_config is present (RFC §6.2)." }, "sink_topics": { "type": "array", "items": { "type": "string" }, "description": "Explicit topics the derived Iceberg sink consumes (else derived from source streams)." }, "iceberg_catalog_overrides": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Operator escape hatch: connector config keys merged LAST over the derived Iceberg sink config." }, "streamingSink": { "type": "object", "additionalProperties": false, "description": "Optional Iceberg streaming-sink tuning (RFC §6.2). All keys optional.", "properties": { "commitIntervalMs": { "type": "integer", "minimum": 1 }, "routeField": { "type": "string" }, "dynamicEnabled": { "type": "boolean" }, "autoCreate": { "type": "boolean" }, "evolveSchema": { "type": "boolean" }, "upsertMode": { "type": "boolean" }, "controlTopic": { "type": "string" } } } } }, "debezium": { "type": "object", "additionalProperties": true, "properties": { "connector_class": { "type": "string" }, "snapshot_mode": { "type": "string", "enum": [ "initial", "schema_only", "never", "when_needed", "always" ] }, "deployment": { "$ref": "#/$defs/acquisitionDeployment" } } } } }, "acquisitionSource": { "type": "object", "description": "External source system specification.", "additionalProperties": false, "required": [ "kind", "mode" ], "properties": { "kind": { "type": "string", "description": "Source system kind: filesystem, postgres, mysql, sqlite, http, salesforce, stripe, github, kafka, etc. Engine-specific catalog of supported kinds." }, "connection": { "type": "object", "description": "Connection details. Use ${VAR} placeholders for env values; use secretRef for credentials. Inline secrets are forbidden by validator.", "additionalProperties": true, "properties": { "uri": { "type": "string" }, "host": { "type": "string" }, "port": { "oneOf": [ { "type": "integer" }, { "type": "string", "description": "Allows ${ENV_VAR} placeholders that resolve to an integer at runtime." } ] }, "database": { "type": "string" }, "instance_url": { "type": "string" }, "secretRef": { "type": "string", "pattern": "^[a-z][a-z0-9_+.-]*://.+", "description": "Reference to a secret in the configured backend (vault://, aws://, gcp://, azure://, env://)." }, "user": { "type": "string", "description": "NEW in v0.7.3: Username/principal for SQL- or REST-style sources. Required by most database drivers (Postgres/MySQL/Oracle/etc.) — without it the underlying client falls back to the OS-user identity and typically fails authentication. Goes alongside ``secretRef`` (or an inline ``password``) for the credential pair." }, "schema": { "type": "string", "description": "NEW in v0.7.3: Default schema/namespace within the source database to read from. Maps to engine-specific config: dlt sql_database schema=, Meltano tap-postgres filter_schemas=[], Airbyte source-postgres schemas=[], Debezium schema.include.list, Kafka Connect JDBC schema.pattern. Use ``schemas`` (array) to scope to multiple schemas at once." }, "schemas": { "type": "array", "items": { "type": "string" }, "uniqueItems": true, "description": "NEW in v0.7.3: Multi-schema variant of ``schema``. When both are set, ``schemas`` wins and ``schema`` is treated as a deprecated alias. Engines that don't natively support multi-schema reads will iterate the list." } } }, "mode": { "type": "string", "enum": [ "full_refresh", "incremental_append", "incremental_dedup", "incremental_merge", "cdc", "streaming" ] }, "cursor_field": { "type": "string", "description": "Column used for incremental cursor (e.g., updated_at, SystemModstamp)." }, "watermark": { "type": "object", "additionalProperties": false, "properties": { "strategy": { "type": "string", "enum": [ "high_water_mark", "log_position", "lsn" ] }, "allowedLateness": { "$ref": "#/$defs/isoDuration" } } }, "streams": { "type": "array", "description": "List of streams/tables/objects to ingest.", "items": { "type": "string" } }, "reader": { "type": "object", "additionalProperties": true, "properties": { "format": { "type": "string", "enum": [ "csv", "parquet", "json", "ndjson", "avro", "orc" ] }, "options": { "type": "object", "additionalProperties": true } } } } }, "acquisitionSink": { "type": "object", "description": "Destination format for ingested data. binding.platform on the expose remains the source of truth for where; sink.format describes how.", "additionalProperties": false, "properties": { "format": { "type": "string", "enum": [ "iceberg", "delta", "parquet", "csv", "json", "snowflake_table", "bigquery_table", "redshift_table", "duckdb_table" ] }, "catalog": { "type": "string", "enum": [ "rest", "glue", "nessie", "unity", "snowflake-managed", "hive", "lakekeeper", "polaris" ] }, "partitionBy": { "type": "array", "items": { "type": "string" } }, "catalogAuth": { "type": "object", "description": "NEW in v0.7.6: How the Iceberg sink config forge-cli DERIVES (kafka-connect with sink.format iceberg, embedded debezium) authenticates to an Iceberg REST catalog (rest, lakekeeper, polaris, unity, snowflake-managed). Names an environment variable, never a secret: forge-cli emits the secret as the catalog property value env:, which Apache Iceberg's REST client resolves from the environment of the process that runs the sink (the Kafka Connect worker, or the Debezium Server forge-cli launches). Iceberg drops a property whose variable is unset, so the embedded Debezium run refuses to start without it; forge-cli cannot see a Connect worker's environment. type oauth2 (client credentials): credentialEnv names a variable holding client_id:client_secret, oauth2ServerUri is the identity provider's token endpoint, scope is required (lakekeeper on a Keycloak-fronted Lakekeeper; PRINCIPAL_ROLE:ALL on Polaris with internal auth, whose oauth2ServerUri is /api/catalog/v1/oauth/tokens). type bearer: tokenEnv names a variable holding a bearer token (Unity Catalog OSS). oauth2ServerUri, scope, audience and resource are written as they are and may not hold a ${...} expression or start with env: (Debezium Server would expand it, Kafka Connect would not). fluid validate refuses it on a hand-written sink config, on a catalog not reached over REST, and next to an override that sets rest.auth.type, credential, token, oauth2-server-uri, scope, audience, resource, rest.sigv4-enabled or a header.Authorization.", "additionalProperties": false, "required": [ "type" ], "properties": { "type": { "type": "string", "enum": [ "oauth2", "bearer" ], "description": "oauth2: OAuth2 client credentials against oauth2ServerUri. bearer: a static bearer token." }, "credentialEnv": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", "not": { "pattern": "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" }, "description": "type oauth2: the NAME of the environment variable holding client_id:client_secret (Iceberg splits it on the first ':'). Emitted as credential=env:. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores) is refused." }, "oauth2ServerUri": { "type": "string", "pattern": "^https?://[^\\s/?#@]+(?:[/?#]\\S*)?$", "not": { "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" }, "description": "type oauth2: absolute URL starting with a lowercase http:// or https:// of the identity provider's token endpoint, with no user:password@ part (the client id and secret go in credentialEnv's variable), e.g. a Keycloak realm's /realms//protocol/openid-connect/token. Emitted as oauth2-server-uri; without it Iceberg would send the credential to the catalog's own v1/oauth/tokens." }, "scope": { "type": "string", "minLength": 1, "not": { "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" }, "description": "type oauth2: the OAuth2 scope to request (Iceberg's default, catalog, is not one most identity providers know). Emitted as scope." }, "audience": { "type": "string", "minLength": 1, "not": { "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" }, "description": "type oauth2, optional: sent as the audience parameter on the token request." }, "resource": { "type": "string", "minLength": 1, "not": { "pattern": "\\$\\{|[\\x00-\\x1f\\x7f]|^\\s*env:" }, "description": "type oauth2, optional: sent as the resource parameter on the token request." }, "tokenEnv": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?![\\s\\S])", "not": { "pattern": "(?:^|_)(?=[A-Za-z]*[0-9])[A-Za-z0-9]{20,}(?:_|$)" }, "description": "type bearer: the NAME of the environment variable holding the bearer token. Emitted as token=env:. A value shaped like a token (a run of 20 or more letters and digits, with a digit, between underscores, such as a dapi... personal access token) is refused." } }, "allOf": [ { "if": { "required": [ "type" ], "properties": { "type": { "const": "oauth2" } }, "type": "object" }, "then": { "required": [ "credentialEnv", "oauth2ServerUri", "scope" ], "allOf": [ { "not": { "required": [ "tokenEnv" ] } } ] } }, { "if": { "required": [ "type" ], "properties": { "type": { "const": "bearer" } }, "type": "object" }, "then": { "required": [ "tokenEnv" ], "allOf": [ { "not": { "required": [ "credentialEnv" ] } }, { "not": { "required": [ "oauth2ServerUri" ] } }, { "not": { "required": [ "scope" ] } }, { "not": { "required": [ "audience" ] } }, { "not": { "required": [ "resource" ] } } ] } } ] } } }, "acquisitionDelivery": { "type": "object", "description": "Delivery semantics + idempotency + DLQ configuration.", "additionalProperties": false, "properties": { "guarantee": { "type": "string", "enum": [ "at_most_once", "at_least_once", "exactly_once" ], "default": "at_least_once" }, "idempotencyKey": { "type": "string", "description": "Template for the idempotency key. Default: {run_id}:{stream}:{record_pk}", "default": "{run_id}:{stream}:{record_pk}" }, "dlq": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": true }, "sink": { "type": "object", "additionalProperties": false, "properties": { "format": { "type": "string", "enum": [ "parquet", "json", "ndjson" ] }, "location": { "type": "string" } } }, "maxRecordsBeforeAbort": { "type": "integer", "minimum": 0, "default": 10000 }, "alertOn": { "type": "array", "items": { "type": "string", "enum": [ "pii_classification_failed", "schema_violation", "destination_write_failed", "quality_gate_failed" ] } } } } } }, "acquisitionSchemaEvolution": { "type": "object", "description": "Schema evolution behavior. Stricter-wins between policy and per-change overrides.", "additionalProperties": false, "properties": { "policy": { "type": "string", "enum": [ "strict", "discover_and_freeze", "evolve_safe", "evolve_all" ], "default": "strict" }, "onAddedColumn": { "type": "string", "enum": [ "include", "warn", "fail" ] }, "onRemovedColumn": { "type": "string", "enum": [ "drop", "warn", "fail" ] }, "onTypeChange": { "type": "string", "enum": [ "cast", "warn", "fail" ] }, "sourceFingerprint": { "type": "string", "enum": [ "required", "optional", "disabled" ], "default": "required" } } }, "acquisitionQuality": { "type": "object", "description": "Pre-land quality gates + anomaly signals.", "additionalProperties": false, "properties": { "gates": { "type": "array", "items": { "type": "object", "additionalProperties": true, "required": [ "rule", "severity" ], "properties": { "rule": { "type": "string", "enum": [ "not_null", "unique", "regex", "range", "row_count_anomaly", "freshness" ] }, "columns": { "type": "array", "items": { "type": "string" } }, "column": { "type": "string" }, "pattern": { "type": "string" }, "min": { "type": "number" }, "max": { "type": "number" }, "algorithm": { "type": "string", "enum": [ "ewma", "iqr", "exact" ] }, "sensitivity": { "type": "number" }, "severity": { "type": "string", "enum": [ "info", "warn", "error" ] } } } }, "onError": { "type": "string", "enum": [ "route_to_dlq", "abort_run", "best_effort" ], "default": "route_to_dlq" }, "anomalies": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "signal", "severity" ], "properties": { "signal": { "type": "string", "enum": [ "record_count_drop", "record_count_surge", "bytes_per_record_outlier", "latency_outlier", "dlq_rate_spike", "cursor_stalled", "schema_fingerprint_changed" ] }, "algorithm": { "type": "string", "enum": [ "ewma", "iqr", "exact" ] }, "sensitivity": { "type": "number" }, "severity": { "type": "string", "enum": [ "info", "warn", "error" ] } } } } } }, "acquisitionCost": { "type": "object", "description": "Cost tracking + budget enforcement.", "additionalProperties": false, "properties": { "budget": { "type": "object", "additionalProperties": false, "properties": { "monthly": { "type": "object", "additionalProperties": false, "properties": { "rows": { "type": "integer", "minimum": 0 }, "bytes": { "type": "string", "description": "Human-readable size with unit suffix (e.g., '50GB', '500MB')." }, "computeMinutes": { "type": "integer", "minimum": 0 } } }, "onExceed": { "type": "string", "enum": [ "warn", "abort" ], "default": "warn" } } }, "chargeback": { "type": "object", "additionalProperties": true, "properties": { "team": { "type": "string" }, "project": { "type": "string" }, "costCenter": { "type": "string" } } } } }, "acquisitionCatalog": { "type": "object", "description": "Catalog auto-registration on first apply.", "additionalProperties": false, "properties": { "register": { "type": "array", "items": { "type": "string", "enum": [ "datahub", "openmetadata", "datamesh_manager", "unity", "glue", "snowflake_horizon" ] } }, "documentation": { "type": "string", "enum": [ "auto", "manual", "none" ], "default": "auto" } } }, "acquisitionConcurrency": { "type": "object", "description": "Single-flight concurrency control.", "additionalProperties": false, "properties": { "lock": { "type": "object", "additionalProperties": false, "properties": { "scope": { "type": "string", "enum": [ "product", "build" ], "default": "product" }, "timeout": { "$ref": "#/$defs/isoDuration" }, "onContended": { "type": "string", "enum": [ "abort", "queue", "replace" ], "default": "abort" } } } } }, "acquisitionImageSignature": { "type": "object", "description": "Connector image supply-chain verification.", "additionalProperties": false, "properties": { "verifier": { "type": "string", "enum": [ "cosign" ], "default": "cosign" }, "publicKey": { "type": "string" }, "slsaProvenance": { "type": "string", "enum": [ "required", "optional", "disabled" ], "default": "optional" } } }, "acquisitionDeployment": { "type": "object", "description": "Engine deployment mode and target.", "additionalProperties": false, "required": [ "mode" ], "properties": { "mode": { "type": "string", "enum": [ "embedded", "bring-your-own", "managed" ], "default": "embedded" }, "server_url": { "type": "string", "description": "URL for bring-your-own mode." }, "auth": { "type": "object", "additionalProperties": true, "properties": { "secretRef": { "type": "string" } } }, "managed": { "type": "object", "additionalProperties": false, "required": [ "target" ], "properties": { "target": { "type": "string", "enum": [ "docker", "kubernetes", "terraform", "opentofu" ] }, "profile": { "type": "string", "enum": [ "small", "medium", "large" ], "default": "small" }, "chart": { "type": "object", "additionalProperties": true, "properties": { "repo": { "type": "string" }, "name": { "type": "string" }, "version": { "type": "string" } } }, "values_overlay": { "type": "object", "additionalProperties": true }, "secrets": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string" }, "ref": { "type": "string" } } } }, "network": { "type": "object", "additionalProperties": false, "properties": { "egressAllowList": { "type": "array", "items": { "type": "string" } } } } } } } }, "buildExecution": { "type": "object", "description": "Build execution configuration from v0.4.0 enhanced.", "additionalProperties": false, "properties": { "trigger": { "$ref": "#/$defs/trigger" }, "runtime": { "$ref": "#/$defs/runtime" }, "retries": { "$ref": "#/$defs/retryPolicy" }, "notifications": { "type": "array", "items": { "$ref": "#/$defs/notification" } }, "orchestration": { "type": "object", "required": [ "engine" ], "properties": { "engine": { "type": "string", "enum": [ "airflow", "dagster", "prefect", "kubeflow", "custom", "none" ] }, "mode": { "type": "string", "enum": [ "generated", "manual", "hybrid" ], "default": "generated" }, "generateOnChange": { "type": "boolean", "default": true }, "airflow": { "$ref": "#/$defs/airflowOrchestration" }, "dagster": { "type": "object" }, "prefect": { "type": "object" } } } } }, "trigger": { "type": "object", "properties": { "type": { "type": "string", "enum": [ "schedule", "event", "manual", "dependency", "dataset", "schedule_and_dataset", "timetable", "continuous" ], "description": "NEW in v0.7.6: continuous, for a build that runs as a long-lived job on its own platform, such as an engine: flink job on the Flink Kubernetes Operator or AWS Managed Service for Apache Flink, or a kafka-connect or debezium connector. The platform keeps the job running, so no scheduler starts runs: fluid generate schedule, fluid schedule-sync and the fluid apply DAGs leave the build out. A continuous trigger takes no other trigger, so cron, schedule, datasets, event and timetable are refused next to it, as Databricks Jobs refuses a continuous trigger next to any other." }, "schedule": { "type": "string" }, "event": { "type": "string" }, "condition": { "type": "string" }, "datasets": { "type": "array", "description": "NEW in v0.7.0: Can reference data products directly via exposeRef for strong typing and auto-URI generation. Backward compatible with raw URIs.", "items": { "oneOf": [ { "type": "string", "description": "Raw dataset URI (legacy, e.g., 'dataset://gcp/project/dataset/table')" }, { "type": "object", "description": "NEW in v0.7.0: Data product reference with strong typing", "required": [ "productId", "exposeId" ], "properties": { "productId": { "$ref": "#/$defs/identifier", "description": "Data product ID to depend on" }, "exposeId": { "$ref": "#/$defs/identifier", "description": "Expose ID within the data product" }, "versionConstraint": { "$ref": "#/$defs/semverRange", "description": "Optional version constraint (e.g., '^2.0.0')" } } } ] } }, "datasetsOperator": { "type": "string", "enum": [ "any", "all" ], "default": "all" }, "cron": { "type": "string", "description": "Cron expression for schedule trigger" }, "timezone": { "type": "string", "default": "UTC" }, "timetable": { "type": "object", "description": "Custom timetable configuration" } }, "allOf": [ { "if": { "type": "object", "required": [ "type" ], "properties": { "type": { "const": "continuous" } } }, "then": { "allOf": [ { "description": "cron is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove cron, or set type: schedule.", "not": { "required": [ "cron" ] } }, { "description": "schedule is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove schedule, or set type: schedule.", "not": { "required": [ "schedule" ] } }, { "description": "datasets is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove datasets, or set type: dataset.", "not": { "required": [ "datasets" ] } }, { "description": "event is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove event, or set type: event.", "not": { "required": [ "event" ] } }, { "description": "timetable is not allowed with type continuous: a continuous job runs all the time, so its trigger takes no other trigger. Remove timetable, or set type: timetable.", "not": { "required": [ "timetable" ] } } ] } } ] }, "runtime": { "type": "object", "properties": { "resources": { "type": "object", "properties": { "cpu": { "type": "string" }, "memory": { "type": "string" }, "disk": { "type": "string" } } }, "timeout": { "type": "string" }, "environment": { "type": "object", "additionalProperties": { "type": "string" } }, "platform": { "type": "string", "enum": [ "gcp", "aws", "azure", "local", "kubernetes", "composer", "mwaa", "databricks", "snowflake", "athena", "glue", "redshift" ] }, "executor": { "type": "string", "enum": [ "LocalExecutor", "CeleryExecutor", "KubernetesExecutor", "SequentialExecutor" ] }, "image": { "type": "string", "description": "Container image for execution" }, "serviceAccount": { "type": "string", "description": "Service account for cloud execution" } } }, "retryPolicy": { "type": "object", "properties": { "maxAttempts": { "type": "integer", "minimum": 1 }, "backoffStrategy": { "type": "string", "enum": [ "fixed", "exponential", "linear" ] }, "initialDelay": { "type": "string" }, "maxDelay": { "type": "string" } } }, "notification": { "type": "object", "additionalProperties": false, "properties": { "type": { "type": "string", "enum": [ "email", "slack", "webhook", "pagerduty" ] }, "target": { "type": "string" }, "condition": { "type": "string", "enum": [ "success", "failure", "always" ] } } }, "transformation": { "type": "object", "additionalProperties": false, "required": [ "name" ], "properties": { "name": { "type": "string" }, "model": { "type": "string", "description": "Path/script identifier (dbt model, sql file, etc.)" }, "outputs": { "type": "array", "description": "List of exposeIds materialized by this step. VALIDATION: Tools must validate that every output exists in exposes[].exposeId.", "items": { "$ref": "#/$defs/identifier" } }, "tags": { "$ref": "#/$defs/tags", "description": "Transformation step tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Transformation step labels." } } }, "lineage": { "type": "object", "additionalProperties": false, "properties": { "granularity": { "type": "string", "enum": [ "table_level", "field_level" ], "default": "table_level", "description": "Level of lineage tracking granularity." }, "upstream": { "type": "array", "description": "Upstream dependencies and their field mappings.", "items": { "type": "object", "additionalProperties": false, "required": [ "productId", "exposeId" ], "properties": { "productId": { "$ref": "#/$defs/identifier", "description": "Upstream data product ID." }, "exposeId": { "$ref": "#/$defs/identifier", "description": "Upstream expose ID being consumed." }, "fieldMappings": { "type": "array", "description": "Field-level lineage mappings (optional).", "items": { "type": "object", "additionalProperties": false, "required": [ "sourceField", "targetField" ], "properties": { "sourceField": { "type": "string" }, "targetField": { "type": "string" }, "transformation": { "type": "object", "additionalProperties": false, "properties": { "type": { "type": "string", "enum": [ "direct_mapping", "aggregation", "calculation", "lookup" ] }, "expression": { "type": "string", "description": "Transformation logic or SQL expression." }, "tags": { "$ref": "#/$defs/tags", "description": "Transformation tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Transformation labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Field mapping tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Field mapping labels." } } } }, "relationship": { "type": "string", "enum": [ "direct_consumption", "lookup_enrichment", "aggregation" ], "description": "Type of data relationship." }, "tags": { "$ref": "#/$defs/tags", "description": "Upstream relationship tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Upstream relationship labels." } } } }, "downstream": { "type": "array", "description": "Known downstream consumers (optional).", "items": { "type": "object", "additionalProperties": false, "properties": { "consumer": { "$ref": "#/$defs/identifier", "description": "Downstream data product ID." }, "impact": { "type": "string", "enum": [ "critical", "high", "medium", "low" ], "description": "Impact level if this data product changes." }, "tags": { "$ref": "#/$defs/tags", "description": "Downstream relationship tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Downstream relationship labels." } } } }, "tags": { "$ref": "#/$defs/tags", "description": "Lineage configuration tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Lineage configuration labels." } } }, "schemaEvolution": { "type": "object", "additionalProperties": false, "properties": { "strategy": { "type": "string", "enum": [ "semantic_versioning", "date_based", "sequential" ], "description": "Versioning strategy for schema changes." }, "compatibility": { "type": "string", "enum": [ "backward_compatible", "forward_compatible", "full_compatible", "breaking" ], "description": "Compatibility approach for schema evolution." }, "changePolicy": { "type": "object", "additionalProperties": false, "properties": { "changeWindowDays": { "type": "integer", "minimum": 0, "description": "Advance notice period for breaking changes." }, "approvalRequired": { "type": "boolean", "description": "Whether schema changes require approval." }, "approvers": { "type": "array", "items": { "type": "string" }, "description": "List of approvers for schema changes." }, "tags": { "$ref": "#/$defs/tags", "description": "Change policy tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Change policy labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "Schema evolution tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Schema evolution labels." } } }, "machineLearning": { "type": "object", "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "default": false }, "framework": { "type": "string", "enum": [ "mlflow", "kubeflow", "sagemaker", "vertex_ai", "custom" ] }, "models": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "name", "version", "type" ], "properties": { "name": { "type": "string" }, "version": { "$ref": "#/$defs/semver" }, "type": { "type": "string", "enum": [ "classification", "regression", "clustering", "recommendation", "other" ] }, "algorithm": { "type": "string" }, "features": { "type": "array", "description": "List of feature names used by this model.", "items": { "type": "string" } }, "target": { "type": "string", "description": "Target variable for supervised learning." }, "validation": { "type": "object", "additionalProperties": false, "properties": { "metrics": { "type": "array", "items": { "type": "string" }, "description": "Validation metrics (e.g., accuracy, auc_roc)." }, "thresholds": { "type": "object", "additionalProperties": { "type": "number" }, "description": "Minimum acceptable metric values." }, "tags": { "$ref": "#/$defs/tags", "description": "Model validation tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Model validation labels." } } }, "tags": { "$ref": "#/$defs/tags", "description": "ML model tags." }, "labels": { "$ref": "#/$defs/labels", "description": "ML model labels." } } } }, "tags": { "$ref": "#/$defs/tags", "description": "ML configuration tags." }, "labels": { "$ref": "#/$defs/labels", "description": "ML configuration labels." } } }, "environmentConfig": { "type": "object", "additionalProperties": false, "properties": { "metadata": { "type": "object", "description": "Environment-specific metadata overrides.", "additionalProperties": false, "properties": { "owner": { "type": "object", "additionalProperties": false, "properties": { "team": { "type": "string" }, "email": { "type": "string" }, "slack": { "type": "string" } } }, "tags": { "$ref": "#/$defs/tags", "description": "Environment-specific metadata tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Environment-specific metadata labels." } } }, "exposes": { "type": "array", "description": "Environment-specific expose overrides.", "items": { "type": "object", "additionalProperties": false, "properties": { "exposeId": { "$ref": "#/$defs/identifier" }, "binding": { "$ref": "#/$defs/binding" }, "qos": { "$ref": "#/$defs/exposeQoS" }, "tags": { "$ref": "#/$defs/tags", "description": "Environment-specific expose tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Environment-specific expose labels." } } } }, "tags": { "$ref": "#/$defs/tags", "description": "Environment configuration tags." }, "labels": { "$ref": "#/$defs/labels", "description": "Environment configuration labels." } } }, "orchestration": { "type": "object", "required": [ "engine" ], "properties": { "engine": { "type": "string", "enum": [ "airflow", "dagster", "prefect", "kubeflow", "custom", "none" ] }, "mode": { "type": "string", "enum": [ "generated", "manual", "hybrid" ], "default": "generated" }, "generateOnChange": { "type": "boolean", "default": true }, "airflow": { "$ref": "#/$defs/airflowOrchestration" }, "dagster": { "type": "object" }, "prefect": { "type": "object" } } }, "airflowOrchestration": { "type": "object", "required": [ "dagId" ], "properties": { "dagId": { "type": "string", "pattern": "^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$" }, "dagConfig": { "type": "object", "properties": { "description": { "type": "string" }, "schedule": { "type": "string" }, "startDate": { "type": "string" }, "endDate": { "type": "string" }, "catchup": { "type": "boolean", "default": false }, "maxActiveRuns": { "type": "integer", "minimum": 1 }, "dagRunTimeout": { "$ref": "#/$defs/isoDuration" }, "defaultArgs": { "type": "object" }, "tags": { "type": "array", "items": { "type": "string" }, "description": "Airflow DAG tags (different from FLUID tags)" }, "onSuccessCallback": { "type": "string" }, "onFailureCallback": { "type": "string" }, "slaCallback": { "type": "string" } } }, "taskDefaults": { "type": "object", "description": "Default configuration for all tasks (can be overridden per task)", "properties": { "retries": { "type": "integer", "minimum": 0 }, "retryDelay": { "$ref": "#/$defs/isoDuration" }, "executionTimeout": { "$ref": "#/$defs/isoDuration" }, "pool": { "type": "string" }, "queue": { "type": "string" }, "priority": { "type": "integer" }, "triggerRule": { "type": "string", "enum": [ "all_success", "all_failed", "all_done", "one_success", "one_failed", "none_failed", "none_skipped" ] }, "errorHandling": { "type": "object", "description": "NEW in v0.7.0: Enhanced error categorization and handling", "properties": { "strategy": { "type": "string", "enum": [ "fixed", "exponential_backoff", "linear_backoff" ], "default": "exponential_backoff" }, "retryableErrors": { "type": "array", "items": { "type": "string" }, "description": "Error types that should trigger retries", "examples": [ [ "TemporaryFailure", "RateLimitExceeded", "ConnectionTimeout", "DeadlineExceeded" ] ] }, "nonRetryableErrors": { "type": "array", "items": { "type": "string" }, "description": "Error types that should fail immediately", "examples": [ [ "ValidationError", "SchemaViolation", "PermissionDenied", "InvalidCredentials" ] ] }, "onError": { "type": "array", "items": { "type": "object", "properties": { "action": { "type": "string", "enum": [ "notify", "create_incident", "log", "custom" ] }, "channel": { "type": "string" }, "severity": { "type": "string", "enum": [ "low", "medium", "high", "critical" ] }, "when": { "type": "string", "enum": [ "always", "maxAttemptsExceeded", "nonRetryableError" ] } } } } } } } }, "tasks": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/airflowTask" } }, "sensors": { "type": "array", "items": { "$ref": "#/$defs/airflowSensor" } }, "taskGroups": { "type": "array", "items": { "type": "object", "required": [ "groupId" ], "properties": { "groupId": { "type": "string" }, "description": { "type": "string" }, "tasks": { "type": "array", "items": { "$ref": "#/$defs/airflowTask" } } } } } } }, "airflowTask": { "type": "object", "required": [ "taskId" ], "properties": { "taskId": { "type": "string", "pattern": "^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$" }, "type": { "type": "string", "description": "NEW in v0.7.0: Simplified task type taxonomy. Replaces 'operator' field (backward compatible). Types: provider_action (NEW!), fluid_validate, fluid_plan, fluid_apply, fluid_execute, fluid_verify, fluid_dq_check, bash, python, sensor, or provider-specific operators.", "enum": [ "provider_action", "fluid_validate", "fluid_plan", "fluid_apply", "fluid_execute", "fluid_verify", "fluid_dq_check", "bash", "python", "branch_python", "sensor", "email", "http", "snowflake_query", "bigquery_query", "bigquery_job", "glue_job", "databricks_job", "custom" ] }, "operator": { "type": "string", "description": "DEPRECATED in v0.7.0: Use 'type' instead. Maintained for backward compatibility with v0.6.1 contracts. Maps to 'type' automatically (FluidExecuteOperator → fluid_execute, BashOperator → bash)." }, "provider": { "type": "string", "description": "NEW in v0.7.0: Provider name when type=provider_action. Examples: aws, gcp, azure, snowflake, databricks.", "enum": [ "aws", "gcp", "azure", "snowflake", "databricks", "kafka", "kubernetes", "local", "custom" ] }, "action": { "type": "string", "pattern": "^[a-z0-9_]+\\.[a-z0-9_]+$", "description": "NEW in v0.7.0: Provider action when type=provider_action. Format: service.operation (e.g., s3.ensure_bucket, table.ensure, kinesis.create_stream).", "examples": [ "s3.ensure_bucket", "table.ensure", "database.ensure", "kinesis.create_stream", "glue.create_job", "bigquery.create_table" ] }, "buildStepRef": { "type": "string", "description": "NEW in v0.7.0: Reference to transformation step in builds[].transformations[]. Links orchestration tasks to build steps for clearer mapping." }, "dependencies": { "type": "array", "items": { "type": "string" } }, "params": { "type": "object", "additionalProperties": true }, "produces": { "type": "array", "description": "NEW in v0.7.0: Declares what exposes[] this task produces. Enables automatic dataset registration and lineage tracking.", "items": { "type": "object", "required": [ "exposeId" ], "properties": { "exposeId": { "$ref": "#/$defs/identifier", "description": "The exposeId from exposes[] that this task produces" }, "dataset": { "type": "string", "description": "Optional: Airflow dataset URI. Auto-generated from binding.location if not specified." }, "updateStrategy": { "type": "string", "enum": [ "replace", "append", "merge", "upsert" ], "default": "replace", "description": "How this task updates the data" } } } }, "dataProductDependencies": { "type": "array", "description": "NEW in v0.7.0: Tasks can declare dependencies on other data products with version constraints and freshness requirements.", "items": { "type": "object", "required": [ "productId", "exposeId" ], "properties": { "productId": { "$ref": "#/$defs/identifier", "description": "Data product ID to depend on" }, "exposeId": { "$ref": "#/$defs/identifier", "description": "Specific expose within the product" }, "versionConstraint": { "$ref": "#/$defs/semverRange", "description": "Required version (e.g., '^2.0.0', '>=1.5.0')" }, "awaitFreshness": { "$ref": "#/$defs/isoDuration", "description": "Maximum age of data to wait for (e.g., PT6H = 6 hours)" } } } }, "overrides": { "type": "object", "description": "Task-specific overrides for taskDefaults" }, "doc": { "type": "string" }, "docMd": { "type": "string" }, "uiColor": { "type": "string" }, "uiFgcolor": { "type": "string" }, "xcom": { "type": "object", "description": "XCom configuration for inter-task communication (from v0.6.1)", "properties": { "push": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": {}, "serializer": { "type": "string", "enum": [ "json", "pickle", "cloudpickle", "custom" ] } } } }, "pull": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "task_id": { "type": "string" }, "dag_id": { "type": "string" }, "map_indexes": { "type": "array" } } } } } }, "accessControl": { "type": "object", "description": "Task-level access control (from v0.6.1)", "properties": { "requiredRoles": { "type": "array", "items": { "type": "string" } }, "approvalRequired": { "type": "boolean" }, "approvers": { "type": "array", "items": { "type": "string" } } } }, "costEstimate": { "type": "object", "description": "Cost tracking (enhanced in v0.7.0)", "properties": { "compute": { "type": "object", "properties": { "units": { "type": "number" }, "pricePerUnit": { "type": "number" }, "currency": { "type": "string", "default": "USD" } } }, "storage": { "type": "object", "properties": { "units": { "type": "number" }, "pricePerUnit": { "type": "number" } } }, "total": { "type": "number" }, "trackActual": { "type": "boolean", "default": false, "description": "NEW in v0.7.0: Track actual costs vs estimate" }, "alertThreshold": { "type": "number", "description": "NEW in v0.7.0: Alert if actual > threshold * estimate (e.g., 1.5 = 150%)" }, "budget": { "type": "object", "description": "NEW in v0.7.0: Budget enforcement", "properties": { "monthly": { "type": "number" }, "stopOnExceed": { "type": "boolean", "default": false }, "alertOnExceed": { "type": "boolean", "default": true } } } } } }, "allOf": [ { "description": "Backward compatibility: If 'operator' is specified but not 'type', map operator to type", "if": { "required": [ "operator" ], "not": { "required": [ "type" ] } }, "then": { "description": "Operator field is deprecated but still works. Maps: FluidExecuteOperator→fluid_execute, BashOperator→bash, etc." } }, { "description": "NEW in v0.7.0: Validate provider_action tasks have required fields", "if": { "properties": { "type": { "const": "provider_action" } }, "required": [ "type" ] }, "then": { "required": [ "provider", "action", "params" ], "errorMessage": "Tasks with type=provider_action must specify provider, action, and params" } }, { "description": "Validate fluid_execute tasks", "if": { "anyOf": [ { "properties": { "type": { "const": "fluid_execute" } } }, { "properties": { "operator": { "pattern": "FluidExecute" } } } ] }, "then": { "properties": { "params": { "type": "object", "required": [ "contract_path" ], "properties": { "contract_path": { "type": "string" }, "build_id": { "type": "string" } } } } } }, { "description": "Validate bash tasks", "if": { "anyOf": [ { "properties": { "type": { "const": "bash" } } }, { "properties": { "operator": { "pattern": "Bash" } } } ] }, "then": { "properties": { "params": { "anyOf": [ { "required": [ "bash_command" ] }, { "required": [ "command" ] } ] } } } }, { "description": "Validate python tasks", "if": { "anyOf": [ { "properties": { "type": { "const": "python" } } }, { "properties": { "operator": { "pattern": "Python" } } } ] }, "then": { "properties": { "params": { "anyOf": [ { "required": [ "python_callable" ] }, { "required": [ "callable" ] } ] } } } } ] }, "airflowSensor": { "type": "object", "required": [ "taskId", "sensorType" ], "properties": { "taskId": { "type": "string" }, "sensorType": { "type": "string", "enum": [ "FileSensor", "ExternalTaskSensor", "TimeSensor", "HttpSensor", "S3KeySensor", "GCSObjectSensor", "custom" ] }, "params": { "type": "object" }, "pokInterval": { "type": "integer", "minimum": 1, "default": 60 }, "timeout": { "type": "integer", "minimum": 1, "default": 3600 }, "mode": { "type": "string", "enum": [ "poke", "reschedule" ], "default": "poke" }, "dependencies": { "type": "array", "items": { "type": "string" } } } }, "semanticModel": { "type": "object", "description": "NEW in v0.7.2: Semantic model definition for an exposed interface. Maps physical columns to business concepts (entities, measures, dimensions, metrics) enabling AI agents and BI tools to generate correct queries without semantic hallucination. Compatible with dbt MetricFlow, Snowflake Semantic Views, and OSI-aligned formats.", "additionalProperties": false, "properties": { "name": { "type": "string", "description": "Human-readable name for this semantic model." }, "description": { "type": "string", "description": "Business context explaining what this semantic model represents." }, "defaultAggTimeDimension": { "type": "string", "description": "Default time dimension used for measure aggregation. Can be overridden per-measure." }, "entities": { "type": "array", "description": "Join keys with type annotations. Entities connect this table to others in the semantic graph.", "items": { "type": "object", "additionalProperties": false, "required": [ "name", "type" ], "properties": { "name": { "type": "string", "description": "Entity name (e.g., 'order', 'customer')." }, "type": { "type": "string", "enum": [ "primary", "foreign", "unique", "natural" ], "description": "Entity key type: primary (unique, complete), foreign (references another table), unique (unique, possibly partial), natural (real-world identifier)." }, "expr": { "type": "string", "description": "Column expression. Defaults to entity name if omitted." }, "description": { "type": "string" } } } }, "measures": { "type": "array", "description": "Aggregatable column expressions. Measures are the atomic building blocks from which metrics are composed.", "items": { "type": "object", "additionalProperties": false, "required": [ "name", "agg" ], "properties": { "name": { "type": "string", "description": "Measure name (e.g., 'total_amount', 'order_count')." }, "description": { "type": "string" }, "agg": { "type": "string", "enum": [ "sum", "avg", "count", "count_distinct", "min", "max", "median", "percentile" ], "description": "Aggregation function applied to the expression." }, "expr": { "type": "string", "description": "SQL expression to aggregate. Defaults to measure name if omitted." }, "aggParams": { "type": "object", "additionalProperties": false, "description": "Aggregation parameters (mirrors dbt-semantic-interfaces agg_params). Currently used by agg=percentile.", "properties": { "percentile": { "type": "number", "minimum": 0, "maximum": 1, "description": "Percentile in [0, 1] for agg=percentile (0.5 = median). Consumers default to 0.5 when omitted." }, "useDiscretePercentile": { "type": "boolean", "description": "Use the discrete percentile (PERCENTILE_DISC / use_discrete_percentile) instead of continuous interpolation." } } }, "aggTimeDimension": { "type": "string", "description": "Override the model-level default time dimension for this measure." }, "nonAdditiveDimension": { "type": "object", "additionalProperties": false, "description": "Dimension over which this measure cannot be naively aggregated (e.g., account balances).", "properties": { "name": { "type": "string" }, "windowChoice": { "type": "string", "enum": [ "min", "max" ], "description": "Which value to pick when the non-additive dimension is present." } } }, "createMetric": { "type": "boolean", "default": false, "description": "If true, automatically create a simple metric from this measure." } } } }, "dimensions": { "type": "array", "description": "Categorical and time-based grouping axes. Dimensions define how data can be sliced in queries.", "items": { "type": "object", "additionalProperties": false, "required": [ "name", "type" ], "properties": { "name": { "type": "string", "description": "Dimension name (e.g., 'order_date', 'region', 'product_category')." }, "type": { "type": "string", "enum": [ "categorical", "time" ], "description": "Dimension type: categorical (text/enum grouping) or time (temporal axis)." }, "expr": { "type": "string", "description": "SQL expression. Defaults to dimension name if omitted." }, "description": { "type": "string" }, "typeParams": { "type": "object", "additionalProperties": false, "description": "Type-specific parameters.", "properties": { "timeGranularity": { "type": "string", "enum": [ "day", "week", "month", "quarter", "year", "hour", "minute" ], "description": "Default time granularity for this dimension." } } } } } }, "metrics": { "type": "array", "description": "Composable KPI definitions built from measures. Metrics are the named business calculations that agents and tools resolve.", "items": { "type": "object", "additionalProperties": false, "required": [ "name", "type" ], "properties": { "name": { "type": "string", "description": "Metric name (e.g., 'net_revenue', 'customer_churn_rate')." }, "description": { "type": "string", "description": "Business definition of this metric." }, "type": { "type": "string", "enum": [ "simple", "derived", "ratio" ], "description": "Metric type: simple (single measure + filter), derived (expression over other metrics), ratio (numerator/denominator)." }, "measure": { "type": "string", "description": "Reference to a measure name (for simple metrics)." }, "filter": { "type": "string", "description": "SQL WHERE clause filter (e.g., \"status = 'completed' AND is_refunded = FALSE\")." }, "inputMetrics": { "type": "array", "items": { "type": "string" }, "description": "References to other metric names (for derived/ratio metrics)." }, "expr": { "type": "string", "description": "Mathematical expression combining input metrics (for derived metrics)." }, "numerator": { "type": "string", "description": "Numerator metric name (for ratio metrics)." }, "denominator": { "type": "string", "description": "Denominator metric name (for ratio metrics)." }, "owner": { "type": "string", "description": "Business owner of this metric definition." } } } }, "tags": { "$ref": "#/$defs/tags", "description": "Semantic model tags for discovery and categorization." }, "labels": { "$ref": "#/$defs/labels", "description": "Semantic model labels for automation and metadata." } }, "examples": [ { "name": "orders_revenue_model", "description": "Canonical semantic model for revenue analytics", "defaultAggTimeDimension": "order_date", "entities": [ { "name": "order", "type": "primary", "expr": "order_id" }, { "name": "customer", "type": "foreign", "expr": "customer_id" } ], "measures": [ { "name": "order_amount", "agg": "sum", "expr": "amount", "description": "Sum of all order amounts" }, { "name": "order_count", "agg": "count_distinct", "expr": "order_id", "createMetric": true } ], "dimensions": [ { "name": "order_date", "type": "time", "expr": "completed_at", "typeParams": { "timeGranularity": "day" } }, { "name": "region", "type": "categorical" } ], "metrics": [ { "name": "net_revenue", "description": "GAAP-compliant net revenue: completed orders minus discounts, excluding refunds", "type": "derived", "inputMetrics": [ "gross_revenue", "total_discounts" ], "expr": "gross_revenue - total_discounts", "owner": "finance-data@company.com" }, { "name": "gross_revenue", "description": "Total value of completed orders", "type": "simple", "measure": "order_amount", "filter": "status = 'completed'" } ] } ] }, "exposeMcp": { "type": "object", "additionalProperties": false, "description": "NEW in v0.7.4: Per-expose overrides for the MCP output-port server — sampling caps + classification. The presence of this (non-empty) block opts the expose into the Fluid MCP gateway, where policy.agentPolicy is enforced at every read.", "properties": { "sampling": { "type": "object", "additionalProperties": false, "properties": { "maxRows": { "type": "integer", "minimum": 1, "description": "Hard cap for the `sample` MCP tool. Defends against an over-curious agent asking for petabyte-scale row counts. Default: 100." } } }, "classification": { "type": "object", "additionalProperties": false, "properties": { "dataClass": { "type": "string", "enum": [ "public", "internal", "confidential", "restricted" ], "description": "Surfaced on the `describe` tool so consumer agents can declare downstream handling. Advisory; the real gate is `policy.agentPolicy`." } } } } }, "governance": { "type": "object", "description": "Account/project-wide governance settings. Today only the lakeFormation block is supported; future cloud-specific blocks may live alongside it (e.g. snowflake masking-policy registries, gcp organization-policies).", "additionalProperties": false, "properties": { "lakeFormation": { "type": "object", "additionalProperties": false, "description": "AWS Lake Formation account-wide settings: data-lake admins and LF-tag (TBAC) definitions. Applies to the AWS account the contract's IaC apply targets.", "properties": { "admins": { "type": "array", "description": "IAM principal ARNs (roles or users) granted Lake Formation admin privileges. Emitted as aws_lakeformation_data_lake_settings.admins. AUTHORITATIVE, not additive: Lake Formation's PutDataLakeSettings replaces the account's admin list with the one sent, so an admin NOT listed here is REMOVED on apply. List every principal that must stay a data lake admin, including the role or user that runs the apply: left out, it loses Lake Formation admin on that apply, and the Lake Formation grants and registrations that need an admin can then fail. The terraform-provider-aws resource also clears the data lake settings its configuration omits, and this block sets only the admins, so the account's create-database and create-table default permissions (the IAMAllowedPrincipals default for new databases and tables), trusted resource owners and parameters are cleared too. Destroying the resource empties the admin list and resets CROSS_ACCOUNT_VERSION to 1. Re-applying the same list is a no-op.", "items": { "type": "string", "pattern": "^arn:aws[a-z0-9-]*:iam::" }, "uniqueItems": true }, "tagDefinitions": { "type": "object", "description": "LF-TBAC tag-key definitions. Each entry is a tag key mapped to its allowed string values. Emitted as one aws_lakeformation_lf_tag per key; values are the tag's allowed-values list. The apply CREATES every tag defined here, so a key defined here must not also be created elsewhere: not by a platform bootstrap, another contract or by hand in the same account. A tag owned outside the contract is not defined here, and a binding's governance.lakeFormation.tags can still associate it. An association with a key defined here waits for the tag, and its value must be one of the tag's values, compared case-insensitively because Lake Formation stores keys and values lower-cased. A key with no values is refused, and so are two keys that would be one resource name, such as a-b and a_b. Field idiom borrowed from OpenMetadata's classification-label model — see ODCS roles + OpenMetadata data contracts spec.", "additionalProperties": { "type": "array", "items": { "type": "string" }, "uniqueItems": true, "minItems": 1 } } } } } }, "bindingGovernance": { "type": "object", "description": "Per-resource (per-exposure) governance. Currently scoped to AWS Lake Formation; future cloud-specific blocks may sit alongside (e.g. gcpIamPolicy, snowflakeMaskingPolicy).", "additionalProperties": false, "properties": { "lakeFormation": { "type": "object", "additionalProperties": false, "description": "Per-resource LF settings: location registration, principal grants, LF-tag associations, row/column filters. Applies only to formats backed by S3 + Glue catalog (iceberg/parquet/csv/json/avro/orc/delta).", "properties": { "registerLocation": { "type": "boolean", "default": false, "description": "When true, emit aws_lakeformation_resource that registers the binding's S3 path with Lake Formation. Required before LF can manage access to objects under that path." }, "bucketPolicy": { "type": "string", "enum": [ "cross-account", "none", "all-grantees" ], "default": "cross-account", "description": "NEW in v0.7.6: Which grants[] principals get a statement in the aws_s3_bucket_policy emitted beside the grants (s3:ListBucket and s3:GetBucketLocation on the bucket, s3:GetObject under location.path). A statement lets its principal read the objects straight from S3, which skips Lake Formation's column, row and cell filters and its revocations. 'cross-account' (the default): only principals whose ARN names an AWS account other than the one running the apply, decided at plan time against the caller identity; same-account grantees get no statement, since on a registered location Lake Formation vends them credentials. 'none': no bucket policy at all; the tightest setting when the location is registered (registerLocation) and every reader, cross-account ones included, queries through a Lake Formation integrated engine, or when the bucket policy is managed elsewhere. 'all-grantees': every grantee, same-account included, which is the output before this field existed and reopens the direct S3 read path. An emitted aws_s3_bucket_policy is authoritative: it replaces every other statement on the bucket. When no grantee is left none is emitted." }, "grants": { "type": "array", "description": "Principal-based LF grants on this exposure's database/table. Each entry maps a principal to a permission set (and optional column projection). Emitted as aws_lakeformation_permissions resources. Field idiom adapted from ODCS v3 roles[] (principal + access-type), bound to IAM ARNs since LF is cloud-specific.", "items": { "type": "object", "additionalProperties": false, "required": [ "principal", "permissions" ], "properties": { "principal": { "type": "string", "description": "IAM principal ARN (role or user) receiving the grant.", "pattern": "^arn:aws[a-z0-9-]*:iam::" }, "permissions": { "type": "array", "description": "LF permissions granted. Valid values per the hashicorp/aws aws_lakeformation_permissions resource.", "items": { "type": "string", "enum": [ "ALL", "SELECT", "ALTER", "DROP", "DELETE", "INSERT", "DESCRIBE", "CREATE_TABLE", "CREATE_DATABASE", "DATA_LOCATION_ACCESS" ] }, "uniqueItems": true, "minItems": 1 }, "permissionsWithGrantOption": { "type": "array", "description": "Subset of permissions the principal can re-grant. Maps to aws_lakeformation_permissions.permissions_with_grant_option.", "items": { "type": "string" }, "uniqueItems": true }, "columns": { "type": "array", "description": "Optional column-level restriction. When set, the grant is emitted against aws_lakeformation_permissions.table_with_columns instead of .table, allowing the principal to read only the listed columns.", "items": { "type": "string" }, "uniqueItems": true }, "excludedColumns": { "type": "array", "description": "Opposite of columns: every column EXCEPT these is granted. Maps to table_with_columns.excluded_column_names. Mutually exclusive with columns.", "items": { "type": "string" }, "uniqueItems": true } } } }, "tags": { "type": "object", "description": "LF-tag (TBAC) associations applied to this table, as tag key to value. Emitted as aws_lakeformation_resource_lf_tags; a key whose value is empty is not associated. A key defined in the contract's top-level governance.lakeFormation.tagDefinitions is a tag this contract creates: the association waits for it, and the value must be one of the tag's values. A key the contract does not define is a tag owned outside the contract, such as a platform's central LF-tag ontology: it must already exist in the account, the principal that runs the apply needs the Lake Formation ASSOCIATE permission on it, and only Lake Formation checks its key and value, at apply. A key the contract defines must not also be created elsewhere. Refused when the binding names no Glue table (location.database and location.table).", "additionalProperties": { "type": "string" } }, "rowFilter": { "type": "object", "additionalProperties": false, "description": "Row-level (and optionally column-level) filter applied to all reads of this table via LF. Emitted as aws_lakeformation_data_cells_filter. The filter must then be granted to principals via grants[].permissions including SELECT — LF enforces both layers.", "properties": { "name": { "type": "string", "description": "Filter name (must be unique per table)." }, "rowExpression": { "type": "string", "description": "PartiQL-flavoured row predicate. ``ALL`` lets every row through (use when only column filtering is intended)." }, "columnNames": { "type": "array", "description": "Columns visible through the filter. Mutually exclusive with allColumns / excludedColumnNames.", "items": { "type": "string" }, "uniqueItems": true }, "excludedColumnNames": { "type": "array", "description": "Columns hidden through the filter (everything else is visible).", "items": { "type": "string" }, "uniqueItems": true }, "allColumns": { "type": "boolean", "default": false, "description": "Convenience for the LF column-wildcard form (every column visible). When true, columnNames / excludedColumnNames must be omitted." } }, "required": [ "name", "rowExpression" ] }, "maskedViews": { "type": "object", "additionalProperties": false, "required": [ "definer", "validationConnection" ], "description": "Where the protected Glue Data Catalog views of the expose's masked columns come from (NEW in v0.7.6). Lake Formation cannot mask, so a principal an access: mask column restriction names reads a protected, SECURITY DEFINER multi-dialect view, _masked, that returns the columns masked by the restriction's rule and applies its row filter; its grant on the table excludes the columns. The definer is granted SELECT with grant option on the table and DESCRIBE on the database; the principal SELECT and DESCRIBE on the view.", "properties": { "definer": { "type": "string", "description": "ARN of the IAM role the views are defined as: Glue and Lake Formation must be allowed to assume it (sts:AssumeRole, sts:SetContext), and the applying principal to pass it (iam:PassRole)." }, "validationConnection": { "type": "string", "description": "The Glue connection of type VIEW_VALIDATION_ATHENA (WORKGROUP_NAME) Glue validates each view's SQL through; the applying principal and the definer need glue:PassConnection on it." } } } } } } }, "consumerRef": { "type": "object", "additionalProperties": false, "required": [ "name", "type" ], "$comment": "dbt-exposures-shaped (name/label/type/owner/url/maturity/description); exposeIds is the FLUID-native refinement tying a consumer to specific output ports.", "properties": { "name": { "$ref": "#/$defs/identifier", "description": "Stable identifier for this consumer (unique within the contract)." }, "label": { "type": "string", "description": "Human-facing display name, e.g. \"Weekly Revenue Dashboard\"." }, "type": { "type": "string", "enum": [ "dashboard", "notebook", "analysis", "ml", "application" ], "description": "What kind of artifact consumes this product." }, "owner": { "type": "object", "additionalProperties": false, "$comment": "Mirrors metadata.owner's vocabulary so 'owner' means one thing across the standard: team is the routable identity, email the contact. A dbt exposure's owner.name maps to team on import.", "properties": { "team": { "type": "string", "description": "Owning team for this consumer (routable identity, as in metadata.owner)." }, "email": { "type": "string", "format": "email", "description": "Contact address for the consumer's owner." } } }, "url": { "type": "string", "description": "Link to the live artifact (dashboard URL, notebook, app)." }, "maturity": { "type": "string", "enum": [ "high", "medium", "low" ], "description": "How production-hardened the consumer is (dbt exposures semantics)." }, "description": { "type": "string" }, "exposeIds": { "type": "array", "items": { "$ref": "#/$defs/identifier" }, "description": "Which output ports this consumer reads. Absent = all exposes." } } } } }