Spark read check for sample-release/ (local Apache Spark session) Started (UTC): 2026-09-13T12:24:49+00:00 Command: .venv/bin/python pipeline/spark_read_check.py --log sample-release/spark-read-log.txt Working directory: repository root; paths below are relative to it Python: 3.12.3 (CPython) OS: Linux 7.0.0-31-generic x86_64 pyarrow: 25.0.1 pyspark package: 4.2.0 java on PATH: /usr/bin/java; JAVA_HOME: not set java -version: openjdk version "21.0.12" 2026-07-21 OpenJDK Runtime Environment (build 21.0.12+8-1-24.04-Ubuntu) OpenJDK 64-Bit Server VM (build 21.0.12+8-1-24.04-Ubuntu, mixed mode, sharing) SPARK_LOCAL_IP: 127.0.0.1 Temporary Spark directory (removed at the end): [2026-09-13T12:24:50+00:00] starting SparkSession WARNING: Using incubator modules: jdk.incubator.vector Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties Setting default log level to "WARN". To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). 26/09/13 12:25:04 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 26/09/13 12:25:04 WARN SparkConf: Note that spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in standalone/kubernetes and LOCAL_DIRS in YARN). Spark version (spark.version): 4.2.0 JVM java.version seen by Spark: 21.0.12 Master: local[*]; catalog implementation: in-memory ==== sample-release/product.parquet ==== [2026-09-13T12:25:17+00:00] spark.read.parquet('sample-release/product.parquet') [PASS] product.parquet SHA-256 matches manifest: e5606b71d95f3fb603a8465f768f434392ace9632f0b40d2af8b837038d98f28 26/09/13 12:25:25 WARN SparkStringUtils: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'. root |-- ve_product_id: string (nullable = true) |-- prototype_id: string (nullable = true) |-- name: string (nullable = true) |-- kind: string (nullable = true) |-- category_key: string (nullable = true) |-- category_label: string (nullable = true) |-- product_role: string (nullable = true) |-- chain_brand: string (nullable = true) |-- type_in_source: string (nullable = true) |-- lvep_id: string (nullable = true) |-- lvep_name: string (nullable = true) |-- lad_code: string (nullable = true) |-- lad_name: string (nullable = true) |-- latitude: double (nullable = true) |-- longitude: double (nullable = true) |-- website: string (nullable = true) |-- description: string (nullable = true) |-- opening_hours: string (nullable = true) |-- months_open: string (nullable = true) |-- months_open_method: string (nullable = true) |-- seasonality_evidence: string (nullable = true) |-- seasonal_tag: string (nullable = true) |-- wheelchair_tag: string (nullable = true) |-- heritage_designations: array (nullable = true) | |-- element: string (containsNull = true) |-- category_rule: string (nullable = true) |-- category_confidence: double (nullable = true) |-- source_count: integer (nullable = true) |-- primary_source: string (nullable = true) |-- primary_source_record_id: string (nullable = true) |-- primary_source_url: string (nullable = true) |-- checks: array (nullable = true) | |-- element: string (containsNull = true) |-- possible_duplicate_ids: array (nullable = true) | |-- element: string (containsNull = true) |-- same_name_nearby_ids: array (nullable = true) | |-- element: string (containsNull = true) |-- nearest_product_m: integer (nullable = true) |-- grouped_units: integer (nullable = true) |-- osm_derived: boolean (nullable = true) |-- name_source: string (nullable = true) |-- location_source: string (nullable = true) |-- category_source: string (nullable = true) |-- website_source: string (nullable = true) |-- description_source: string (nullable = true) |-- opening_hours_source: string (nullable = true) |-- months_open_source: string (nullable = true) |-- seasonal_tag_source: string (nullable = true) |-- wheelchair_tag_source: string (nullable = true) |-- heritage_designations_source: string (nullable = true) |-- snapshot_date: date (nullable = true) |-- release_id: string (nullable = true) schema.simpleString: struct,category_rule:string,category_confidence:double,source_count:int,primary_source:string,primary_source_record_id:string,primary_source_url:string,checks:array,possible_duplicate_ids:array,same_name_nearby_ids:array,nearest_product_m:int,grouped_units:int,osm_derived:boolean,name_source:string,location_source:string,category_source:string,website_source:string,description_source:string,opening_hours_source:string,months_open_source:string,seasonal_tag_source:string,wheelchair_tag_source:string,heritage_designations_source:string,snapshot_date:date,release_id:string> [PASS] product.parquet row count = manifest: spark count 19846, manifest 19846 [PASS] product.parquet columns and types = data dictionary: same [PASS] product.parquet NOT NULL columns have no nulls: 30 columns checked ==== sample-release/product_source_xref.parquet ==== [2026-09-13T12:25:41+00:00] spark.read.parquet('sample-release/product_source_xref.parquet') [PASS] product_source_xref.parquet SHA-256 matches manifest: d244f981a53a413a53b5a619500e21313cb9c64070affb2f1d03ce2ec01725a2 root |-- ve_product_id: string (nullable = true) |-- prototype_id: string (nullable = true) |-- source: string (nullable = true) |-- source_record_id: string (nullable = true) |-- source_url: string (nullable = true) |-- licence: string (nullable = true) |-- licence_url: string (nullable = true) |-- retrieved_date: date (nullable = true) |-- last_edited_in_source: date (nullable = true) |-- checked_date: date (nullable = true) |-- link_method: string (nullable = true) |-- link_method_label: string (nullable = true) |-- match_score: double (nullable = true) |-- distance_m: integer (nullable = true) |-- is_primary: boolean (nullable = true) |-- snapshot_date: date (nullable = true) |-- release_id: string (nullable = true) schema.simpleString: struct [PASS] product_source_xref.parquet row count = manifest: spark count 22316, manifest 22316 [PASS] product_source_xref.parquet columns and types = data dictionary: same [PASS] product_source_xref.parquet NOT NULL columns have no nulls: 13 columns checked ==== sample-release/product_lineage_event.parquet ==== [2026-09-13T12:25:45+00:00] spark.read.parquet('sample-release/product_lineage_event.parquet') [PASS] product_lineage_event.parquet SHA-256 matches manifest: b2ef9cc448b161af4644b7af0575d70020a8cb201539f0764059229ddd6a5773 root |-- event_id: string (nullable = true) |-- release_id: string (nullable = true) |-- event_type: string (nullable = true) |-- event_date: date (nullable = true) |-- ve_product_id: string (nullable = true) |-- prototype_id: string (nullable = true) |-- related_ve_product_id: string (nullable = true) |-- related_prototype_id: string (nullable = true) |-- related_source: string (nullable = true) |-- related_source_record_id: string (nullable = true) |-- method: string (nullable = true) |-- method_label: string (nullable = true) |-- score: double (nullable = true) |-- distance_m: integer (nullable = true) |-- unit_count: integer (nullable = true) |-- note: string (nullable = true) |-- snapshot_date: date (nullable = true) schema.simpleString: struct [PASS] product_lineage_event.parquet row count = manifest: spark count 2808, manifest 2808 [PASS] product_lineage_event.parquet columns and types = data dictionary: same [PASS] product_lineage_event.parquet NOT NULL columns have no nulls: 10 columns checked ==== sample-release/lvep_membership.parquet ==== [2026-09-13T12:25:48+00:00] spark.read.parquet('sample-release/lvep_membership.parquet') [PASS] lvep_membership.parquet SHA-256 matches manifest: 821e5ca741f7c77dc45e47273a0c09e4a4332db1b38bbf0a34e7a7001857bcc6 root |-- lvep_id: string (nullable = true) |-- lvep_name: string (nullable = true) |-- lad_code: string (nullable = true) |-- lad_name: string (nullable = true) |-- lad_population: integer (nullable = true) |-- population_year: integer (nullable = true) |-- membership_version: string (nullable = true) |-- lad_vintage: string (nullable = true) |-- snapshot_date: date (nullable = true) |-- release_id: string (nullable = true) schema.simpleString: struct [PASS] lvep_membership.parquet row count = manifest: spark count 13, manifest 13 [PASS] lvep_membership.parquet columns and types = data dictionary: same [PASS] lvep_membership.parquet NOT NULL columns have no nulls: 10 columns checked ==== sample-release/product_attribute_provenance.parquet ==== [2026-09-13T12:25:52+00:00] spark.read.parquet('sample-release/product_attribute_provenance.parquet') [PASS] product_attribute_provenance.parquet SHA-256 matches manifest: 4b37d16c25ff112b25c5363337cc2c563a49ce0144f5b961ecd0cef82f4f8961 root |-- ve_product_id: string (nullable = true) |-- attribute: string (nullable = true) |-- value_index: integer (nullable = true) |-- source: string (nullable = true) |-- source_record_id: string (nullable = true) |-- method: string (nullable = true) |-- confidence: double (nullable = true) |-- last_verified_date: date (nullable = true) |-- retrieved_date: date (nullable = true) |-- snapshot_date: date (nullable = true) |-- release_id: string (nullable = true) schema.simpleString: struct [PASS] product_attribute_provenance.parquet row count = manifest: spark count 95921, manifest 95921 [PASS] product_attribute_provenance.parquet columns and types = data dictionary: same [PASS] product_attribute_provenance.parquet NOT NULL columns have no nulls: 7 columns checked ==== sample-release/product_relationship.parquet ==== [2026-09-13T12:25:55+00:00] spark.read.parquet('sample-release/product_relationship.parquet') [PASS] product_relationship.parquet SHA-256 matches manifest: 0b642ccf62a69bcf0760cd9dfeeeace43b8ae26869095e0c773b852ab24c226f root |-- relationship_id: string (nullable = true) |-- relationship_type: string (nullable = true) |-- ve_product_id: string (nullable = true) |-- related_ve_product_id: string (nullable = true) |-- prototype_id: string (nullable = true) |-- related_prototype_id: string (nullable = true) |-- distance_m: integer (nullable = true) |-- score: double (nullable = true) |-- method: string (nullable = true) |-- snapshot_date: date (nullable = true) |-- release_id: string (nullable = true) schema.simpleString: struct [PASS] product_relationship.parquet row count = manifest: spark count 3043, manifest 3043 [PASS] product_relationship.parquet columns and types = data dictionary: same [PASS] product_relationship.parquet NOT NULL columns have no nulls: 10 columns checked ==== sample-release/product_change_set.parquet ==== [2026-09-13T12:25:57+00:00] spark.read.parquet('sample-release/product_change_set.parquet') [PASS] product_change_set.parquet SHA-256 matches manifest: f42f27c8ce3c9356c19e4f981b9e48cb0885834538e0aeb67f3645017fbc293b root |-- change_id: string (nullable = true) |-- release_id: string (nullable = true) |-- previous_release_id: string (nullable = true) |-- change_type: string (nullable = true) |-- ve_product_id: string (nullable = true) |-- prototype_id: string (nullable = true) |-- attribute: string (nullable = true) |-- previous_value: string (nullable = true) |-- new_value: string (nullable = true) |-- note: string (nullable = true) |-- snapshot_date: date (nullable = true) schema.simpleString: struct [PASS] product_change_set.parquet row count = manifest: spark count 93, manifest 93 [PASS] product_change_set.parquet columns and types = data dictionary: same [PASS] product_change_set.parquet NOT NULL columns have no nulls: 7 columns checked ==== keys ==== [PASS] product.ve_product_id unique: 19846 distinct of 19846 [PASS] ve_product_id_registry.csv SHA-256 matches manifest [PASS] every product.ve_product_id is active in the id registry, and every active registry id is in product: products not active in registry 0; active ids not in product 0 [PASS] no retired registry id is used in product: retired ids in product 0 +-----------------------------+------+-----+ |minted_in_release |status|ids | +-----------------------------+------+-----+ |ve-audit-sample-r1-2026-09-12|active|19846| +-----------------------------+------+-----+ [PASS] product_attribute_provenance (ve_product_id, attribute, value_index) unique: duplicate keys 0 [PASS] every product has a name provenance row: missing 0 [PASS] every non-rule provenance row points to a product_source_xref record: orphans 0 [PASS] product_relationship.relationship_id unique: 3043 rows [PASS] product_change_set.change_id unique: 93 rows change set by type: {'updated': 93} [PASS] change set counts by type = manifest: spark {'updated': 93}, manifest {'inserted': 0, 'updated': 93, 'retired': 0} [PASS] change set: inserted and updated ids are in product, retired ids are not: inserted/updated not in product 0; retired still in product 0 +-----------------+----+ |attribute |rows| +-----------------+----+ |nearest_product_m|72 | |checks |21 | +-----------------+----+ [PASS] product_source_xref (source, source_record_id) unique: duplicate keys 0 [PASS] exactly one is_primary row per product in product_source_xref: products violating 0 [PASS] product.source_count = xref rows per product: mismatches 0 [PASS] product_lineage_event.event_id unique: 2808 distinct of 2808 lineage events by type: {'merge': 2470, 'review_candidate_kept_separate': 228, 'same_name_candidate': 108, 'unit_group': 2} [PASS] lineage event counts by type = manifest: spark {'merge': 2470, 'review_candidate_kept_separate': 228, 'same_name_candidate': 108, 'unit_group': 2}, manifest {'merge': 2470, 'review_candidate_kept_separate': 228, 'same_name_candidate': 108, 'unit_group': 2, 'split': 0, 'product_merge': 0, 'retire': 0} [PASS] merge events = non-primary xref rows: 2470 vs 2470 ==== foreign keys ==== [PASS] FK product_source_xref.ve_product_id -> product: orphans 0 [PASS] FK product_attribute_provenance.ve_product_id -> product: orphans 0 [PASS] FK product_relationship.ve_product_id -> product: orphans 0 [PASS] FK product_relationship.related_ve_product_id -> product: orphans 0 [PASS] FK product_lineage_event.ve_product_id -> product (except retire): orphans 0 [PASS] FK product_lineage_event.related_ve_product_id -> product (except product_merge, whose related id is retired): orphans 0 [PASS] FK product.lad_code -> lvep_membership: orphans 0 [PASS] product.lvep_id agrees with lvep_membership for lad_code: mismatches 0 +--------------------+-------+-----------+---------+ |lvep_id |records|osm_derived|dup_links| +--------------------+-------+-----------+---------+ |cornwall |5630 |3861 |132 | |greater-manchester |6021 |5055 |122 | |york-north-yorkshire|8195 |6202 |202 | +--------------------+-------+-----------+---------+ +------------+----------------+-----------------------------------------------------------------+----------+ |prototype_id|name |heritage_designations |checks | +------------+----------------+-----------------------------------------------------------------+----------+ |CW-00475 |Caerhays Castle |[Listed building, Grade I, Registered park and garden, Grade II*]|[] | |CW-01371 |Godolphin House |[Listed building, Grade I, Registered park and garden, Grade II*]|[no_hours]| |CW-01897 |Lanhydrock House|[Listed building, Grade I, Registered park and garden, Grade II*]|[no_hours]| +------------+----------------+-----------------------------------------------------------------+----------+ ==== CSV ==== [PASS] product.csv SHA-256 matches manifest: 44ab879f5a5fd895622315226cb0be4ef7c151d5edd5485b66016e34839f1414 [2026-09-13T12:26:33+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/product.csv') [PASS] product.csv row count = manifest: spark count 19846, manifest 19846 [PASS] product.csv header = Parquet columns: 48 columns [PASS] product.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] product_source_xref.csv SHA-256 matches manifest: d148980030b22fe6ec2d129b4bca9fd42e51f5d786a62205bcc2d3ddc8edffc9 [2026-09-13T12:26:35+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/product_source_xref.csv') [PASS] product_source_xref.csv row count = manifest: spark count 22316, manifest 22316 [PASS] product_source_xref.csv header = Parquet columns: 17 columns [PASS] product_source_xref.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] product_lineage_event.csv SHA-256 matches manifest: 62d20e70bcec4a4ef5cd3739e80592064f3767334d791810703d35eed22f8a59 [2026-09-13T12:26:36+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/product_lineage_event.csv') [PASS] product_lineage_event.csv row count = manifest: spark count 2808, manifest 2808 [PASS] product_lineage_event.csv header = Parquet columns: 17 columns [PASS] product_lineage_event.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] lvep_membership.csv SHA-256 matches manifest: 5e3a1c502de2cf9670d3b84b8f606fa096e2d21a1ca34e7df6c76ddf65f683a3 [2026-09-13T12:26:36+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/lvep_membership.csv') [PASS] lvep_membership.csv row count = manifest: spark count 13, manifest 13 [PASS] lvep_membership.csv header = Parquet columns: 10 columns [PASS] lvep_membership.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] product_attribute_provenance.csv SHA-256 matches manifest: 9ba6f4012d9edc92a0a193253e15f2e946026b1005a12d1e4a6959daa95bd4c2 [2026-09-13T12:26:38+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/product_attribute_provenance.csv') [PASS] product_attribute_provenance.csv row count = manifest: spark count 95921, manifest 95921 [PASS] product_attribute_provenance.csv header = Parquet columns: 11 columns [PASS] product_attribute_provenance.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] product_relationship.csv SHA-256 matches manifest: 42deba8c60beecb038609429bd44afec94cf7652f5cd7e89dffdbd97d58d60d8 [2026-09-13T12:26:39+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/product_relationship.csv') [PASS] product_relationship.csv row count = manifest: spark count 3043, manifest 3043 [PASS] product_relationship.csv header = Parquet columns: 11 columns [PASS] product_relationship.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] product_change_set.csv SHA-256 matches manifest: 56370f5a1216d0a4b7ddac3bc44e6ad64732ec465b487168c702b627ca5e339a [2026-09-13T12:26:41+00:00] spark.read.option('header', True).option('multiLine', True).option('escape', '"').csv('sample-release/product_change_set.csv') [PASS] product_change_set.csv row count = manifest: spark count 93, manifest 93 [PASS] product_change_set.csv header = Parquet columns: 11 columns [PASS] product_change_set.csv release_id values (no stray CR): ['ve-audit-sample-r3-2026-09-12'] [PASS] product.csv values = product.parquet (8 text and array columns, full outer join on ve_product_id): differing rows 0 [PASS] product_source_xref.csv keys = Parquet: differing rows 0 ==== DDL (ddl.sql) ==== [PASS] ddl.sql SHA-256 matches manifest statements in ddl.sql: 9 (7 CREATE TABLE; other: ['CREATE CATALOG IF NOT EXISTS ve_audit', 'CREATE SCHEMA IF NOT EXISTS ve_audit.sample']) substitution: 've_audit.sample.' -> 've_audit_sample_local.' (local Spark has no Unity Catalog three-part names) substitution: 'USING DELTA' -> 'USING PARQUET' (Delta Lake is not installed in local Spark) substitution: 'CREATE CATALOG IF NOT EXISTS ve_audit' not run (not supported by the local session catalog) substitution: 'CREATE SCHEMA IF NOT EXISTS ve_audit.sample' -> 'CREATE DATABASE IF NOT EXISTS ve_audit_sample_local' [2026-09-13T12:26:47+00:00] attempt 1: CREATE TABLE statements with substitutions, PRIMARY KEY / FOREIGN KEY constraints kept lvep_membership: failed with constraints: AnalysisException: [UNSUPPORTED_FEATURE.TABLE_OPERATION] The feature is not supported: Table `spark_catalog`.`ve_audit_sample_local`.`lvep_membership` does not support CONSTRAINT. Please check the current catalog and namespace to make sure the qualified table name is expected, and also check the catalog implementation which is configured by "spark.sql.catalog". SQLSTATE: 0A000 [2026-09-13T12:26:48+00:00] attempt 2: same substitutions, CONSTRAINT clauses removed created ve_audit_sample_local.lvep_membership (constraints removed) created ve_audit_sample_local.product (constraints removed) created ve_audit_sample_local.product_attribute_provenance (constraints removed) created ve_audit_sample_local.product_source_xref (constraints removed) created ve_audit_sample_local.product_relationship (constraints removed) created ve_audit_sample_local.product_lineage_event (constraints removed) created ve_audit_sample_local.product_change_set (constraints removed) [PASS] CREATE TABLE statements executed in local Spark: without CONSTRAINT clauses [PASS] DDL lvep_membership: column names, order and types = Parquet: same [PASS] DDL lvep_membership: column comments = data dictionary: 10 comments lvep_membership: columns non-nullable in the created table schema: 0 [PASS] DDL lvep_membership: Parquet rows inserted into the created table: 13 rows [PASS] DDL product: column names, order and types = Parquet: same [PASS] DDL product: column comments = data dictionary: 48 comments product: columns non-nullable in the created table schema: 0 [PASS] DDL product: Parquet rows inserted into the created table: 19846 rows [PASS] DDL product_attribute_provenance: column names, order and types = Parquet: same [PASS] DDL product_attribute_provenance: column comments = data dictionary: 11 comments product_attribute_provenance: columns non-nullable in the created table schema: 0 [PASS] DDL product_attribute_provenance: Parquet rows inserted into the created table: 95921 rows [PASS] DDL product_source_xref: column names, order and types = Parquet: same [PASS] DDL product_source_xref: column comments = data dictionary: 17 comments product_source_xref: columns non-nullable in the created table schema: 0 [PASS] DDL product_source_xref: Parquet rows inserted into the created table: 22316 rows [PASS] DDL product_relationship: column names, order and types = Parquet: same [PASS] DDL product_relationship: column comments = data dictionary: 11 comments product_relationship: columns non-nullable in the created table schema: 0 [PASS] DDL product_relationship: Parquet rows inserted into the created table: 3043 rows [PASS] DDL product_lineage_event: column names, order and types = Parquet: same [PASS] DDL product_lineage_event: column comments = data dictionary: 17 comments product_lineage_event: columns non-nullable in the created table schema: 0 [PASS] DDL product_lineage_event: Parquet rows inserted into the created table: 2808 rows [PASS] DDL product_change_set: column names, order and types = Parquet: same [PASS] DDL product_change_set: column comments = data dictionary: 11 comments product_change_set: columns non-nullable in the created table schema: 0 [PASS] DDL product_change_set: Parquet rows inserted into the created table: 93 rows +----------------------------+--------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ | col_name| data_type| comment| +----------------------------+--------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ | lvep_id| string| Sample LVEP identifier.| | lvep_name| string| Sample LVEP name.| | lad_code| string| ONS local authority district code (May 2026).| | lad_name| string| ONS local authority district name.| | lad_population| int| ONS mid-year population estimate for the local authority.| | population_year| int| Year of the mid-year population estimate.| | membership_version| string|Version of the illustrative LVEP to local authority membership table. Not Vis...| | lad_vintage| string| ONS boundary vintage the codes refer to.| | snapshot_date| date| Date of the data snapshot: sources retrieved and pipeline run on 2026-09-12.| | release_id| string| Identifier of the release this row belongs to.| | | | | |# Detailed Table Information| | | | Catalog| spark_catalog| | | Database| ve_audit_sample_local| | | Table| lvep_membership| | | Created Time| Sun Sep 13 12:26:48 UTC 2026| | | Last Access| UNKNOWN| | | Created By| Spark 4.2.0| | | Type| MANAGED| | | Provider| PARQUET| | | Comment|Illustrative membership of the sample LVEPs: which ONS local authority distri...| | | Location|file:/warehouse/ve_audit_sample_local.db/lv...| | +----------------------------+--------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ [PASS] no spark-warehouse, metastore_db or derby.log left in the repository: none Finished (UTC): 2026-09-13T12:27:03+00:00; 107 checks, 0 failed SUMMARY_JSON: {"all_checks_passed": true, "checks_failed": 0, "checks_total": 107, "counts": {"lvep_membership": {"csv": 13, "manifest": 13, "parquet": 13}, "product": {"csv": 19846, "manifest": 19846, "parquet": 19846}, "product_attribute_provenance": {"csv": 95921, "manifest": 95921, "parquet": 95921}, "product_change_set": {"csv": 93, "manifest": 93, "parquet": 93}, "product_lineage_event": {"csv": 2808, "manifest": 2808, "parquet": 2808}, "product_relationship": {"csv": 3043, "manifest": 3043, "parquet": 3043}, "product_source_xref": {"csv": 22316, "manifest": 22316, "parquet": 22316}}, "counts_match": true, "csv_ok": true, "ddl_check": {"comments_match": true, "constraints_accepted_locally": false, "constraints_note": "Local Spark rejected the informational PRIMARY KEY / FOREIGN KEY clauses (they need Unity Catalog), so they were removed for this check; their syntax was not validated locally.", "executed": true, "first_error_with_constraints": "lvep_membership: AnalysisException: [UNSUPPORTED_FEATURE.TABLE_OPERATION] The feature is not supported: Table `spark_catalog`.`ve_audit_sample_local`.`lvep_membership` does not support CONSTRAINT. Please check the current catalog and namespace to make sure the qualified table name is expected, and also check the catalog implementation which is configured by \"spark.sql.catalog\". SQLSTATE: 0A000", "insert_ok": true, "local_database": "ve_audit_sample_local", "schemas_match": true, "substitutions": ["'ve_audit.sample.' -> 've_audit_sample_local.' (local Spark has no Unity Catalog three-part names)", "'USING DELTA' -> 'USING PARQUET' (Delta Lake is not installed in local Spark)", "'CREATE CATALOG IF NOT EXISTS ve_audit' not run (not supported by the local session catalog)", "'CREATE SCHEMA IF NOT EXISTS ve_audit.sample' -> 'CREATE DATABASE IF NOT EXISTS ve_audit_sample_local'", "CONSTRAINT ... PRIMARY KEY / FOREIGN KEY ... NOT ENFORCED clauses removed (attempt 1 failed in local Spark, see log)"]}, "failed_checks": [], "files_sha256": {"ddl.sql": "7ced54343801ba00f7a564cb733e79b0b60b842847db06669e71cb95a7d34f94", "lvep_membership.csv": "5e3a1c502de2cf9670d3b84b8f606fa096e2d21a1ca34e7df6c76ddf65f683a3", "lvep_membership.parquet": "821e5ca741f7c77dc45e47273a0c09e4a4332db1b38bbf0a34e7a7001857bcc6", "product.csv": "44ab879f5a5fd895622315226cb0be4ef7c151d5edd5485b66016e34839f1414", "product.parquet": "e5606b71d95f3fb603a8465f768f434392ace9632f0b40d2af8b837038d98f28", "product_attribute_provenance.csv": "9ba6f4012d9edc92a0a193253e15f2e946026b1005a12d1e4a6959daa95bd4c2", "product_attribute_provenance.parquet": "4b37d16c25ff112b25c5363337cc2c563a49ce0144f5b961ecd0cef82f4f8961", "product_change_set.csv": "56370f5a1216d0a4b7ddac3bc44e6ad64732ec465b487168c702b627ca5e339a", "product_change_set.parquet": "f42f27c8ce3c9356c19e4f981b9e48cb0885834538e0aeb67f3645017fbc293b", "product_lineage_event.csv": "62d20e70bcec4a4ef5cd3739e80592064f3767334d791810703d35eed22f8a59", "product_lineage_event.parquet": "b2ef9cc448b161af4644b7af0575d70020a8cb201539f0764059229ddd6a5773", "product_relationship.csv": "42deba8c60beecb038609429bd44afec94cf7652f5cd7e89dffdbd97d58d60d8", "product_relationship.parquet": "0b642ccf62a69bcf0760cd9dfeeeace43b8ae26869095e0c773b852ab24c226f", "product_source_xref.csv": "d148980030b22fe6ec2d129b4bca9fd42e51f5d786a62205bcc2d3ddc8edffc9", "product_source_xref.parquet": "d244f981a53a413a53b5a619500e21313cb9c64070affb2f1d03ce2ec01725a2", "ve_product_id_registry.csv": "47c9f259dd98787d471d836fbd16f0b9d749c6d0201cf176a9907b37a7c67f3d"}, "finished": "2026-09-13T12:27:03+00:00", "foreign_keys_ok": true, "java_version": "21.0.12", "master": "local[*]", "pyarrow_version": "25.0.1", "pyspark_version": "4.2.0", "python_version": "3.12.3", "release_id": "ve-audit-sample-r3-2026-09-12", "spark_version": "4.2.0", "ve_product_id_in_registry": true, "ve_product_id_unique": true}