{ "release_id": "ve-audit-sample-r3-2026-09-12", "snapshot_date": "2026-09-12", "catalog": "ve_audit", "schema": "sample", "title": "Data dictionary: visitor product inventory sample release (illustrative, open data only)", "conventions": { "naming": "lower_snake_case table and column names, ready for Unity Catalog.", "types": "spark_sql_type is the Spark SQL / Databricks SQL type. Parquet files use the matching physical types (INT = int32, DATE = date32, ARRAY = list).", "nulls": "Nullable columns are null when the source has no value. Array columns are never null: an empty array means none.", "csv": "UTF-8 without BOM, header row, comma separator, CRLF line endings, RFC 4180 quoting (fields with commas or quotes are quoted; quotes doubled). Null and empty arrays are empty fields. Arrays are joined with '; '. Booleans are true/false. Dates are YYYY-MM-DD. In Spark read with header=true, multiLine=true, escape='\"'.", "keys": "Primary and foreign keys are informational (not enforced). Each table holds one release; to keep several releases in one table, add release_id to the keys.", "identifiers": "Use ve_product_id: a surrogate key read from the id registry (ve_product_id_registry.csv), never recomputed. prototype_id is the identifier shown in the web prototype and is renumbered at every build.", "row_order": "Rows are sorted as listed in sort_order so that files are byte-for-byte reproducible.", "examples": "example is the first non-null value of the column in file order." }, "enums": { "kind": { "visitor_product": "At least one OpenStreetMap or Wikidata member classified in a visitor product category", "heritage_asset": "Designated or recorded heritage with no visitor product evidence in the sources" }, "product_role": { "core_visitor_product": "Core visitor product: attractions, heritage sites, museums & galleries, arts, outdoors, activities (not a chain outlet)", "supporting_amenity": "Supporting amenity: food & drink, accommodation, visitor information, or any chain outlet", "for_review": "For review: a ferry terminal, which open data cannot tell apart as a scenic crossing or boat trip (visitor product) or a commuter link (amenity)", "heritage_asset": "Designated or recorded heritage asset with no visitor product evidence" }, "category_key": { "attractions": "Attractions", "heritage": "Heritage sites", "museums": "Museums & galleries", "arts": "Arts, entertainment & events", "outdoors": "Outdoors & nature", "activities": "Activities & sport", "food": "Food & drink", "accommodation": "Accommodation", "info_travel": "Visitor information & travel", "heritage_asset": "Designated or recorded heritage (no visitor product evidence)" }, "source": { "osm": "OpenStreetMap (Open Database Licence (ODbL) 1.0)", "wd": "Wikidata (Creative Commons CC0 1.0)", "nhle": "National Heritage List for England (Open Government Licence v3.0)" }, "link_method": { "primary": "Primary record for this product", "wikidata_tag": "Linked by shared Wikidata ID (OSM wikidata tag)", "he_ref_tag": "Linked by shared NHLE list entry number (OSM HE_ref tag)", "p1216": "Linked by shared NHLE list entry number (Wikidata P1216)", "osm_dup": "Duplicate OSM feature: same name and category within 75 m", "name_rule": "Name + distance + category rule, auto-merge band (score >= 0.85)", "nhle_name": "Name match (>= 0.90) within 150 m (400 m for landscapes)" }, "months_open_method": { "oh_months": "Month rules in OSM opening_hours (stated)", "seasonal_tag": "OSM seasonal tag (stated)", "oh_all_year": "OSM opening_hours without month rules, or 24/7, read as open all year (inferred)" }, "seasonality_evidence": { "stated": "The source states which months", "inferred": "All-year opening assumed from hours without month rules" }, "checks": { "no_website": "Visitor product with no website in OpenStreetMap or Wikidata", "no_hours": "Attraction, museum, arts, activity or food record with no OpenStreetMap opening_hours", "single_source": "All source records of the product come from one source", "low_category_confidence": "Category rule confidence below 0.75", "stale_osm": "First OpenStreetMap record last edited or checked more than 3 years before the build", "possible_duplicate": "Another record scored in the review band (0.6 to 0.85); kept separate", "near_boundary": "Location just outside the generalised local authority boundary; assigned to the nearest authority", "latent_asset": "Heritage asset with no visitor product record within 500 m. World Heritage Sites, registered parks and gardens and registered battlefields are areas: a product inside the Historic England boundary or within 500 m of it counts (distance to the boundary, 0 inside). An area entry without a boundary in the raw data keeps the point rule.", "same_name_nearby": "Another record with the same normalised name within 150 m, not linked" }, "event_type": { "merge": { "definition": "A non-primary source record merged into a product (one row per product_source_xref row with is_primary = false).", "rows_in_release": 2470 }, "review_candidate_kept_separate": { "definition": "Two products scored in the review band and kept separate (one row per unordered pair).", "rows_in_release": 228 }, "same_name_candidate": { "definition": "Two products with the same normalised name close together, not linked (one row per unordered pair).", "rows_in_release": 108 }, "unit_group": { "definition": "OpenStreetMap unit features (numbered or lettered) grouped into one site record.", "rows_in_release": 2 }, "split": { "definition": "One ve_product_id from a previous release becomes two or more products in this release. ve_product_id holds the previous id, which stays on the part with most of its source records, and related_ve_product_id each new id.", "rows_in_release": 0 }, "product_merge": { "definition": "Two products from a previous release are one product in this release. ve_product_id holds the id that is kept; related_ve_product_id the retired id (status retired, replaced_by in ve_product_id_registry.csv).", "rows_in_release": 0 }, "retire": { "definition": "A ve_product_id from a previous release has no source record in this release; it is retired, not reused.", "rows_in_release": 0 } }, "attribute": { "name": "Product or asset name", "location": "Latitude and longitude", "category": "category_key, type_in_source, category_rule and category_confidence", "website": "Website", "description": "Description", "opening_hours": "OpenStreetMap opening_hours", "months_open": "months_open, months_open_method and seasonality_evidence", "seasonal_tag": "OpenStreetMap seasonal tag", "wheelchair_tag": "OpenStreetMap wheelchair tag", "heritage_designations": "One row per linked NHLE entry", "product_role": "Product role (rule)", "chain_brand": "Chain brand (rule)" }, "relationship_type": { "nearest_visitor_product": "Heritage asset to its nearest visitor product record, within 1,000 m", "possible_duplicate": "Two products scored in the review band (0.60 to 0.85) and kept separate for a reviewer", "same_name_nearby": "Two products with the same normalised name close together, not linked by the matcher" }, "change_type": { "inserted": "ve_product_id not in the previous release", "updated": "One row per compared column that changed", "retired": "ve_product_id in the previous release and not in this one" }, "product_role_note": "for_review: a ferry terminal; open data cannot tell a scenic crossing or boat trip from a commuter link, so no role is set by rule." }, "tables": [ { "name": "product", "full_name": "ve_audit.sample.product", "description": "One row per product record in the prototype inventory for 3 sample LVEPs: visitor products and heritage assets built from open data (OpenStreetMap, Wikidata, Historic England NHLE) with rules-only matching. Illustrative sample, not VisitEngland data.", "rows": 19846, "files": [ "product.parquet", "product.csv" ], "primary_key": [ "ve_product_id" ], "foreign_keys": [ { "columns": [ "lad_code" ], "references_table": "lvep_membership", "references_columns": [ "lad_code" ] } ], "sort_order": [ "lvep_id", "prototype_id" ], "columns": [ { "name": "ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "Persistent product identifier (surrogate key). Use this key to join tables and to track products between releases, not prototype_id.", "derivation": "Read from the id registry (pipeline/assign_ids.py, data/registry; published as ve_product_id_registry.csv). Minted once, when the product is first published, and never recomputed: every source record of the product is stored against it, so the id stays when a different source record becomes primary. A merge keeps the id holding most source records and retires the other (product_merge event); a split keeps the id on the part holding most of its source records and mints a new one for the other part (split event). Release 1 ids were minted from the SHA-256 of the then-primary source key; that formula is not applied again.", "example": "VEP-213f7c664e7a" }, { "name": "prototype_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier shown in the web prototype (CW, GM or YN prefix and a sequence number). Not stable between builds.", "derivation": "records-.json field id: build.py sorts each LVEP by kind, normalised name and primary source id and numbers the records from 00001. Any new or removed record renumbers the rest.", "example": "CW-00001" }, { "name": "name", "spark_sql_type": "STRING", "nullable": false, "description": "Product or heritage asset name.", "derivation": "First non-empty name from OpenStreetMap, then Wikidata, then NHLE (see name_source). NHLE names written in capitals are converted to title case. Unit sites get a generated name such as 'Chalet site (117 numbered units)'; a lone OSM unit named only with a number gets its type added ('Apartment 34').", "example": "1 Pengelly Court" }, { "name": "kind", "spark_sql_type": "STRING", "nullable": false, "description": "visitor_product or heritage_asset.", "derivation": "records field kind: 'product' -> visitor_product (at least one OSM or Wikidata member classified in a visitor product category); 'asset' -> heritage_asset (only NHLE entries and/or Wikidata items classed as recorded sites with no website).", "example": "visitor_product" }, { "name": "category_key", "spark_sql_type": "STRING", "nullable": false, "description": "Category key: attractions, heritage, museums, arts, outdoors, activities, food, accommodation, info_travel, heritage_asset.", "derivation": "records field cat. Visitor products take the category of the member with the highest rule confidence (OSM wins ties); heritage assets are always heritage_asset. Rules are listed in build.py (OSM_RULES, WD_CATEGORY).", "example": "accommodation" }, { "name": "category_label", "spark_sql_type": "STRING", "nullable": false, "description": "Human-readable category label.", "derivation": "meta.json categories[].label for category_key.", "example": "Accommodation" }, { "name": "product_role", "spark_sql_type": "STRING", "nullable": false, "description": "core_visitor_product, supporting_amenity or heritage_asset.", "derivation": "records field role from postprocess_review.py: heritage_asset for kind asset; supporting_amenity for food, accommodation, info_travel or any chain outlet; otherwise core_visitor_product. A rule for review, not a judgement.", "example": "supporting_amenity" }, { "name": "chain_brand", "spark_sql_type": "STRING", "nullable": true, "description": "Chain brand when the record is detected as a chain outlet; otherwise null.", "derivation": "records field chain: normalised name equals a brand name in pipeline/chains.json (version illustrative-2026-09-12) or starts with it followed by a space, and the category is one the brand operates in. Outlets named differently from the brand are not detected.", "example": "Asda café" }, { "name": "type_in_source", "spark_sql_type": "STRING", "nullable": false, "description": "Type label taken from the source record that set the category.", "derivation": "records field type: OSM tag value as a label (tourism=chalet -> 'Chalet'), Wikidata instance-of (P31) class labels, or for heritage assets the NHLE designation and grade.", "example": "Chalet" }, { "name": "lvep_id", "spark_sql_type": "STRING", "nullable": false, "description": "Sample LVEP identifier.", "derivation": "records-.json file the record comes from; equals lvep_membership.lvep_id for lad_code.", "example": "cornwall" }, { "name": "lvep_name", "spark_sql_type": "STRING", "nullable": false, "description": "Sample LVEP name.", "derivation": "meta.json lveps[].name.", "example": "Cornwall" }, { "name": "lad_code", "spark_sql_type": "STRING", "nullable": false, "description": "ONS local authority district code (May 2026) that contains the record location.", "derivation": "records field lad: point-in-polygon on ONS LAD boundaries (BGC). Points within about 300 m outside the generalised coastline are nudged in and carry the check near_boundary.", "example": "E06000052" }, { "name": "lad_name", "spark_sql_type": "STRING", "nullable": false, "description": "ONS local authority district name.", "derivation": "meta.json lveps[].local_authorities[].name.", "example": "Cornwall" }, { "name": "latitude", "spark_sql_type": "DOUBLE", "nullable": false, "description": "Latitude, WGS84 decimal degrees (6 decimal places).", "derivation": "records field lat: location of the first OSM member, else Wikidata (P625), else NHLE (see location_source). OSM ways and relations use the Overpass centre; unit sites use the mean of the unit locations.", "example": 50.077261 }, { "name": "longitude", "spark_sql_type": "DOUBLE", "nullable": false, "description": "Longitude, WGS84 decimal degrees (6 decimal places).", "derivation": "records field lon; as latitude.", "example": -5.70287 }, { "name": "website", "spark_sql_type": "STRING", "nullable": true, "description": "Website of the product, as recorded in the source.", "derivation": "records field web: Wikidata official website (P856) first, then OSM website, contact:website or url tag (see website_source).", "example": "http://www.10treverbynroad.co.uk/" }, { "name": "description", "spark_sql_type": "STRING", "nullable": true, "description": "Short English description.", "derivation": "records field desc: Wikidata English description (schema:description).", "example": "hotel in Penzance, Cornwall, England, UK" }, { "name": "opening_hours", "spark_sql_type": "STRING", "nullable": true, "description": "OpenStreetMap opening_hours tag, verbatim.", "derivation": "records field oh.", "example": "Mo-Sa 08:00-16:00" }, { "name": "months_open", "spark_sql_type": "STRING", "nullable": true, "description": "Months open as 12 characters for January to December: 1 open, 0 closed. Null when there is no month evidence.", "derivation": "records field months: parsed by build.py months_from_osm from OSM opening_hours month rules, or the OSM seasonal tag when opening_hours has no month rule.", "example": "111111111111" }, { "name": "months_open_method", "spark_sql_type": "STRING", "nullable": true, "description": "How months_open was derived: oh_months, seasonal_tag or oh_all_year.", "derivation": "records field mm. oh_months: month rules in opening_hours; seasonal_tag: OSM seasonal tag; oh_all_year: opening_hours without any month rule (or 24/7), read as open all year.", "example": "oh_all_year" }, { "name": "seasonality_evidence", "spark_sql_type": "STRING", "nullable": true, "description": "stated, inferred or null.", "derivation": "From months_open_method: oh_months and seasonal_tag -> stated (the source states months); oh_all_year -> inferred (all-year opening assumed from hours without month rules); no months_open -> null.", "example": "inferred" }, { "name": "seasonal_tag", "spark_sql_type": "STRING", "nullable": true, "description": "OpenStreetMap seasonal tag, verbatim.", "derivation": "records field seas.", "example": "yes" }, { "name": "wheelchair_tag", "spark_sql_type": "STRING", "nullable": true, "description": "OpenStreetMap wheelchair tag, verbatim (yes, no, limited, designated).", "derivation": "records field wc.", "example": "no" }, { "name": "heritage_designations", "spark_sql_type": "ARRAY", "nullable": false, "description": "Historic England designations and grades of linked NHLE entries, sorted. Empty array when none.", "derivation": "records field desig (NHLE designation + ', Grade ' + grade). CSV: values joined with '; '.", "example": [ "Listed building, Grade II*" ] }, { "name": "category_rule", "spark_sql_type": "STRING", "nullable": false, "description": "Rule that set the category.", "derivation": "records field rule (build.py; unit grouping appends a note).", "example": "OSM tag tourism=chalet" }, { "name": "category_confidence", "spark_sql_type": "DOUBLE", "nullable": false, "description": "Fixed confidence of the category rule, 0 to 1.", "derivation": "records field conf: 0.95 specific OSM tag, 0.8 any historic=* tag, 0.7 tourism=attraction or Wikidata tourist attraction, 0.9 other Wikidata class, 0.75 Wikidata recorded site with no website, 1.0 NHLE designation.", "example": 0.95 }, { "name": "source_count", "spark_sql_type": "INT", "nullable": false, "description": "Number of source records merged into this product (rows in product_source_xref).", "derivation": "len(records field src).", "example": 1 }, { "name": "primary_source", "spark_sql_type": "STRING", "nullable": false, "description": "Source of the primary record: osm, wd or nhle.", "derivation": "records field src[] entry with m = 'primary': first visitor-product member ordered by source (OSM, Wikidata, NHLE) and source record key; for heritage assets the first member in the same order.", "example": "osm" }, { "name": "primary_source_record_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the primary record in its source: OSM node/, way/ or relation/; Wikidata QID; NHLE list entry number.", "derivation": "records field src[].id of the primary entry.", "example": "way/993980541" }, { "name": "primary_source_url", "spark_sql_type": "STRING", "nullable": false, "description": "Link to the primary record at its source.", "derivation": "meta.json sources[].url_template with {id} replaced by primary_source_record_id.", "example": "https://www.openstreetmap.org/way/993980541" }, { "name": "checks", "spark_sql_type": "ARRAY", "nullable": false, "description": "Automatic checks raised for the record, in pipeline order. Empty array when none.", "derivation": "records field flags (build.py and postprocess_review.py). Values and meanings: data_dictionary.json enums.checks. CSV: values joined with '; '.", "example": [ "no_website", "single_source", "stale_osm" ] }, { "name": "possible_duplicate_ids", "spark_sql_type": "ARRAY", "nullable": false, "description": "ve_product_id of records scored in the review band (0.60 to 0.85) against this one and kept separate. Empty array when none.", "derivation": "records field dups (prototype ids) mapped to ve_product_id. Each pair is also one review_candidate_kept_separate row in product_lineage_event. CSV: values joined with '; '.", "example": [ "VEP-57c1d2333a2c" ] }, { "name": "same_name_nearby_ids", "spark_sql_type": "ARRAY", "nullable": false, "description": "ve_product_id of records with the same normalised name within 150 m that the matcher did not link. Empty array when none.", "derivation": "records field sn (prototype ids) mapped to ve_product_id; postprocess_review.py. Each pair is also one same_name_candidate row in product_lineage_event. CSV: values joined with '; '.", "example": [ "VEP-b1c1d6798779" ] }, { "name": "nearest_product_m", "spark_sql_type": "INT", "nullable": true, "description": "Heritage assets only: straight-line distance in metres to the nearest visitor product record, when within 1,000 m. For World Heritage Sites, registered parks and gardens and battlefields, distance from the site boundary (0 when a product is inside).", "derivation": "records field near_m (build.py; area_rule.py for area designations). Null for visitor products and for assets with no product within 1,000 m.", "example": 5 }, { "name": "grouped_units", "spark_sql_type": "INT", "nullable": true, "description": "Number of OpenStreetMap unit features grouped into this site record; null otherwise.", "derivation": "records field units (build.py group_numbered_units). See unit_group rows in product_lineage_event.", "example": 117 }, { "name": "osm_derived", "spark_sql_type": "BOOLEAN", "nullable": false, "description": "True when any value in the row, or any merged source record, comes from OpenStreetMap (ODbL 1.0 layer).", "derivation": "True if any *_source column is osm, months_open is not null, or any product_source_xref row for the product has source osm.", "example": true }, { "name": "name_source", "spark_sql_type": "STRING", "nullable": false, "description": "Source that supplied name: osm, wd or nhle.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "location_source", "spark_sql_type": "STRING", "nullable": false, "description": "Source that supplied latitude and longitude: osm, wd or nhle.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "category_source", "spark_sql_type": "STRING", "nullable": false, "description": "Source that supplied category_key, type_in_source, category_rule and category_confidence: osm, wd or nhle.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "website_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied website: osm, wd or nhle; null when website is null.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "description_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied description: osm, wd or nhle; null when description is null.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "wd" }, { "name": "opening_hours_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied opening_hours: osm, wd or nhle; null when opening_hours is null.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "months_open_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied months_open: osm; null when months_open is null.", "derivation": "Always osm when months_open is set (build.py derives months from OSM tags only).", "example": "osm" }, { "name": "seasonal_tag_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied seasonal_tag: osm, wd or nhle; null when seasonal_tag is null.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "wheelchair_tag_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied wheelchair_tag: osm, wd or nhle; null when wheelchair_tag is null.", "derivation": "records field fs (field-level provenance) written by build.py.", "example": "osm" }, { "name": "heritage_designations_source", "spark_sql_type": "STRING", "nullable": true, "description": "Source that supplied heritage_designations: osm, wd or nhle; null when heritage_designations is null.", "derivation": "records field fs (field-level provenance) written by build.py; null when the array is empty.", "example": "nhle" }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot: sources retrieved and pipeline run on 2026-09-12.", "derivation": "meta.json generated (date part). Same value on every row.", "example": "2026-09-12" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the release this row belongs to.", "derivation": "Fixed for this release: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" } ] }, { "name": "product_attribute_provenance", "full_name": "ve_audit.sample.product_attribute_provenance", "description": "One row per attribute value of a product: the source record that supplied it, the method, the rule confidence where one exists, and the last edit or check date of that record.", "rows": 95921, "files": [ "product_attribute_provenance.parquet", "product_attribute_provenance.csv" ], "primary_key": [ "ve_product_id", "attribute", "value_index" ], "foreign_keys": [ { "columns": [ "ve_product_id" ], "references_table": "product", "references_columns": [ "ve_product_id" ] } ], "sort_order": [ "prototype_id of the product", "attribute (enum order)", "value_index" ], "columns": [ { "name": "ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "Product the attribute value belongs to.", "derivation": "product.ve_product_id.", "example": "VEP-213f7c664e7a" }, { "name": "attribute", "spark_sql_type": "STRING", "nullable": false, "description": "Product column the row describes. Values: data_dictionary.json enums.attribute.", "derivation": "One row for each product column below that has a value: name, location (latitude and longitude), category (category_key, type_in_source, category_rule, category_confidence), website, description, opening_hours, months_open (with months_open_method and seasonality_evidence), seasonal_tag, wheelchair_tag, heritage_designations (one row per linked NHLE entry), product_role, chain_brand.", "example": "name" }, { "name": "value_index", "spark_sql_type": "INT", "nullable": false, "description": "0 for single values; for heritage_designations, the position of the NHLE entry among the linked entries (0, 1, ...).", "derivation": "Order of the NHLE entries in product_source_xref.", "example": 0 }, { "name": "source", "spark_sql_type": "STRING", "nullable": false, "description": "Where the value came from: osm, wd or nhle for a source record; rule for values set by a pipeline rule.", "derivation": "records field fs (field-level source) written by build.py; rule for product_role and chain_brand (postprocess_review.py).", "example": "osm" }, { "name": "source_record_id", "spark_sql_type": "STRING", "nullable": true, "description": "Identifier of the source record that supplied the value; null when source is rule.", "derivation": "The only record of that source in the product, or records field fr (build.py) when the product holds several records of that source. Joins product_source_xref on (source, source_record_id).", "example": "way/993980541" }, { "name": "method", "spark_sql_type": "STRING", "nullable": false, "description": "How the value was obtained: the tag, property or rule.", "derivation": "Fixed text per attribute and source (for example OSM website, contact:website or url tag; Wikidata official website P856); for category the category rule; for months_open the parsing method.", "example": "OSM name tag" }, { "name": "confidence", "spark_sql_type": "DOUBLE", "nullable": true, "description": "Category only: fixed confidence of the category rule, 0 to 1, set per rule and not calibrated against labels. Null for every other attribute, where no confidence was measured.", "derivation": "records field conf.", "example": 0.95 }, { "name": "last_verified_date", "spark_sql_type": "DATE", "nullable": true, "description": "Latest date the source record was edited or checked in its source (OSM timestamp or check_date, Wikidata modified date, NHLE amendment date). Not a human verification. Null for rule values.", "derivation": "max(product_source_xref.last_edited_in_source, product_source_xref.checked_date) of the source record.", "example": "2021-10-19" }, { "name": "retrieved_date", "spark_sql_type": "DATE", "nullable": true, "description": "Date the source record was retrieved; null for rule values.", "derivation": "product_source_xref.retrieved_date.", "example": "2026-09-12" }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot: sources retrieved and pipeline run on 2026-09-12.", "derivation": "meta.json generated (date part). Same value on every row.", "example": "2026-09-12" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the release this row belongs to.", "derivation": "Fixed for this release: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" } ] }, { "name": "product_source_xref", "full_name": "ve_audit.sample.product_source_xref", "description": "One row per source record merged into a product, with source link, licence, dates and how it was linked.", "rows": 22316, "files": [ "product_source_xref.parquet", "product_source_xref.csv" ], "primary_key": [ "source", "source_record_id" ], "foreign_keys": [ { "columns": [ "ve_product_id" ], "references_table": "product", "references_columns": [ "ve_product_id" ] } ], "sort_order": [ "prototype_id", "is_primary desc", "source (osm, wd, nhle)", "source_record_id" ], "columns": [ { "name": "ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "Product the source record is merged into.", "derivation": "product.ve_product_id.", "example": "VEP-213f7c664e7a" }, { "name": "prototype_id", "spark_sql_type": "STRING", "nullable": false, "description": "Prototype identifier of the product (not stable).", "derivation": "product.prototype_id.", "example": "CW-00001" }, { "name": "source", "spark_sql_type": "STRING", "nullable": false, "description": "Source key: osm, wd or nhle.", "derivation": "records field src[].s.", "example": "osm" }, { "name": "source_record_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier in the source: OSM node/, way/ or relation/; Wikidata QID; NHLE list entry number.", "derivation": "records field src[].id. (source, source_record_id) is unique across the release.", "example": "way/993980541" }, { "name": "source_url", "spark_sql_type": "STRING", "nullable": false, "description": "Link to the record at its source.", "derivation": "meta.json sources[].url_template with {id} replaced.", "example": "https://www.openstreetmap.org/way/993980541" }, { "name": "licence", "spark_sql_type": "STRING", "nullable": false, "description": "Licence of the source.", "derivation": "meta.json sources[].licence.", "example": "Open Database Licence (ODbL) 1.0" }, { "name": "licence_url", "spark_sql_type": "STRING", "nullable": false, "description": "Licence URL.", "derivation": "meta.json sources[].licence_url.", "example": "https://opendatacommons.org/licenses/odbl/1-0/" }, { "name": "retrieved_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date the record was retrieved from the source.", "derivation": "records field src[].rt.", "example": "2026-09-12" }, { "name": "last_edited_in_source", "spark_sql_type": "DATE", "nullable": true, "description": "Last edit date in the source: OSM element timestamp, Wikidata modified date, NHLE amendment or list date.", "derivation": "records field src[].ed. For unit sites, the latest edit among the grouped units.", "example": "2021-10-19" }, { "name": "checked_date", "spark_sql_type": "DATE", "nullable": true, "description": "OSM check_date or survey:date tag, when present.", "derivation": "records field src[].ck.", "example": "2022-11-05" }, { "name": "link_method", "spark_sql_type": "STRING", "nullable": false, "description": "How the record was linked: primary, wikidata_tag, he_ref_tag, p1216, osm_dup, name_rule or nhle_name.", "derivation": "records field src[].m. For 29 osm_dup rows the pipeline did not keep the pairwise link; the method is assigned from the source type and match_score and distance_m are null.", "example": "primary" }, { "name": "link_method_label", "spark_sql_type": "STRING", "nullable": false, "description": "Plain description of link_method.", "derivation": "meta.json match_methods.", "example": "Primary record for this product" }, { "name": "match_score", "spark_sql_type": "DOUBLE", "nullable": true, "description": "Score of the link, 0 to 1: 1.0 for shared identifiers and duplicate OSM features, the rule score for name_rule, name similarity for nhle_name. Null for primary rows.", "derivation": "records field src[].sc.", "example": 1.0 }, { "name": "distance_m", "spark_sql_type": "INT", "nullable": true, "description": "Straight-line distance in metres between this record and the record it was linked to. Null for primary rows.", "derivation": "records field src[].d. The linked-to record is not always the primary record.", "example": 13 }, { "name": "is_primary", "spark_sql_type": "BOOLEAN", "nullable": false, "description": "True for the primary record of the product (exactly one per product).", "derivation": "link_method = primary.", "example": true }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot: sources retrieved and pipeline run on 2026-09-12.", "derivation": "meta.json generated (date part). Same value on every row.", "example": "2026-09-12" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the release this row belongs to.", "derivation": "Fixed for this release: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" } ] }, { "name": "product_relationship", "full_name": "ve_audit.sample.product_relationship", "description": "Relationships between products in this release: the nearest visitor product of each heritage asset within 1,000 m, and the possible duplicate and same-name pairs kept separate for a reviewer.", "rows": 3043, "files": [ "product_relationship.parquet", "product_relationship.csv" ], "primary_key": [ "relationship_id" ], "foreign_keys": [ { "columns": [ "ve_product_id" ], "references_table": "product", "references_columns": [ "ve_product_id" ] }, { "columns": [ "related_ve_product_id" ], "references_table": "product", "references_columns": [ "ve_product_id" ] } ], "sort_order": [ "relationship_type (enum order)", "prototype_id", "related_prototype_id" ], "columns": [ { "name": "relationship_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the relationship row, unique within the release.", "derivation": "'VER-' + first 16 hexadecimal characters of SHA-256 over 'release_id|relationship_type|ve_product_id|related_ve_product_id'.", "example": "VER-335ef309940a213e" }, { "name": "relationship_type", "spark_sql_type": "STRING", "nullable": false, "description": "nearest_visitor_product, possible_duplicate or same_name_nearby. Definitions: data_dictionary.json enums.relationship_type.", "derivation": "nearest_visitor_product: build.py nearest visitor product record to a heritage asset, within 1,000 m (area designations: area_rule.py, from the site boundary); possible_duplicate: records field dups; same_name_nearby: records field sn (postprocess_review.py). One row per unordered pair.", "example": "nearest_visitor_product" }, { "name": "ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "Product the relationship starts from: the heritage asset for nearest_visitor_product, otherwise the lower ve_product_id of the pair.", "derivation": "product.ve_product_id.", "example": "VEP-2f4bf8b36d48" }, { "name": "related_ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "The other product.", "derivation": "product.ve_product_id.", "example": "VEP-fdf411012169" }, { "name": "prototype_id", "spark_sql_type": "STRING", "nullable": false, "description": "Prototype identifier of ve_product_id (not stable).", "derivation": "product.prototype_id.", "example": "CW-04199" }, { "name": "related_prototype_id", "spark_sql_type": "STRING", "nullable": false, "description": "Prototype identifier of related_ve_product_id (not stable).", "derivation": "product.prototype_id.", "example": "CW-03093" }, { "name": "distance_m", "spark_sql_type": "INT", "nullable": false, "description": "Straight-line distance in metres between the two records; for an area designation in nearest_visitor_product, from the site boundary (0 inside).", "derivation": "nearest_visitor_product: product.nearest_product_m of the asset; pairs: records fields dups[].d and sn[].d.", "example": 5 }, { "name": "score", "spark_sql_type": "DOUBLE", "nullable": true, "description": "possible_duplicate only: match score of the pair, 0.60 to 0.85; null otherwise.", "derivation": "records field dups[].sc.", "example": 0.784 }, { "name": "method", "spark_sql_type": "STRING", "nullable": false, "description": "Rule that produced the relationship.", "derivation": "Fixed text per relationship_type, with the thresholds from meta.json.", "example": "Nearest visitor product record to a heritage asset, straight line, within 1,000 m (latent_asset beyond 500 m)" }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot: sources retrieved and pipeline run on 2026-09-12.", "derivation": "meta.json generated (date part). Same value on every row.", "example": "2026-09-12" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the release this row belongs to.", "derivation": "Fixed for this release: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" } ] }, { "name": "product_lineage_event", "full_name": "ve_audit.sample.product_lineage_event", "description": "Merge and split events and review candidates recorded in this release, one row per event, including id registry events (split, product_merge, retire).", "rows": 2808, "files": [ "product_lineage_event.parquet", "product_lineage_event.csv" ], "primary_key": [ "event_id" ], "foreign_keys": [ { "columns": [ "ve_product_id" ], "references_table": "product", "references_columns": [ "ve_product_id" ] }, { "columns": [ "related_ve_product_id" ], "references_table": "product", "references_columns": [ "ve_product_id" ] } ], "sort_order": [ "event_type (enum order)", "prototype_id", "related_prototype_id", "related_source_record_id" ], "columns": [ { "name": "event_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the event, unique within the release.", "derivation": "'VEE-' + first 16 hexadecimal characters of SHA-256 over 'release_id|event_type|ve_product_id|related key', where the related key is related_ve_product_id, or 'source:source_record_id', or 'units'.", "example": "VEE-5ee9bbb3ea1d8d5e" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Release in which the event was recorded.", "derivation": "Fixed: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" }, { "name": "event_type", "spark_sql_type": "STRING", "nullable": false, "description": "merge, review_candidate_kept_separate, same_name_candidate, unit_group, split, product_merge or retire. Definitions: data_dictionary.json enums.event_type.", "derivation": "merge: non-primary product_source_xref rows; review_candidate_kept_separate: records field dups, one row per unordered pair; same_name_candidate: records field sn, one row per unordered pair; unit_group: records field units; split, product_merge and retire: id registry events of this release (data/registry/after-r/id_events.json).", "example": "merge" }, { "name": "event_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date the event was recorded (the snapshot date of the release).", "derivation": "meta.json generated (date part).", "example": "2026-09-12" }, { "name": "ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "Product the event applies to. For pair events, the lower ve_product_id of the pair.", "derivation": "product.ve_product_id.", "example": "VEP-527ca504e1bb" }, { "name": "prototype_id", "spark_sql_type": "STRING", "nullable": false, "description": "Prototype identifier of ve_product_id (not stable).", "derivation": "product.prototype_id.", "example": "CW-00024" }, { "name": "related_ve_product_id", "spark_sql_type": "STRING", "nullable": true, "description": "Other product of a pair event (review_candidate_kept_separate, same_name_candidate); for split, the new product. Null otherwise.", "derivation": "records fields dups[].id and sn[].id mapped to ve_product_id.", "example": "VEP-a064f3d851f4" }, { "name": "related_prototype_id", "spark_sql_type": "STRING", "nullable": true, "description": "Prototype identifier of related_ve_product_id (not stable).", "derivation": "records fields dups[].id, sn[].id.", "example": "CW-00099" }, { "name": "related_source", "spark_sql_type": "STRING", "nullable": true, "description": "For merge and unit_group: source of the related source record (osm, wd, nhle). Null otherwise.", "derivation": "records field src[].s.", "example": "wd" }, { "name": "related_source_record_id", "spark_sql_type": "STRING", "nullable": true, "description": "For merge: the source record merged into the product. For unit_group: the first unit, which is kept as the primary record. Null otherwise.", "derivation": "records field src[].id. The identifiers of the other grouped units were not retained by the pipeline.", "example": "Q26579423" }, { "name": "method", "spark_sql_type": "STRING", "nullable": false, "description": "Rule that produced the event.", "derivation": "merge: link_method (wikidata_tag, he_ref_tag, p1216, osm_dup, name_rule, nhle_name); review_candidate_kept_separate: name_rule_review_band; same_name_candidate: same_name_within_150m; unit_group: unit_group_shared_website or unit_group_within_300m.", "example": "wikidata_tag" }, { "name": "method_label", "spark_sql_type": "STRING", "nullable": false, "description": "Plain description of method.", "derivation": "meta.json match_methods and meta.json config / review_aids.", "example": "Linked by shared Wikidata ID (OSM wikidata tag)" }, { "name": "score", "spark_sql_type": "DOUBLE", "nullable": true, "description": "Match score of the event, 0 to 1, when the rule produces one; otherwise null.", "derivation": "merge: src[].sc; review_candidate_kept_separate: dups[].sc.", "example": 1.0 }, { "name": "distance_m", "spark_sql_type": "INT", "nullable": true, "description": "Straight-line distance in metres between the two records, when recorded.", "derivation": "merge: src[].d; review_candidate_kept_separate: dups[].d; same_name_candidate: sn[].d.", "example": 13 }, { "name": "unit_count", "spark_sql_type": "INT", "nullable": true, "description": "unit_group only: number of unit features grouped into the site.", "derivation": "records field units.", "example": 117 }, { "name": "note", "spark_sql_type": "STRING", "nullable": false, "description": "Plain-English summary of the event.", "derivation": "Generated by export_release.py from the fields above.", "example": "Wikidata record Q26579423 merged into the product whose primary record is OpenStreetMap node/682347683 (wikidata_tag; score 1.0, 13 m apart)." }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot (2026-09-12).", "derivation": "meta.json generated (date part).", "example": "2026-09-12" } ] }, { "name": "product_change_set", "full_name": "ve_audit.sample.product_change_set", "description": "Changes between the previous release (ve-audit-sample-r2-2026-09-12) and this one, keyed on ve_product_id: inserted and retired products, and one row per changed column of an updated product.", "rows": 93, "files": [ "product_change_set.parquet", "product_change_set.csv" ], "primary_key": [ "change_id" ], "foreign_keys": [], "sort_order": [ "ve_product_id", "change_type", "attribute (compared column order)" ], "columns": [ { "name": "change_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the change row, unique within the release.", "derivation": "'VEC-' + first 16 hexadecimal characters of SHA-256 over 'release_id|change_type|ve_product_id|attribute'.", "example": "VEC-863e96e3ce9b9935" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Release in which the change was published.", "derivation": "Fixed: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" }, { "name": "previous_release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Release this one is compared with.", "derivation": "pipeline/release.json previous_release_id: ve-audit-sample-r2-2026-09-12.", "example": "ve-audit-sample-r2-2026-09-12" }, { "name": "change_type", "spark_sql_type": "STRING", "nullable": false, "description": "inserted, updated or retired.", "derivation": "inserted: ve_product_id not in the previous release; retired: in the previous release and not in this one; updated: one row per compared product column whose value differs.", "example": "updated" }, { "name": "ve_product_id", "spark_sql_type": "STRING", "nullable": false, "description": "Product the change applies to. A retired id is not in product; ve_product_id_registry.csv gives its status and any replacing id.", "derivation": "product.ve_product_id, or the previous release product.ve_product_id for retired rows.", "example": "VEP-090ae391e4d6" }, { "name": "prototype_id", "spark_sql_type": "STRING", "nullable": true, "description": "Prototype identifier in this release (not stable); null for retired rows.", "derivation": "product.prototype_id.", "example": "GM-05976" }, { "name": "attribute", "spark_sql_type": "STRING", "nullable": true, "description": "updated rows: the product column that changed; null otherwise.", "derivation": "Compared columns: name, kind, category_key, product_role, chain_brand, type_in_source, lvep_id, lad_code, latitude, longitude, website, description, opening_hours, months_open, months_open_method, seasonality_evidence, seasonal_tag, wheelchair_tag, heritage_designations, category_rule, category_confidence, source_count, primary_source, primary_source_record_id, checks, possible_duplicate_ids, same_name_nearby_ids, nearest_product_m, grouped_units. prototype_id is renumbered at every build and is not compared.", "example": "nearest_product_m" }, { "name": "previous_value", "spark_sql_type": "STRING", "nullable": true, "description": "updated rows: value in the previous release as written in its product.csv; null when empty.", "derivation": "data/releases/ve-audit-sample-r2-2026-09-12/product.csv.gz (the product.csv published with the previous release).", "example": "87" }, { "name": "new_value", "spark_sql_type": "STRING", "nullable": true, "description": "updated rows: value in this release as written in product.csv; null when empty.", "derivation": "product.csv of this release.", "example": "0" }, { "name": "note", "spark_sql_type": "STRING", "nullable": false, "description": "Why the value changed.", "derivation": "pipeline/release.json changes_since_previous, matched on the columns that changed for the product; otherwise a generic note.", "example": "Latent heritage rule for area designations: World Heritage Sites, registered parks and gardens and registered battlefields are now measured from their Historic England boundary (boundaries retrieved 13 September 2026, pipeline/fetch_nhle_areas.py and area_rule.py) instead of one representative point. A visitor product record inside the site or within 500 m of its boundary counts, so the check latent_asset is removed where one exists (for example the Cornwall and West Devon Mining Landscape, with East Pool Mine and Heartlands inside), and nearest_product_m is the distance to the boundary (0 inside). Other sources keep the 12 September 2026 snapshot; no other rule changed." }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot (2026-09-12).", "derivation": "meta.json generated (date part).", "example": "2026-09-12" } ] }, { "name": "lvep_membership", "full_name": "ve_audit.sample.lvep_membership", "description": "Illustrative membership of the sample LVEPs: which ONS local authority districts each covers, with mid-year population. Our reading of public information, not VisitEngland membership data.", "rows": 13, "files": [ "lvep_membership.parquet", "lvep_membership.csv" ], "primary_key": [ "lad_code" ], "foreign_keys": [], "sort_order": [ "lvep_id", "lad_code" ], "columns": [ { "name": "lvep_id", "spark_sql_type": "STRING", "nullable": false, "description": "Sample LVEP identifier.", "derivation": "pipeline/lveps.json, as copied to meta.json lveps[].id.", "example": "cornwall" }, { "name": "lvep_name", "spark_sql_type": "STRING", "nullable": false, "description": "Sample LVEP name.", "derivation": "meta.json lveps[].name.", "example": "Cornwall" }, { "name": "lad_code", "spark_sql_type": "STRING", "nullable": false, "description": "ONS local authority district code (May 2026).", "derivation": "meta.json lveps[].local_authorities[].code.", "example": "E06000052" }, { "name": "lad_name", "spark_sql_type": "STRING", "nullable": false, "description": "ONS local authority district name.", "derivation": "meta.json lveps[].local_authorities[].name.", "example": "Cornwall" }, { "name": "lad_population", "spark_sql_type": "INT", "nullable": false, "description": "ONS mid-year population estimate for the local authority.", "derivation": "meta.json lveps[].local_authorities[].population (Nomis NM_2002_1).", "example": 584782 }, { "name": "population_year", "spark_sql_type": "INT", "nullable": false, "description": "Year of the mid-year population estimate.", "derivation": "meta.json lveps[].population_year.", "example": 2025 }, { "name": "membership_version", "spark_sql_type": "STRING", "nullable": false, "description": "Version of the illustrative LVEP to local authority membership table. Not VisitEngland membership data.", "derivation": "meta.json membership_version.", "example": "illustrative-2026-09-12" }, { "name": "lad_vintage", "spark_sql_type": "STRING", "nullable": false, "description": "ONS boundary vintage the codes refer to.", "derivation": "meta.json lad_vintage.", "example": "ONS Local Authority Districts, May 2026" }, { "name": "snapshot_date", "spark_sql_type": "DATE", "nullable": false, "description": "Date of the data snapshot: sources retrieved and pipeline run on 2026-09-12.", "derivation": "meta.json generated (date part). Same value on every row.", "example": "2026-09-12" }, { "name": "release_id", "spark_sql_type": "STRING", "nullable": false, "description": "Identifier of the release this row belongs to.", "derivation": "Fixed for this release: ve-audit-sample-r3-2026-09-12.", "example": "ve-audit-sample-r3-2026-09-12" } ] } ] }