diff --git a/README.md b/README.md index 8eafc18..779685f 100644 --- a/README.md +++ b/README.md @@ -123,11 +123,14 @@ The repository includes an automated pipeline for publishing tool/layer column d ### Source Metadata Files - Each tool now publishes a single unified `layer_definitions.json` containing both descriptive metadata and column definitions (no separate per-layer files or `def_path` indirection). -- These live beside the tool's code (e.g. under `scripts//`). +- These live beside the tool's code (e.g. under `pipelines//`). ### Export / Flatten Script -`scripts/metadata/export_layer_definitions_for_s3.py` scans the repo for every `layer_definitions.json` and produces partitioned Parquet (default) output. Partition hierarchy (three levels): +`export_layer_definitions_for_s3.py` in the riverscapesXML repo scans the repo for every `layer_definitions.json` and produces partitioned Parquet (default) output. +We've loaded it as a requirement in the toml: `https://github.com/Riverscapes/RiverscapesXML.git@master#subdirectory=riverscapes_metadata` + + Partition hierarchy (three levels): 1. `authority` – repository root name (e.g. `data-exchange-scripts`). Derived automatically from the git repo folder name. 2. `authority_name` – the tool / package authority publishing the layer definitions (from JSON). diff --git a/pipelines/rscontext_to_athena/layer_definitions.json b/pipelines/rscontext_to_athena/layer_definitions.json index 51ffc29..8717e2d 100644 --- a/pipelines/rscontext_to_athena/layer_definitions.json +++ b/pipelines/rscontext_to_athena/layer_definitions.json @@ -22,53 +22,130 @@ "default_value": null }, { - "name": "hucname", + "name": "hucName", "dtype": "TEXT", - "friendly_name": "HUC Name", + "friendly_name": "Watershed Name", "data_unit": "", "description": "", "is_key": false, "is_required": false, "theme": "", - "preferred_bin_definition": "", "default_value": null }, { - "name": "hucstates", + "name": "hucStates", "dtype": "TEXT", - "friendly_name": "", + "friendly_name": "State(s)", "data_unit": "", "description": "", - "is_key": false, - "is_required": false, - "theme": "", - "preferred_bin_definition": "", - "default_value": null + "theme": "" }, { - "name": "hucareasqkm", + "name": "hucAreaSqKm", "dtype": "REAL", - "friendly_name": "HUC Area", - "data_unit": "", + "friendly_name": "Watershed Area", + "data_unit": "km^2", "description": "", "is_key": false, "is_required": false, "theme": "", - "preferred_bin_definition": "", "default_value": null }, { - "name": "hucareaacres", + "name": "hucAreaAcres", "dtype": "REAL", - "friendly_name": "HUC Area", - "data_unit": "", + "friendly_name": "Watershed Area (acres)", + "data_unit": "acre", "description": "", "is_key": false, "is_required": false, "theme": "", - "preferred_bin_definition": "", "default_value": null }, + { + "name": "flowlineLengthPerennialKm", + "dtype": "REAL", + "friendly_name": "Perrenial Stream Length", + "data_unit": "km" + }, + { + "name": "flowlineLengthIntermittentKm", + "dtype": "REAL", + "friendly_name": "Intermittent Stream Length", + "data_unit": "km" + }, + { + "name": "flowlineLengthEphemeralKm", + "dtype": "REAL", + "friendly_name": "Ephemeral Stream Length", + "data_unit": "km" + }, + { + "name": "flowlineLengthCanalsKm", + "dtype": "REAL", + "friendly_name": "Canal Length", + "data_unit": "km" + }, + { + "name": "flowlineLengthAllKm", + "dtype": "REAL", + "friendly_name": "Total Stream Length", + "data_unit": "km" + }, + { + "name": "flowlineFeatureCount", + "dtype": "INTEGER", + "friendly_name": "Number of Stream Segments", + "data_unit": "Unitless" + }, + { + "name": "waterbodyAreaSqKm", + "dtype": "REAL", + "friendly_name": "Total Waterbody Area", + "data_unit": "km^2" + }, + { + "name": "waterbodyFeatureCount", + "dtype": "INTEGER", + "friendly_name": "Total count of Waterbodies", + "data_unit": "Unitless" + }, + { + "name": "waterbodyLakesPondsAreaSqKm", + "dtype": "REAL", + "friendly_name": "Lake Area", + "data_unit": "km^2" + }, + { + "name": "waterbodyReservoirAreaSqKm", + "dtype": "REAL", + "friendly_name": "Reservoir Area", + "data_unit": "km^2" + }, + { + "name": "waterbodyEstuariesAreaSqKm", + "dtype": "REAL", + "friendly_name": "Estuaries Area", + "data_unit": "km^2" + }, + { + "name": "waterbodyPlayaAreaSqKm", + "dtype": "REAL", + "friendly_name": "Playa Area", + "data_unit": "km^2" + }, + { + "name": "waterbodySwampMarshAreaSqKm", + "dtype": "REAL", + "friendly_name": "Swamp/Marsh Area", + "data_unit": "km^2" + }, + { + "name": "waterbodyIceSnowAreaSqKm", + "dtype": "REAL", + "friendly_name": "Ice/Snow Area", + "data_unit": "km^2" + }, { "name": "dem_bins.min", "dtype": "REAL",