Skip to content

Commit c1c1c52

Browse files
Sam-BollingSam-Bolling
committed
Apply NIMS total enrichment pack (2026-03-11)
- Bootstrap: enriched procedure (Swagger docs, Pattern A keywords, richer lineage/constraints), datastream schema (per-camera metadata, site/rawItem discovery links), deployments (richer descriptions, camera/site external links) - cameras.json: added cameraDiscoveryUrl, siteDiscoveryUrl, listFilesUrl, listFilesRawItemUrl, timelapseUrl per camera - Re-bootstrapped with --clean: all resources recreated with enriched metadata - VM deployed and publisher restarted (8/8 cameras publishing)
1 parent caab820 commit c1c1c52

2 files changed

Lines changed: 124 additions & 62 deletions

File tree

‎publishers/usgs_nims/bootstrap_usgs_nims.py‎

Lines changed: 76 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@
6666
USGS_OGC_API = "https://api.waterdata.usgs.gov/ogcapi/v0/"
6767
USGS_NIMS_CAMERAS = "https://api.waterdata.usgs.gov/nims/v0/cameras"
6868
USGS_NIMS_LIST_FILES = "https://api.waterdata.usgs.gov/nims/v0/listFiles"
69+
USGS_NIMS_DOCS = "https://api.waterdata.usgs.gov/nims/v0/docs"
70+
USGS_NIMS_SITE_DISCOVERY = "https://api.waterdata.usgs.gov/nims/v0/cameras?siteId="
71+
USGS_NIMS_RAWITEM_NOTE = "https://api.waterdata.usgs.gov/nims/v0/listFiles"
6972

7073

7174
def _load_cameras() -> list[dict]:
@@ -98,6 +101,14 @@ def _timelapse_url(cam: dict) -> str:
98101
return f"{tl_dir}{cam_id}_720.mp4"
99102

100103

104+
def _site_cameras_url(nwis_id: str) -> str:
105+
return f"{NIMS_API_BASE}cameras?siteId={nwis_id}"
106+
107+
108+
def _list_files_rawitem_url(cam_id: str, limit: int = 5) -> str:
109+
return f"{NIMS_API_BASE}listFiles?camId={cam_id}&limit={limit}&recent=true&rawItem=true"
110+
111+
101112
# ═══════════════════════════════════════════════════════════════════════════
102113
# Resource definitions
103114
# ═══════════════════════════════════════════════════════════════════════════
@@ -110,28 +121,28 @@ def _timelapse_url(cam: dict) -> str:
110121
"featureType": "sosa:ObservingProcedure",
111122
"name": "USGS NIMS Station Imagery v1",
112123
"description": (
113-
"Publishes gaging-station imagery from the USGS National Imagery Management "
114-
"System (NIMS). The publisher polls each camera's latest image list via the "
115-
"NIMS v0 API, resolves stable S3-hosted image URLs for overlay, thumbnail, and "
116-
"720px resolutions, and publishes image-reference observations. Each observation "
117-
"contains URLs and metadata — not binary image data. Timelapse video URLs are "
118-
"included when the camera supports time-lapse generation."
124+
"Publishes image-reference observations from the USGS National Imagery Management "
125+
"System (NIMS). The current publisher model reuses existing USGS water monitoring "
126+
"station systems and attaches one selected imagery datastream per curated camera as "
127+
"a Pattern A companion datastream. Runtime polls NIMS listFiles, derives image time, "
128+
"constructs stable S3-hosted URLs for multiple image resolutions, and publishes URLs "
129+
"and metadata instead of binary image payloads."
119130
),
120131
"keywords": [
121132
"USGS",
122133
"NIMS",
123-
"gaging station",
124-
"camera",
125-
"imagery",
126-
"streamgage",
127-
"visual monitoring",
128-
"JPEG",
129-
"timelapse",
130134
"station imagery",
135+
"camera discovery",
136+
"listFiles",
137+
"image reference",
138+
"timelapse",
139+
"shared system",
140+
"Pattern A",
131141
],
132142
"documentation": [
133143
{"title": "NIMS v0 Camera Discovery", "href": USGS_NIMS_CAMERAS, "rel": "documentation"},
134144
{"title": "NIMS v0 Image Listing", "href": USGS_NIMS_LIST_FILES, "rel": "documentation"},
145+
{"title": "NIMS v0 Swagger Docs", "href": USGS_NIMS_DOCS, "rel": "describedby"},
135146
{"title": "NIMS Image Bucket (S3)", "href": NIMS_S3_BASE, "rel": "alternate"},
136147
{"title": "USGS API Registration", "href": USGS_API_REGISTRATION, "rel": "related"},
137148
{"title": "USGS Water Data Home", "href": USGS_WATER_HOME, "rel": "about"},
@@ -151,33 +162,40 @@ def _timelapse_url(cam: dict) -> str:
151162
"lineage": {
152163
"source": "U.S. Geological Survey / National Imagery Management System (NIMS)",
153164
"upstream": (
154-
"Camera metadata comes from the NIMS /cameras endpoint. Image filenames "
155-
"come from /listFiles. Full-size, thumbnail, and 720px image URLs are "
156-
"constructed from S3 bucket directory paths returned by /cameras."
165+
"Camera identity and directory metadata come from NIMS cameras responses. "
166+
"Newest image filenames come from listFiles. Resolution-specific URLs are "
167+
"constructed from the returned directory paths and filenames."
157168
),
158169
"normalization": (
159-
"Publisher extracts timestamp from the image filename pattern "
160-
"({camId}___YYYY-MM-DDTHH-mm-ssZ.jpg), constructs resolution-specific "
161-
"image URLs, and publishes a JSON observation record referencing the URLs."
170+
"The current runtime uses listFiles string-array mode, parses image time from the "
171+
"filename pattern, and publishes imageUrl, thumbUrl, smallUrl, filename, mediaType, "
172+
"and optional timeLapseUrl in the observation result body."
162173
),
163174
},
164175
"usageConstraints": {
165176
"apiKeyNote": (
166-
"A USGS API key is recommended for higher rate-limit ceilings. Register at "
167-
"https://api.usgs.gov. Pass via query parameter or X-Api-Key header."
177+
"A USGS API key is recommended for higher request ceilings. Register at "
178+
"https://api.usgs.gov."
168179
),
169180
"nimsVersionNote": (
170-
"NIMS v0 is fully supported but classified as legacy. USGS recommends "
171-
"migrating to v1 once available. The endpoint version is configurable."
181+
"NIMS v0 is the active verified endpoint as of 2026-03-11. The package keeps v0 URLs "
182+
"and does not assume a v1 migration path is live yet."
172183
),
173-
"imageryNote": (
174-
"NIMS cameras are typically daylight-only with variable refresh intervals "
175-
"(15-60 minutes). Images may be stale at night or during outages."
184+
"sharedSystemNote": (
185+
"This publisher uses Pattern A and reuses existing USGS water station systems. "
186+
"It does not create NIMS-specific systems."
187+
),
188+
"selectionNote": (
189+
"The current curated model selects one camera per station system even though some "
190+
"NIMS sites now expose multiple live cameras."
191+
),
192+
"rawItemNote": (
193+
"NIMS listFiles also supports rawItem=true responses with timestamp and file-size "
194+
"fields. The current runtime does not yet use that richer mode."
176195
),
177-
"disclaimer": (
178-
"USGS imagery is provided as-is. Data are released on the condition that "
179-
"neither the USGS nor the United States Government may be held liable for "
180-
"damages resulting from use."
196+
"imageryNote": (
197+
"NIMS cameras may be daylight-only or 24x7. Refresh intervals vary by camera. "
198+
"Images may be stale during darkness, outages, or reduced capture windows."
181199
),
182200
},
183201
"validTime": [VALID_TIME_START, ".."],
@@ -189,19 +207,26 @@ def _imagery_datastream_schema(cam: dict) -> dict:
189207
"""SWE DataRecord schema for the NIMS imagery datastream."""
190208
cam_id = cam["camId"]
191209
nwis_id = cam["nwisId"]
210+
station_name = cam.get("stationName", cam.get("camName", nwis_id))
211+
ingest_period = cam.get("ingestPeriod", "unknown")
212+
ingest_interval = cam.get("ingestIntervalMin", "unknown")
213+
timelapse_enabled = cam.get("TL_enabled", False)
214+
192215
return {
193216
"outputName": DS_OUTPUT_NAME,
194217
"name": "NIMS Station Image",
195218
"description": (
196-
f"Image-reference observations from USGS NIMS camera {cam_id} at gaging "
197-
f"station {nwis_id} ({cam.get('stationName', cam.get('camName', ''))})."
198-
" Each observation provides stable S3-hosted URLs for overlay (full-size), "
199-
"thumbnail, and 720px image resolutions, plus camera metadata and an optional "
200-
"timelapse video URL. Result records contain URLs — not binary image data."
219+
f"Image-reference observations from selected USGS NIMS camera {cam_id} at gaging "
220+
f"station {nwis_id} ({station_name}). This datastream is a Pattern A companion "
221+
f"datastream on the shared USGS water station system. Current camera capture mode is "
222+
f"{ingest_period} with an approximate {ingest_interval}-minute interval. "
223+
f"Timelapse enabled: {str(timelapse_enabled).lower()}."
201224
),
202225
"documentation": [
203-
{"title": "NIMS Camera Discovery", "href": USGS_NIMS_CAMERAS, "rel": "documentation"},
226+
{"title": "NIMS Camera Discovery", "href": _camera_page_url(cam_id), "rel": "documentation"},
227+
{"title": "NIMS Site Discovery", "href": _site_cameras_url(nwis_id), "rel": "documentation"},
204228
{"title": "NIMS Image Listing", "href": _list_files_url(cam_id, 5), "rel": "documentation"},
229+
{"title": "NIMS Raw Item Listing", "href": _list_files_rawitem_url(cam_id, 5), "rel": "documentation"},
205230
{"title": "NIMS S3 Bucket", "href": NIMS_S3_BASE, "rel": "alternate"},
206231
],
207232
"schema": {
@@ -210,7 +235,7 @@ def _imagery_datastream_schema(cam: dict) -> dict:
210235
"type": "DataRecord",
211236
"label": "NIMS Image Reference",
212237
"description": (
213-
"USGS NIMS gaging-station image metadata and resolution-specific URLs. "
238+
"USGS NIMS image-reference metadata and resolution-specific URLs. "
214239
"The time field named timestamp is populated from phenomenonTime and "
215240
"must not be included inside the result body."
216241
),
@@ -292,10 +317,11 @@ def _deploy_root() -> dict:
292317
"description": (
293318
"Top-level CSAPI deployment grouping for USGS NIMS gaging-station imagery "
294319
"published by OSHConnect-Python. Imagery datastreams are companion datastreams "
295-
"on existing USGS water monitoring station systems (Pattern A)."
320+
"attached to existing USGS water monitoring station systems under the Pattern A model."
296321
),
297322
"documentation": [
298323
{"title": "NIMS Camera Discovery", "href": USGS_NIMS_CAMERAS, "rel": "documentation"},
324+
{"title": "NIMS Swagger Docs", "href": USGS_NIMS_DOCS, "rel": "describedby"},
299325
{"title": "NIMS S3 Image Bucket", "href": NIMS_S3_BASE, "rel": "alternate"},
300326
{"title": "USGS Water Data Home", "href": USGS_WATER_HOME, "rel": "about"},
301327
],
@@ -316,12 +342,13 @@ def _deploy_group() -> dict:
316342
"featureType": "sosa:Deployment",
317343
"name": "USGS NIMS Camera Stations",
318344
"description": (
319-
"Grouping deployment for the curated set of USGS NIMS camera-equipped "
320-
"gaging stations. Each child deployment pairs a camera with the existing "
321-
"USGS water monitoring station system."
345+
"Grouping deployment for the curated set of USGS NIMS camera-equipped gaging "
346+
"stations. Each child deployment represents one selected camera linked to one "
347+
"existing USGS water station system."
322348
),
323349
"documentation": [
324350
{"title": "NIMS Camera Discovery", "href": USGS_NIMS_CAMERAS, "rel": "documentation"},
351+
{"title": "USGS Water OGC API", "href": USGS_OGC_API, "rel": "related"},
325352
],
326353
"validTime": [VALID_TIME_START, ".."],
327354
},
@@ -343,21 +370,16 @@ def _deploy_camera(cam: dict, system_server_id: str) -> dict:
343370
"featureType": "sosa:Deployment",
344371
"name": f"NIMS Camera {nwis_id}",
345372
"description": (
346-
f"CSAPI deployment node for NIMS camera {cam_id} at USGS gaging station "
347-
f"{nwis_id} ({station_name}). Links imagery observations to the existing "
348-
f"USGS water monitoring station system."
373+
f"CSAPI deployment node for selected NIMS camera {cam_id} at USGS gaging station "
374+
f"{nwis_id} ({station_name}). This deployment links imagery observations to the "
375+
"existing shared USGS water station system rather than creating a separate camera system."
349376
),
350377
"externalLinks": [
351-
{
352-
"href": _list_files_url(cam_id, 10),
353-
"title": "NIMS Recent Images",
354-
"rel": "latest-version",
355-
},
356-
{
357-
"href": _timelapse_url(cam),
358-
"title": "NIMS Timelapse Video",
359-
"rel": "alternate",
360-
},
378+
{"href": _camera_page_url(cam_id), "title": "NIMS Camera Discovery", "rel": "canonical"},
379+
{"href": _site_cameras_url(nwis_id), "title": "NIMS Site Discovery", "rel": "related"},
380+
{"href": _list_files_url(cam_id, 10), "title": "NIMS Recent Images", "rel": "latest-version"},
381+
{"href": _list_files_rawitem_url(cam_id, 10), "title": "NIMS Recent Images (rawItem)", "rel": "related"},
382+
{"href": _timelapse_url(cam), "title": "NIMS Timelapse Video", "rel": "alternate"},
361383
],
362384
"validTime": [VALID_TIME_START, ".."],
363385
"platform@link": {

0 commit comments

Comments
 (0)