From 2ba9073d6acf1147a6ca2afd05b5ccaffe2c2cdf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:59:12 +0100 Subject: [PATCH 01/24] first round, delete SpaceTimeRegion object and related classes/methods --- src/virtualship/models/__init__.py | 8 --- src/virtualship/models/expedition.py | 12 +--- src/virtualship/models/space_time_region.py | 64 --------------------- src/virtualship/static/expedition.yaml | 11 ---- src/virtualship/utils.py | 30 +--------- tests/expedition/test_expedition.py | 49 ++-------------- tests/test_utils.py | 43 +++----------- 7 files changed, 14 insertions(+), 203 deletions(-) delete mode 100644 src/virtualship/models/space_time_region.py diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index 5eaabb85..d61c1719 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -15,11 +15,6 @@ XBTConfig, ) from .location import Location -from .space_time_region import ( - SpaceTimeRegion, - SpatialRange, - TimeRange, -) from .spacetime import ( Spacetime, ) @@ -36,9 +31,6 @@ "ShipUnderwaterSTConfig", "DrifterConfig", "XBTConfig", - "SpatialRange", - "TimeRange", - "SpaceTimeRegion", "Spacetime", "Expedition", "InstrumentsConfig", diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index e6e80102..6df62d87 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -14,7 +14,6 @@ from virtualship.utils import _get_bathy_data, _validate_numeric_mins_to_timedelta from .location import Location -from .space_time_region import SpaceTimeRegion projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -77,7 +76,6 @@ class Schedule(pydantic.BaseModel): """Schedule of the virtual ship.""" waypoints: list[Waypoint] - space_time_region: SpaceTimeRegion | None = None model_config = pydantic.ConfigDict(extra="forbid") @@ -86,7 +84,6 @@ def verify( ship_speed: float, ignore_land_test: bool = False, *, - check_space_time_region: bool = False, from_data: Path | None = None, ) -> None: """ @@ -101,11 +98,6 @@ def verify( """ print("\nVerifying route... ") - if check_space_time_region and self.space_time_region is None: - raise ScheduleError( - "space_time_region not found in schedule, please define it to proceed." - ) - if len(self.waypoints) == 0: raise ScheduleError("At least one waypoint must be provided.") @@ -129,7 +121,7 @@ def verify( if not ignore_land_test: try: bathymetry_field = _get_bathy_data( - self.space_time_region, + self.space_time_region, # TODO: replace! latlon_buffer=None, from_data=from_data, ).bathymetry @@ -150,7 +142,7 @@ def verify( land_waypoints.append((wp_i, wp)) except Exception as e: raise ScheduleError( - f"Waypoint #{wp_i + 1} at location {wp.location} could not be evaluated against bathymetry data. There may be a problem with the waypoint location being outside of the space_time_region or with the bathymetry data itself.\n\n Original error: {e}" + f"Waypoint #{wp_i + 1} at location {wp.location} could not be evaluated against bathymetry data. \n\n Original error: {e}" ) from e if len(land_waypoints) > 0: diff --git a/src/virtualship/models/space_time_region.py b/src/virtualship/models/space_time_region.py deleted file mode 100644 index 48ad5699..00000000 --- a/src/virtualship/models/space_time_region.py +++ /dev/null @@ -1,64 +0,0 @@ -"""SpaceTimeRegion class.""" - -from datetime import datetime -from typing import Annotated - -from pydantic import BaseModel, Field, model_validator -from typing_extensions import Self - -Longitude = Annotated[float, Field(..., ge=-180, le=180)] -Latitude = Annotated[float, Field(..., ge=-90, le=90)] -Depth = float # TODO: insert a minimum depth here? e.g., `Annotated[float, Field(..., ge=0)]` -# TODO: is_valid_depth in validator_utils.py will alse need to be updated if this TODO is implemented - - -class SpatialRange(BaseModel): - """Defines geographic boundaries.""" - - minimum_longitude: Longitude - maximum_longitude: Longitude - minimum_latitude: Latitude - maximum_latitude: Latitude - minimum_depth: Depth | None = None - maximum_depth: Depth | None = None - - @model_validator(mode="after") - def _check_lon_lat_domain(self) -> Self: - if not self.minimum_longitude < self.maximum_longitude: - raise ValueError("minimum_longitude must be less than maximum_longitude") - if not self.minimum_latitude < self.maximum_latitude: - raise ValueError("minimum_latitude must be less than maximum_latitude") - - if sum([self.minimum_depth is None, self.maximum_depth is None]) == 1: - raise ValueError("Both minimum_depth and maximum_depth must be provided.") - - if self.minimum_depth is None: - return self - - if not self.minimum_depth < self.maximum_depth: - raise ValueError("minimum_depth must be less than maximum_depth") - return self - - -class TimeRange(BaseModel): - """Defines the temporal boundaries for a space-time region.""" - - #! TODO: Remove the `| None` for `start_time` and `end_time`, and have the MFP functionality not use pydantic (with testing to avoid codebase drift) - start_time: datetime | None = None - end_time: datetime | None = None - - @model_validator(mode="after") - def _check_time_range(self) -> Self: - if ( - self.start_time and self.end_time - ): #! TODO: remove this check once `start_time` and `end_time` are required - if not self.start_time < self.end_time: - raise ValueError("start_time must be before end_time") - return self - - -class SpaceTimeRegion(BaseModel): - """An space-time region with spatial and temporal boundaries.""" - - spatial_range: SpatialRange - time_range: TimeRange diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 4c7394a3..34b8a2df 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -1,15 +1,4 @@ schedule: - space_time_region: - spatial_range: - minimum_longitude: -5 - maximum_longitude: 5 - minimum_latitude: -5 - maximum_latitude: 5 - minimum_depth: 0 - maximum_depth: 2000 - time_range: - start_time: 1998-01-01 00:00:00 - end_time: 1998-02-01 00:00:00 waypoints: - instrument: - CTD diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 251bc35f..3d0228c8 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -13,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: @@ -144,9 +144,6 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 InstrumentsConfig, Location, Schedule, - SpaceTimeRegion, - SpatialRange, - TimeRange, Waypoint, ) @@ -155,30 +152,6 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 coordinates_data = validate_coordinates(coordinates_data) - # maximum depth (in meters), buffer (in degrees) for each instrument - instrument_max_depths = { - "XBT": 2000, - "CTD": 5000, - "CTD_BGC": 5000, - "DRIFTER": 1, - "ARGO_FLOAT": 2000, - } - - spatial_range = SpatialRange( - minimum_longitude=coordinates_data["Longitude"].min(), - maximum_longitude=coordinates_data["Longitude"].max(), - minimum_latitude=coordinates_data["Latitude"].min(), - maximum_latitude=coordinates_data["Latitude"].max(), - minimum_depth=0, - maximum_depth=max(instrument_max_depths.values()), - ) - - # Create space-time region object - space_time_region = SpaceTimeRegion( - spatial_range=spatial_range, - time_range=TimeRange(), - ) - # Generate waypoints waypoints = [] for _, row in coordinates_data.iterrows(): @@ -192,7 +165,6 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 # Create Schedule object schedule = Schedule( waypoints=waypoints, - space_time_region=space_time_region, ) # extract instruments config from static diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 78fff2c2..13b4b689 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -6,8 +6,8 @@ import pyproj import pytest import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( Expedition, @@ -15,11 +15,6 @@ Schedule, Waypoint, ) -from virtualship.models.space_time_region import ( - SpaceTimeRegion, - SpatialRange, - TimeRange, -) from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition projection = pyproj.Geod(ellps="WGS84") @@ -136,20 +131,7 @@ def test_verify_on_land(): ), # NaN cell ] - spatial_range = SpatialRange( - minimum_latitude=min(wp.location.lat for wp in waypoints), - maximum_latitude=max(wp.location.lat for wp in waypoints), - minimum_longitude=min(wp.location.lon for wp in waypoints), - maximum_longitude=max(wp.location.lon for wp in waypoints), - ) - time_range = TimeRange( - start_time=min(wp.time for wp in waypoints if wp.time is not None), - end_time=max(wp.time for wp in waypoints if wp.time is not None), - ) - space_time_region = SpaceTimeRegion( - spatial_range=spatial_range, time_range=time_range - ) - schedule = Schedule(waypoints=waypoints, space_time_region=space_time_region) + schedule = Schedule(waypoints=waypoints) ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots with patch( @@ -168,11 +150,10 @@ def test_verify_on_land(): @pytest.mark.parametrize( - "schedule,check_space_time_region,error,match", + "schedule,error,match", [ pytest.param( Schedule(waypoints=[]), - False, ScheduleError, "At least one waypoint must be provided.", id="NoWaypoints", @@ -186,7 +167,6 @@ def test_verify_on_land(): ), ] ), - False, ScheduleError, "First waypoint must have a specified time.", id="FirstWaypointHasTime", @@ -203,7 +183,6 @@ def test_verify_on_land(): ), ] ), - False, ScheduleError, "Waypoint\\(s\\) : each waypoint should be timed after all previous waypoints", id="SequentialWaypoints", @@ -219,39 +198,19 @@ def test_verify_on_land(): ), ] ), - False, ScheduleError, "Waypoint planning is not valid: would arrive too late at waypoint number 2...", id="NotEnoughTime", ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 2, 1, 1, 0) - ), - ] - ), - True, - ScheduleError, - "space_time_region not found in schedule, please define it to proceed.", - id="NoSpaceTimeRegion", - ), ], ) -def test_verify_schedule_errors( - schedule: Schedule, check_space_time_region: bool, error, match -) -> None: +def test_verify_schedule_errors(schedule: Schedule, error, match) -> None: expedition = _get_expedition(expedition_dir) with pytest.raises(error, match=match): schedule.verify( expedition.ship_config.ship_speed_knots, ignore_land_test=True, - check_space_time_region=check_space_time_region, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 8bd2338e..d96f4369 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr -from parcels import FieldSet import virtualship.utils +from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, @@ -24,37 +24,6 @@ def expedition(tmp_file): return Expedition.from_yaml(tmp_file) -@pytest.fixture -def dummy_spatial_range(): - class DummySpatialRange: - minimum_longitude = 0 - maximum_longitude = 1 - minimum_latitude = 0 - maximum_latitude = 1 - minimum_depth = 0 - maximum_depth = 4 - - return DummySpatialRange() - - -@pytest.fixture -def dummy_time_range(): - class DummyTimeRange: - start_time = "2020-01-01" - end_time = "2020-01-02" - - return DummyTimeRange() - - -@pytest.fixture -def dummy_space_time_region(dummy_spatial_range, dummy_time_range): - class DummySpaceTimeRegion: - spatial_range = dummy_spatial_range - time_range = dummy_time_range - - return DummySpaceTimeRegion() - - @pytest.fixture def dummy_instrument(): class DummyInstrument: @@ -155,7 +124,7 @@ def test_start_end_in_product_timerange(expedition): ) -def test_get_bathy_data_local(tmp_path, dummy_space_time_region): +def test_get_bathy_data_local(tmp_path): """Test that _get_bathy_data returns a FieldSet when given a local directory for --from-data.""" # dummy .nc file with 'deptho' variable data = np.array([[1, 2], [3, 4]]) @@ -173,13 +142,15 @@ def test_get_bathy_data_local(tmp_path, dummy_space_time_region): ds.to_netcdf(nc_path) # should return a FieldSet - fieldset = _get_bathy_data(dummy_space_time_region, from_data=tmp_path) + fieldset = _get_bathy_data( + from_data=tmp_path + ) # TODO: will need domain coords; from waypoints? assert isinstance(fieldset, FieldSet) assert hasattr(fieldset, "bathymetry") assert np.allclose(fieldset.bathymetry.data, data) -def test_get_bathy_data_copernicusmarine(monkeypatch, dummy_space_time_region): +def test_get_bathy_data_copernicusmarine(monkeypatch): """Test that _get_bathy_data calls copernicusmarine by default.""" def dummy_copernicusmarine(*args, **kwargs): @@ -190,7 +161,7 @@ def dummy_copernicusmarine(*args, **kwargs): ) try: - _get_bathy_data(dummy_space_time_region) + _get_bathy_data() # TODO: will need domain coords; from waypoints? except RuntimeError as e: assert "copernicusmarine called" in str(e) From e9dbd783cce9c17343a189f46ed1b80f92da660b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:23:36 +0100 Subject: [PATCH 02/24] remove space time region from _plan --- src/virtualship/cli/_plan.py | 249 ----------------------------------- 1 file changed, 249 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 08845e0d..c0a99d54 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -22,10 +22,8 @@ from virtualship.cli.validator_utils import ( get_field_type, group_validators, - is_valid_depth, is_valid_lat, is_valid_lon, - is_valid_timestr, type_to_textual, ) from virtualship.errors import UnexpectedError, UserError @@ -40,8 +38,6 @@ Location, ShipConfig, ShipUnderwaterSTConfig, - SpatialRange, - TimeRange, Waypoint, XBTConfig, ) @@ -331,181 +327,6 @@ def compose(self) -> ComposeResult: yield VerticalScroll(id="waypoint_list", classes="waypoint-list") - # SECTION: "Space-Time Region" - - with Collapsible( - title="[b]Space-Time Region[/b] (advanced users only)", - collapsed=True, - ): - if self.expedition.schedule.space_time_region: - str_data = self.expedition.schedule.space_time_region - - yield Label("Minimum Latitude:") - yield Input( - id="min_lat", - value=str(str_data.spatial_range.minimum_latitude) - if str_data.spatial_range.minimum_latitude - else "", - validators=[ - Function( - is_valid_lat, - f"INVALID: value must be {is_valid_lat.__doc__.lower()}", - ) - ], - type="number", - placeholder="°N", - ) - yield Label( - "", - id="validation-failure-label-min_lat", - classes="-hidden validation-failure", - ) - - yield Label("Maximum Latitude:") - yield Input( - id="max_lat", - value=str(str_data.spatial_range.maximum_latitude), - validators=[ - Function( - is_valid_lat, - f"INVALID: value must be {is_valid_lat.__doc__.lower()}", - ) - ], - type="number", - placeholder="°N", - ) - yield Label( - "", - id="validation-failure-label-max_lat", - classes="-hidden validation-failure", - ) - - yield Label("Minimum Longitude:") - yield Input( - id="min_lon", - value=str(str_data.spatial_range.minimum_longitude), - validators=[ - Function( - is_valid_lon, - f"INVALID: value must be {is_valid_lon.__doc__.lower()}", - ) - ], - type="number", - placeholder="°E", - ) - yield Label( - "", - id="validation-failure-label-min_lon", - classes="-hidden validation-failure", - ) - - yield Label("Maximum Longitude:") - yield Input( - id="max_lon", - value=str(str_data.spatial_range.maximum_longitude), - validators=[ - Function( - is_valid_lon, - f"INVALID: value must be {is_valid_lon.__doc__.lower()}", - ) - ], - type="number", - placeholder="°E", - ) - yield Label( - "", - id="validation-failure-label-max_lon", - classes="-hidden validation-failure", - ) - - yield Label("Minimum Depth (meters):") - yield Input( - id="min_depth", - value=str(str_data.spatial_range.minimum_depth), - validators=[ - Function( - is_valid_depth, - f"INVALID: value must be {is_valid_depth.__doc__.lower()}", - ) - ], - type="number", - placeholder="m", - ) - yield Label( - "", - id="validation-failure-label-min_depth", - classes="-hidden validation-failure", - ) - - yield Label("Maximum Depth (meters):") - yield Input( - id="max_depth", - value=str(str_data.spatial_range.maximum_depth), - validators=[ - Function( - is_valid_depth, - f"INVALID: value must be {is_valid_depth.__doc__.lower()}", - ) - ], - type="number", - placeholder="m", - ) - yield Label( - "", - id="validation-failure-label-max_depth", - classes="-hidden validation-failure", - ) - - yield Label( - "Start Time (will be auto determined from waypoints if left blank):" - ) - yield Input( - id="start_time", - placeholder="YYYY-MM-DD hh:mm:ss", - value=( - str(str_data.time_range.start_time) - if str_data.time_range and str_data.time_range.start_time - else "" - ), - validators=[ - Function( - is_valid_timestr, - f"INVALID: value must be {is_valid_timestr.__doc__.lower()}", - ) - ], - type="text", - ) - yield Label( - "", - id="validation-failure-label-start_time", - classes="-hidden validation-failure", - ) - - yield Label( - "End Time (will be auto determined from waypoints if left blank):" - ) - yield Input( - id="end_time", - placeholder="YYYY-MM-DD hh:mm:ss", - value=( - str(str_data.time_range.end_time) - if str_data.time_range and str_data.time_range.end_time - else "" - ), - validators=[ - Function( - is_valid_timestr, - f"INVALID: value must be {is_valid_timestr.__doc__.lower()}", - ) - ], - type="text", - ) - yield Label( - "", - id="validation-failure-label-end_time", - classes="-hidden validation-failure", - ) - except Exception as e: raise UnexpectedError(unexpected_msg_compose(e)) from None @@ -592,24 +413,6 @@ def _update_instrument_configs(self): ) def _update_schedule(self): - start_time_input = self.query_one("#start_time").value - end_time_input = self.query_one("#end_time").value - waypoint_times = [ - wp.time - for wp in self.expedition.schedule.waypoints - if hasattr(wp, "time") and wp.time - ] - if not start_time_input and waypoint_times: - start_time = min(waypoint_times) - else: - start_time = start_time_input - if not end_time_input and waypoint_times: - end_time = max(waypoint_times) + datetime.timedelta(minutes=60480.0) - else: - end_time = end_time_input - time_range = TimeRange(start_time=start_time, end_time=end_time) - self.expedition.schedule.space_time_region.time_range = time_range - for i, wp in enumerate(self.expedition.schedule.waypoints): wp.location = Location( latitude=float(self.query_one(f"#wp{i}_lat").value), @@ -634,57 +437,6 @@ def _update_schedule(self): elif switch_on: wp.instrument.append(instrument) - # take min/max lat/lon to be most extreme values of waypoints or space_time_region inputs (so as to cover possibility of user edits in either place) - # also prevents situation where e.g. user defines a space time region inconsistent with waypoint locations and vice versa (warning also provided) - waypoint_lats = [ - wp.location.latitude for wp in self.expedition.schedule.waypoints - ] - waypoint_lons = [ - wp.location.longitude for wp in self.expedition.schedule.waypoints - ] - wp_min_lat, wp_max_lat = ( - min(waypoint_lats) if waypoint_lats else -90.0, - max(waypoint_lats) if waypoint_lats else 90.0, - ) - wp_min_lon, wp_max_lon = ( - min(waypoint_lons) if waypoint_lons else -180.0, - max(waypoint_lons) if waypoint_lons else 180.0, - ) - - st_reg_min_lat = float(self.query_one("#min_lat").value) - st_reg_max_lat = float(self.query_one("#max_lat").value) - st_reg_min_lon = float(self.query_one("#min_lon").value) - st_reg_max_lon = float(self.query_one("#max_lon").value) - - min_lat = min(wp_min_lat, st_reg_min_lat) - max_lat = max(wp_max_lat, st_reg_max_lat) - min_lon = min(wp_min_lon, st_reg_min_lon) - max_lon = max(wp_max_lon, st_reg_max_lon) - - spatial_range = SpatialRange( - minimum_longitude=min_lon, - maximum_longitude=max_lon, - minimum_latitude=min_lat, - maximum_latitude=max_lat, - minimum_depth=self.query_one("#min_depth").value, - maximum_depth=self.query_one("#max_depth").value, - ) - self.expedition.schedule.space_time_region.spatial_range = spatial_range - - # provide warning if user defines a space time region inconsistent with waypoint locations - if ( - (wp_min_lat < st_reg_min_lat) - or (wp_max_lat > st_reg_max_lat) - or (wp_min_lon < st_reg_min_lon) - or (wp_max_lon > st_reg_max_lon) - ): - self.notify( - "[b]WARNING[/b]. One or more waypoint locations lie outside the defined space-time region. Take care if manually adjusting the space-time region." - "\n\nThe space-time region will be automatically adjusted on saving to include all waypoint locations.", - severity="warning", - timeout=10, - ) - @on(Input.Changed) def show_invalid_reasons(self, event: Input.Changed) -> None: input_id = event.input.id @@ -1091,7 +843,6 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - check_space_time_region=True, ignore_land_test=True, ) From 5a5e0565b58b0b0ba939f49b9484a9ffabd4e4d7 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:40:41 +0100 Subject: [PATCH 03/24] update base and instrument class logic to use waypoint extremes for spacetime domain --- src/virtualship/cli/_plan.py | 6 +- src/virtualship/instruments/argo_float.py | 4 +- src/virtualship/instruments/base.py | 101 +++++++++++++--------- src/virtualship/instruments/drifter.py | 4 +- src/virtualship/models/expedition.py | 13 ++- src/virtualship/utils.py | 36 +++++--- 6 files changed, 103 insertions(+), 61 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index c0a99d54..3f7ba70a 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -41,7 +41,7 @@ Waypoint, XBTConfig, ) -from virtualship.utils import EXPEDITION +from virtualship.utils import EXPEDITION, _get_waypoint_latlons UNEXPECTED_MSG_ONSAVE = ( "Please ensure that:\n" @@ -262,6 +262,7 @@ def compose(self) -> ComposeResult: f"NOTE: entries will be ignored here if {info['title']} is OFF in Ship Speed & Onboard Measurements." ) with Container(classes="instrument-config"): + # TODO: add validator that Drifter cannot exceed the time buffer defined in DrifterInstrument; and similarly for Argo Float for attr_meta in attributes: attr = attr_meta["name"] is_minutes = attr_meta.get("minutes", False) @@ -841,6 +842,9 @@ def save_pressed(self) -> None: self.sync_ui_waypoints() # call to ensure waypoint inputs are synced # verify schedule + wp_lats, wp_lons = _get_waypoint_latlons( + expedition_editor.expedition.schedule.waypoints + ) expedition_editor.expedition.schedule.verify( ship_speed_value, ignore_land_test=True, diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 204f0b3d..6b692b3f 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -142,7 +142,7 @@ def __init__(self, expedition, from_data): variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} spacetime_buffer_size = { "latlon": 3.0, # [degrees] - "time": 21.0, # [days] + "time": 63.0, # [days] } super().__init__( diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 22b0b54a..4cdb3bdb 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -3,20 +3,22 @@ import abc from collections import OrderedDict from datetime import timedelta +from itertools import pairwise from pathlib import Path from typing import TYPE_CHECKING import copernicusmarine import xarray as xr -from parcels import FieldSet from yaspin import yaspin +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, _find_files_in_timerange, _find_nc_file_with_variable, _get_bathy_data, + _get_waypoint_latlons, _select_product_id, ship_spinner, ) @@ -56,6 +58,19 @@ def __init__( self.spacetime_buffer_size = spacetime_buffer_size self.limit_spec = limit_spec + wp_lats, wp_lons = _get_waypoint_latlons(expedition.schedule.waypoints) + wp_times = [ + wp.time for wp in expedition.schedule.waypoints if wp.time is not None + ] + assert all(earlier <= later for earlier, later in pairwise(wp_times)), ( + "wp_times is not ascending order" + ) + self.wp_times = wp_times + + self.min_time, self.max_time = wp_times[0], wp_times[-1] + self.min_lat, self.max_lat = min(wp_lats), max(wp_lats) + self.min_lon, self.max_lon = min(wp_lons), max(wp_lons) + def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" try: @@ -76,10 +91,10 @@ def load_input_data(self) -> FieldSet: # bathymetry data if self.add_bathymetry: bathymetry_field = _get_bathy_data( - self.expedition.schedule.space_time_region, - latlon_buffer=self.spacetime_buffer_size.get("latlon") - if self.spacetime_buffer_size - else None, + self.min_lat, + self.max_lat, + self.min_lon, + self.max_lon, from_data=self.from_data, ).bathymetry bathymetry_field.data = -bathymetry_field.data @@ -98,53 +113,56 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + TMP = True + + if TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print( + f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... " + ) self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") + else: - print( - f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... " - ) self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, + time_buffer: float | None, physical: bool, var: str, ) -> xr.Dataset: """Get Copernicus Marine dataset for direct ingestion.""" product_id = _select_product_id( physical=physical, - schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, - schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, + schedule_start=self.min_time, + schedule_end=self.max_time, variable=var if not physical else None, ) - latlon_buffer = self._get_spec_value("buffer", "latlon", 0.0) - time_buffer = self._get_spec_value("buffer", "time", 0.0) + latlon_buffer = self._get_spec_value( + "buffer", "latlon", 0.25 + ) # [degrees]; default 0.25 deg buffer to ensure coverage in cell edge cases depth_min = self._get_spec_value("limit", "depth_min", None) depth_max = self._get_spec_value("limit", "depth_max", None) return copernicusmarine.open_dataset( dataset_id=product_id, - minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude - - latlon_buffer, - maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude - + latlon_buffer, - minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude - - latlon_buffer, - maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude - + latlon_buffer, + minimum_longitude=self.min_lon - latlon_buffer, + maximum_longitude=self.max_lon + latlon_buffer, + minimum_latitude=self.min_lat - latlon_buffer, + maximum_latitude=self.max_lat + latlon_buffer, variables=[var], - start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, - end_datetime=self.expedition.schedule.space_time_region.time_range.end_time - + timedelta(days=time_buffer), + start_datetime=self.min_time, + end_datetime=self.max_time + timedelta(days=time_buffer), minimum_depth=depth_min, maximum_depth=depth_max, coordinates_selection_method="outside", @@ -159,6 +177,10 @@ def _generate_fieldset(self) -> FieldSet: fieldsets_list = [] keys = list(self.variables.keys()) + time_buffer = self._get_spec_value("buffer", "time", 0.0) + + # TODO: also limit from-data to spatial domain? + for key in keys: var = self.variables[key] if self.from_data is not None: # load from local data @@ -168,17 +190,10 @@ def _generate_fieldset(self) -> FieldSet: else: data_dir = self.from_data.joinpath("bgc") - schedule_start = ( - self.expedition.schedule.space_time_region.time_range.start_time - ) - schedule_end = ( - self.expedition.schedule.space_time_region.time_range.end_time - ) - files = _find_files_in_timerange( data_dir, - schedule_start, - schedule_end, + self.min_time, + self.max_time + timedelta(days=time_buffer), ) _, full_var_name = _find_nc_file_with_variable( @@ -197,7 +212,11 @@ def _generate_fieldset(self) -> FieldSet: ) else: # stream via Copernicus Marine Service physical = var in COPERNICUSMARINE_PHYS_VARIABLES - ds = self._get_copernicus_ds(physical=physical, var=var) + ds = self._get_copernicus_ds( + time_buffer, + physical=physical, + var=var, + ) fs = FieldSet.from_xarray_dataset( ds, {key: var}, self.dimensions, mesh="spherical" ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8c531455..ea10728f 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -68,7 +68,7 @@ def __init__(self, expedition, from_data): variables = {"U": "uo", "V": "vo", "T": "thetao"} spacetime_buffer_size = { "latlon": 6.0, # [degrees] - "time": 21.0, # [days] + "time": 63.0, # [days] } limit_spec = { "depth_min": 1.0, # [meters] diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 6df62d87..88baa09e 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -11,7 +11,11 @@ from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType -from virtualship.utils import _get_bathy_data, _validate_numeric_mins_to_timedelta +from virtualship.utils import ( + _get_bathy_data, + _get_waypoint_latlons, + _validate_numeric_mins_to_timedelta, +) from .location import Location @@ -120,9 +124,12 @@ def verify( land_waypoints = [] if not ignore_land_test: try: + wp_lats, wp_lons = _get_waypoint_latlons(self.waypoints) bathymetry_field = _get_bathy_data( - self.space_time_region, # TODO: replace! - latlon_buffer=None, + min(wp_lats), + max(wp_lats), + min(wp_lons), + max(wp_lons), from_data=from_data, ).bathymetry except Exception as e: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 3d0228c8..7bdc93b0 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -301,6 +301,8 @@ def add_dummy_UV(fieldset: FieldSet): COPERNICUSMARINE_PHYS_VARIABLES = ["uo", "vo", "so", "thetao"] COPERNICUSMARINE_BGC_VARIABLES = ["o2", "chl", "no3", "po4", "ph", "phyc", "nppv"] +BATHYMETRY_ID = "cmems_mod_glo_phy_my_0.083deg_static" + def _select_product_id( physical: bool, @@ -388,7 +390,11 @@ def _start_end_in_product_timerange( def _get_bathy_data( - space_time_region, latlon_buffer: float | None = None, from_data: Path | None = None + min_lat: float, + max_lat: float, + min_lon: float, + max_lon: float, + from_data: Path | None = None, ) -> FieldSet: """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" if from_data is not None: # load from local data @@ -409,19 +415,15 @@ def _get_bathy_data( ) else: # stream via Copernicus Marine Service + buffer = 0.1 # degrees buffer, always to 0.1 to ensure coverage in edge cases (bathy data grid resolution ~0.8 deg) + ds_bathymetry = copernicusmarine.open_dataset( - dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - minimum_longitude=space_time_region.spatial_range.minimum_longitude - - (latlon_buffer if latlon_buffer is not None else 0), - maximum_longitude=space_time_region.spatial_range.maximum_longitude - + (latlon_buffer if latlon_buffer is not None else 0), - minimum_latitude=space_time_region.spatial_range.minimum_latitude - - (latlon_buffer if latlon_buffer is not None else 0), - maximum_latitude=space_time_region.spatial_range.maximum_latitude - + (latlon_buffer if latlon_buffer is not None else 0), + dataset_id=BATHYMETRY_ID, + minimum_longitude=min_lon - buffer, + maximum_longitude=max_lon + buffer, + minimum_latitude=min_lat - buffer, + maximum_latitude=max_lat + buffer, variables=["deptho"], - start_datetime=space_time_region.time_range.start_time, - end_datetime=space_time_region.time_range.end_time, coordinates_selection_method="outside", ) bathymetry_variables = {"bathymetry": "deptho"} @@ -440,6 +442,7 @@ def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float :param time_past: Time the expedition took. :returns: The calculated cost of the expedition in US$. """ + # TODO: refactor to instrument sub-classes attributes...? SHIP_COST_PER_DAY = 30000 DRIFTER_DEPLOY_COST = 2500 ARGO_DEPLOY_COST = 15000 @@ -538,3 +541,12 @@ def _random_noise(scale: float = 0.01, limit: float = 0.03) -> float: """Generate a small random noise value for drifter seeding locations.""" value = np.random.normal(loc=0.0, scale=scale) return np.clip(value, -limit, limit) # ensure noise is within limits + + +def _get_waypoint_latlons(waypoints): + """Extract latitudes and longitudes from waypoints.""" + wp_lats, wp_lons = zip( + *[(wp.location.latitude, wp.location.longitude) for wp in waypoints], + strict=True, + ) + return wp_lats, wp_lons From b5eea49a9a0a2ed1998e97db98ddf905b886d333 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:59:49 +0100 Subject: [PATCH 04/24] clean up _plan methods --- src/virtualship/cli/_plan.py | 2 -- src/virtualship/cli/validator_utils.py | 29 -------------------------- 2 files changed, 31 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 3f7ba70a..3428af8a 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -245,7 +245,6 @@ def compose(self) -> ComposeResult: for instrument_name, info in INSTRUMENT_FIELDS.items(): config_class = info["class"] attributes = info["attributes"] - # instrument-specific configs now live under instruments_config config_instance = getattr( self.expedition.instruments_config, instrument_name, None ) @@ -262,7 +261,6 @@ def compose(self) -> ComposeResult: f"NOTE: entries will be ignored here if {info['title']} is OFF in Ship Speed & Onboard Measurements." ) with Container(classes="instrument-config"): - # TODO: add validator that Drifter cannot exceed the time buffer defined in DrifterInstrument; and similarly for Argo Float for attr_meta in attributes: attr = attr_meta["name"] is_minutes = attr_meta.get("minutes", False) diff --git a/src/virtualship/cli/validator_utils.py b/src/virtualship/cli/validator_utils.py index 402e48b1..4aaf9389 100644 --- a/src/virtualship/cli/validator_utils.py +++ b/src/virtualship/cli/validator_utils.py @@ -41,35 +41,6 @@ def is_valid_lon(value: str) -> bool: return -180 < v < 360 -@require_docstring -def is_valid_depth(value: str) -> bool: - """Float.""" - try: - v = float(value) - except ValueError: - return None - - # NOTE: depth model in space_time_region.py ONLY specifies that depth must be float (and no conditions < 0) - # NOTE: therefore, this condition is carried forward here to match what currently exists - # NOTE: however, there is a TODO in space_time_region.py to add conditions as Pydantic Field - # TODO: update validator here if/when depth model is updated in space_time_region.py - return isinstance(v, float) - - -@require_docstring -def is_valid_timestr(value: str) -> bool: - """Format YYYY-MM-DD hh:mm:ss.""" - if ( - not value.strip() - ): # return as valid if blank, UI logic will auto fill on save if so - return True - try: - datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") - return True - except Exception: - return False - - # SHIP CONFIG INPUTS VALIDATION FIELD_CONSTRAINT_ATTRS = ( From ffe07fcd144aedbc18769f8f85a5f8221d26e58a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:26:32 +0100 Subject: [PATCH 05/24] extend latlon buffer for drifters --- src/virtualship/instruments/drifter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index ea10728f..35767357 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -67,7 +67,7 @@ def __init__(self, expedition, from_data): """Initialize DrifterInstrument.""" variables = {"U": "uo", "V": "vo", "T": "thetao"} spacetime_buffer_size = { - "latlon": 6.0, # [degrees] + "latlon": 12.0, # [degrees] "time": 63.0, # [days] } limit_spec = { From a5d692a96847b5535ca2151275fbfa60f4fc5449 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:17:46 +0100 Subject: [PATCH 06/24] set time buffer from drifter lifetime config --- src/virtualship/instruments/drifter.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 35767357..2fb3e17f 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -68,7 +68,8 @@ def __init__(self, expedition, from_data): variables = {"U": "uo", "V": "vo", "T": "thetao"} spacetime_buffer_size = { "latlon": 12.0, # [degrees] - "time": 63.0, # [days] + "time": expedition.instruments_config.drifter_config.lifetime.total_seconds() + / (24 * 3600), # [days] } limit_spec = { "depth_min": 1.0, # [meters] @@ -90,7 +91,6 @@ def simulate(self, measurements, out_path) -> None: """Simulate Drifter measurements.""" OUTPUT_DT = timedelta(hours=5) DT = timedelta(minutes=5) - ENDTIME = None if len(measurements) == 0: print( @@ -132,29 +132,14 @@ def simulate(self, measurements, out_path) -> None: chunks=[len(drifter_particleset), 100], ) - # get earliest between fieldset end time and prescribed end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if ENDTIME is None: - actual_endtime = fieldset_endtime - elif ENDTIME > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(ENDTIME) + # determine end time for simulation, from fieldset (which itself is controlled by drifter lifetimes) + endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) # execute simulation drifter_particleset.execute( [AdvectionRK4, _sample_temperature, _check_lifetime], - endtime=actual_endtime, + endtime=endtime, dt=DT, output_file=out_file, verbose_progress=self.verbose_progress, ) - - # if there are more particles left than the number of drifters with an indefinite endtime, warn the user - if len(drifter_particleset.particledata) > len( - [d for d in measurements if d.lifetime is None] - ): - print( - "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." - ) From 6ebb41b8c4922ab5366516a02104de8f1af1177e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:36:01 +0100 Subject: [PATCH 07/24] shift testing to use waypoint space-times instead of space-time-region object --- tests/instruments/test_adcp.py | 18 +++++++-- tests/instruments/test_argo_float.py | 12 +++++- tests/instruments/test_base.py | 40 +++++++++++++++++--- tests/instruments/test_ctd.py | 36 +++++------------- tests/instruments/test_ctd_bgc.py | 14 +++++-- tests/instruments/test_drifter.py | 21 ++++++++-- tests/instruments/test_ship_underwater_st.py | 15 ++++++-- tests/instruments/test_xbt.py | 15 ++++++-- tests/test_utils.py | 21 +++++----- 9 files changed, 130 insertions(+), 62 deletions(-) diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index a2a5418a..0a88b206 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,10 +4,11 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.adcp import ADCPInstrument -from virtualship.models import Location, Spacetime +from virtualship.instruments.types import InstrumentType +from virtualship.models import Location, Spacetime, Waypoint def test_simulate_adcp(tmpdir) -> None: @@ -77,15 +78,24 @@ def test_simulate_adcp(tmpdir) -> None: }, ) - # dummy expedition and directory for ADCPInstrument + # dummy expedition for ADCPInstrument class DummyExpedition: + class schedule: + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + instrument=InstrumentType.ADCP, + ), + ] + class instruments_config: class adcp_config: max_depth_meter = MAX_DEPTH num_bins = NUM_BINS expedition = DummyExpedition() - from_data = None adcp_instrument = ADCPInstrument(expedition, from_data) diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index cbe25d76..c07b4abb 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -8,6 +8,7 @@ from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime +from virtualship.models.expedition import Waypoint def test_simulate_argo_floats(tmpdir) -> None: @@ -53,9 +54,16 @@ def test_simulate_argo_floats(tmpdir) -> None: ) ] - # dummy expedition and directory for ArgoFloatInstrument + # dummy expedition for ArgoFloatInstrument class DummyExpedition: - pass + class schedule: + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + ), + ] expedition = DummyExpedition() from_data = None diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 29d319ba..fea43f02 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -32,8 +32,14 @@ def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_Fie mock_fieldset.gridset.grids = [MagicMock(negate_depth=MagicMock())] mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() mock_copernicusmarine.open_dataset.return_value = MagicMock() + # Create a mock waypoint with latitude and longitude + mock_waypoint = MagicMock() + mock_waypoint.location.latitude = 1.0 + mock_waypoint.location.longitude = 2.0 + mock_schedule = MagicMock() + mock_schedule.waypoints = [mock_waypoint] dummy = DummyInstrument( - expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + expedition=MagicMock(schedule=mock_schedule), variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -47,8 +53,13 @@ def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_Fie def test_execute_calls_simulate(monkeypatch): + mock_waypoint = MagicMock() + mock_waypoint.location.latitude = 1.0 + mock_waypoint.location.longitude = 2.0 + mock_schedule = MagicMock() + mock_schedule.waypoints = [mock_waypoint] dummy = DummyInstrument( - expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + expedition=MagicMock(schedule=mock_schedule), variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -61,8 +72,13 @@ def test_execute_calls_simulate(monkeypatch): def test_get_spec_value_buffer_and_limit(): + mock_waypoint = MagicMock() + mock_waypoint.location.latitude = 1.0 + mock_waypoint.location.longitude = 2.0 + mock_schedule = MagicMock() + mock_schedule.waypoints = [mock_waypoint] dummy = DummyInstrument( - expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + expedition=MagicMock(schedule=mock_schedule), variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -77,8 +93,13 @@ def test_get_spec_value_buffer_and_limit(): def test_generate_fieldset_combines_fields(monkeypatch): + mock_waypoint = MagicMock() + mock_waypoint.location.latitude = 1.0 + mock_waypoint.location.longitude = 2.0 + mock_schedule = MagicMock() + mock_schedule.waypoints = [mock_waypoint] dummy = DummyInstrument( - expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + expedition=MagicMock(schedule=mock_schedule), variables={"A": "a", "B": "b"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -87,7 +108,9 @@ def test_generate_fieldset_combines_fields(monkeypatch): ) dummy.from_data = None - monkeypatch.setattr(dummy, "_get_copernicus_ds", lambda physical, var: MagicMock()) + monkeypatch.setattr( + dummy, "_get_copernicus_ds", lambda *args, **kwargs: MagicMock() + ) fs_A = MagicMock() fs_B = MagicMock() @@ -102,8 +125,13 @@ def test_generate_fieldset_combines_fields(monkeypatch): def test_load_input_data_error(monkeypatch): + mock_waypoint = MagicMock() + mock_waypoint.location.latitude = 1.0 + mock_waypoint.location.longitude = 2.0 + mock_schedule = MagicMock() + mock_schedule.waypoints = [mock_waypoint] dummy = DummyInstrument( - expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + expedition=MagicMock(schedule=mock_schedule), variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index fff5fc4f..954d0b78 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,10 +8,11 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime +from virtualship.models.expedition import Waypoint def test_simulate_ctds(tmpdir) -> None: @@ -101,35 +102,18 @@ def test_simulate_ctds(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # dummy expedition and directory for CTDInstrument + # dummy expedition for CTDInstrument class DummyExpedition: class schedule: - class space_time_region: - time_range = type( - "TimeRange", - (), - { - "start_time": fieldset.T.grid.time_origin.fulltime( - fieldset.T.grid.time_full[0] - ), - "end_time": fieldset.T.grid.time_origin.fulltime( - fieldset.T.grid.time_full[-1] - ), - }, - )() - spatial_range = type( - "SpatialRange", - (), - { - "minimum_longitude": 0, - "maximum_longitude": 1, - "minimum_latitude": 0, - "maximum_latitude": 1, - }, - )() + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + ), + ] expedition = DummyExpedition() - from_data = None ctd_instrument = CTDInstrument(expedition, from_data) diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index 00f30077..39fa6c1f 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,10 +8,11 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime +from virtualship.models.expedition import Waypoint def test_simulate_ctd_bgcs(tmpdir) -> None: @@ -162,9 +163,16 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # dummy expedition and directory for CTD_BGCInstrument + # dummy expedition for CTD_BGCInstrument class DummyExpedition: - pass + class schedule: + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + ), + ] expedition = DummyExpedition() from_data = None diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index 03d04ea8..51f78883 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,10 +4,11 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime +from virtualship.models.expedition import Waypoint def test_simulate_drifters(tmpdir) -> None: @@ -16,6 +17,8 @@ def test_simulate_drifters(tmpdir) -> None: CONST_TEMPERATURE = 1.0 # constant temperature in fieldset + LIFETIME = datetime.timedelta(days=1) + v = np.full((2, 2, 2), 1.0) u = np.full((2, 2, 2), 1.0) t = np.full((2, 2, 2), CONST_TEMPERATURE) @@ -52,12 +55,22 @@ def test_simulate_drifters(tmpdir) -> None: ), ] - # dummy expedition and directory for DrifterInstrument + # dummy expedition for DrifterInstrument class DummyExpedition: - pass + class schedule: + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + ), + ] + + class instruments_config: + class drifter_config: + lifetime = LIFETIME expedition = DummyExpedition() - from_data = None drifter_instrument = DrifterInstrument(expedition, from_data) diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index e7ca18d1..3f1aae65 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,10 +4,11 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime +from virtualship.models.expedition import Waypoint def test_simulate_ship_underwater_st(tmpdir) -> None: @@ -67,12 +68,18 @@ def test_simulate_ship_underwater_st(tmpdir) -> None: }, ) - # dummy expedition and directory for Underwater_STInstrument + # dummy expedition for Underwater_STInstrument class DummyExpedition: - pass + class schedule: + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + ), + ] expedition = DummyExpedition() - from_data = None st_instrument = Underwater_STInstrument(expedition, from_data) diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index d218025a..c6a36631 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,10 +8,11 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime +from virtualship.models.expedition import Waypoint def test_simulate_xbts(tmpdir) -> None: @@ -95,12 +96,18 @@ def test_simulate_xbts(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # dummy expedition and directory for XBTInstrument + # dummy expedition for XBTInstrument class DummyExpedition: - pass + class schedule: + # ruff: noqa + waypoints = [ + Waypoint( + location=Location(1, 2), + time=base_time, + ), + ] expedition = DummyExpedition() - from_data = None xbt_instrument = XBTInstrument(expedition, from_data) diff --git a/tests/test_utils.py b/tests/test_utils.py index d96f4369..6c9398c3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,4 @@ +import datetime from pathlib import Path import numpy as np @@ -101,11 +102,13 @@ def test_add_dummy_UV_adds_fields(): @pytest.mark.usefixtures("copernicus_no_download") def test_select_product_id(expedition): - """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" + """Should return the physical reanalysis product id via the timings prescribed.""" result = _select_product_id( physical=True, - schedule_start=expedition.schedule.space_time_region.time_range.start_time, - schedule_end=expedition.schedule.space_time_region.time_range.end_time, + schedule_start=datetime.datetime( + 1995, 6, 1, 0, 0, 0 + ), # known to be in reanalysis range + schedule_end=datetime.datetime(1995, 6, 30, 0, 0, 0), username="test", password="test", ) @@ -114,11 +117,11 @@ def test_select_product_id(expedition): @pytest.mark.usefixtures("copernicus_no_download") def test_start_end_in_product_timerange(expedition): - """Should return True for valid range ass determined by the static schedule.yaml file.""" + """Should return True for valid range as determined by the static schedule.yaml file.""" assert _start_end_in_product_timerange( selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", - schedule_start=expedition.schedule.space_time_region.time_range.start_time, - schedule_end=expedition.schedule.space_time_region.time_range.end_time, + schedule_start=datetime.datetime(1995, 6, 1, 0, 0, 0), + schedule_end=datetime.datetime(1995, 6, 30, 0, 0, 0), username="test", password="test", ) @@ -143,8 +146,8 @@ def test_get_bathy_data_local(tmp_path): # should return a FieldSet fieldset = _get_bathy_data( - from_data=tmp_path - ) # TODO: will need domain coords; from waypoints? + min_lat=0.25, max_lat=0.75, min_lon=0.25, max_lon=0.75, from_data=tmp_path + ) assert isinstance(fieldset, FieldSet) assert hasattr(fieldset, "bathymetry") assert np.allclose(fieldset.bathymetry.data, data) @@ -161,7 +164,7 @@ def dummy_copernicusmarine(*args, **kwargs): ) try: - _get_bathy_data() # TODO: will need domain coords; from waypoints? + _get_bathy_data(min_lat=0.25, max_lat=0.75, min_lon=0.25, max_lon=0.75) except RuntimeError as e: assert "copernicusmarine called" in str(e) From 1f778ea6caaa08b820b5ff60f798d6bdad2bdafc Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:36:21 +0100 Subject: [PATCH 08/24] fix assert error messaging --- src/virtualship/instruments/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 4cdb3bdb..e1c5d223 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -63,7 +63,7 @@ def __init__( wp.time for wp in expedition.schedule.waypoints if wp.time is not None ] assert all(earlier <= later for earlier, later in pairwise(wp_times)), ( - "wp_times is not ascending order" + "Waypoint times are not in ascending order" ) self.wp_times = wp_times From 14b88337a6508548800a4d02498147aa6a189b8a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:06:01 +0100 Subject: [PATCH 09/24] remove spatial constraint on fieldset ingestion --- src/virtualship/instruments/base.py | 7 ------- src/virtualship/utils.py | 6 ------ 2 files changed, 13 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index e1c5d223..70564b87 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -148,18 +148,11 @@ def _get_copernicus_ds( variable=var if not physical else None, ) - latlon_buffer = self._get_spec_value( - "buffer", "latlon", 0.25 - ) # [degrees]; default 0.25 deg buffer to ensure coverage in cell edge cases depth_min = self._get_spec_value("limit", "depth_min", None) depth_max = self._get_spec_value("limit", "depth_max", None) return copernicusmarine.open_dataset( dataset_id=product_id, - minimum_longitude=self.min_lon - latlon_buffer, - maximum_longitude=self.max_lon + latlon_buffer, - minimum_latitude=self.min_lat - latlon_buffer, - maximum_latitude=self.max_lat + latlon_buffer, variables=[var], start_datetime=self.min_time, end_datetime=self.max_time + timedelta(days=time_buffer), diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 7bdc93b0..d0a5e06a 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -415,14 +415,8 @@ def _get_bathy_data( ) else: # stream via Copernicus Marine Service - buffer = 0.1 # degrees buffer, always to 0.1 to ensure coverage in edge cases (bathy data grid resolution ~0.8 deg) - ds_bathymetry = copernicusmarine.open_dataset( dataset_id=BATHYMETRY_ID, - minimum_longitude=min_lon - buffer, - maximum_longitude=max_lon + buffer, - minimum_latitude=min_lat - buffer, - maximum_latitude=max_lat + buffer, variables=["deptho"], coordinates_selection_method="outside", ) From 714b0bfa49a8494095b5182d9295bc2513f4e7fb Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 9 Dec 2025 10:02:04 +0100 Subject: [PATCH 10/24] add control over whether to constrain spatial region in fieldset --- src/virtualship/instruments/adcp.py | 7 +++++-- src/virtualship/instruments/argo_float.py | 5 ++++- src/virtualship/instruments/base.py | 16 ++++++++++++++++ src/virtualship/instruments/ctd.py | 7 +++++-- src/virtualship/instruments/ctd_bgc.py | 8 ++++++-- src/virtualship/instruments/drifter.py | 3 ++- .../instruments/ship_underwater_st.py | 7 +++++-- src/virtualship/instruments/xbt.py | 8 ++++++-- 8 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 2a761e14..f35e3b3f 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( @@ -57,6 +57,9 @@ class ADCPInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize ADCPInstrument.""" variables = {"U": "uo", "V": "vo"} + limit_spec = { + "spatial": True + } # spatial limits; lat/lon constrained to waypoint locations + buffer super().__init__( expedition, @@ -65,7 +68,7 @@ def __init__(self, expedition, from_data): allow_time_extrapolation=True, verbose_progress=False, spacetime_buffer_size=None, - limit_spec=None, + limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index cb35c623..98c53e34 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -169,6 +169,9 @@ def __init__(self, expedition, from_data): "latlon": 3.0, # [degrees] "time": 63.0, # [days] } + limit_spec = { + "spatial": True + } # spatial limits; lat/lon constrained to waypoint locations + buffer super().__init__( expedition, @@ -177,7 +180,7 @@ def __init__(self, expedition, from_data): allow_time_extrapolation=False, verbose_progress=True, spacetime_buffer_size=spacetime_buffer_size, - limit_spec=None, + limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 70564b87..8189c6b7 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -148,11 +148,27 @@ def _get_copernicus_ds( variable=var if not physical else None, ) + latlon_buffer = self._get_spec_value( + "buffer", "latlon", 0.25 + ) # [degrees]; default 0.25 deg buffer to ensure coverage in field cell edge cases depth_min = self._get_spec_value("limit", "depth_min", None) depth_max = self._get_spec_value("limit", "depth_max", None) + spatial_constraint = self._get_spec_value("limit", "spatial", True) return copernicusmarine.open_dataset( dataset_id=product_id, + minimum_longitude=self.min_lon - latlon_buffer + if spatial_constraint + else None, + maximum_longitude=self.max_lon + latlon_buffer + if spatial_constraint + else None, + minimum_latitude=self.min_lat - latlon_buffer + if spatial_constraint + else None, + maximum_latitude=self.max_lat + latlon_buffer + if spatial_constraint + else None, variables=[var], start_datetime=self.min_time, end_datetime=self.max_time + timedelta(days=time_buffer), diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 73248cf9..1b6269bc 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType @@ -82,6 +82,9 @@ class CTDInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize CTDInstrument.""" variables = {"S": "so", "T": "thetao"} + limit_spec = { + "spatial": True + } # spatial limits; lat/lon constrained to waypoint locations + buffer super().__init__( expedition, @@ -90,7 +93,7 @@ def __init__(self, expedition, from_data): allow_time_extrapolation=True, verbose_progress=False, spacetime_buffer_size=None, - limit_spec=None, + limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index fab9e07b..3537b468 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -112,6 +112,10 @@ def __init__(self, expedition, from_data): "phyc": "phyc", "nppv": "nppv", } + limit_spec = { + "spatial": True + } # spatial limits; lat/lon constrained to waypoint locations + buffer + super().__init__( expedition, variables, @@ -119,7 +123,7 @@ def __init__(self, expedition, from_data): allow_time_extrapolation=True, verbose_progress=False, spacetime_buffer_size=None, - limit_spec=None, + limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 2fb3e17f..886b20b4 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -67,11 +67,12 @@ def __init__(self, expedition, from_data): """Initialize DrifterInstrument.""" variables = {"U": "uo", "V": "vo", "T": "thetao"} spacetime_buffer_size = { - "latlon": 12.0, # [degrees] + "latlon": None, "time": expedition.instruments_config.drifter_config.lifetime.total_seconds() / (24 * 3600), # [days] } limit_spec = { + "spatial": False, # no spatial limits; generate global fieldset "depth_min": 1.0, # [meters] "depth_max": 1.0, # [meters] } diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 1e66ba50..23d36ec0 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument @@ -62,6 +62,9 @@ def __init__(self, expedition, from_data): "latlon": 0.25, # [degrees] "time": 0.0, # [days] } + limit_spec = { + "spatial": True + } # spatial limits; lat/lon constrained to waypoint locations + buffer super().__init__( expedition, @@ -70,7 +73,7 @@ def __init__(self, expedition, from_data): allow_time_extrapolation=True, verbose_progress=False, spacetime_buffer_size=spacetime_buffer_size, - limit_spec=None, + limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index f0f5d130..333289ac 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -80,6 +80,10 @@ class XBTInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize XBTInstrument.""" variables = {"T": "thetao"} + limit_spec = { + "spatial": True + } # spatial limits; lat/lon constrained to waypoint locations + buffer + super().__init__( expedition, variables, @@ -87,7 +91,7 @@ def __init__(self, expedition, from_data): allow_time_extrapolation=True, verbose_progress=False, spacetime_buffer_size=None, - limit_spec=None, + limit_spec=limit_spec, from_data=from_data, ) From e820584df174fbba26f98141bce9904c023f4ee8 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 9 Dec 2025 10:30:43 +0100 Subject: [PATCH 11/24] give argos a prescribed lifetime --- src/virtualship/instruments/argo_float.py | 3 ++- src/virtualship/static/expedition.yaml | 1 + tests/expedition/expedition_dir/expedition.yaml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 98c53e34..f2a750b8 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -167,7 +167,8 @@ def __init__(self, expedition, from_data): variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} spacetime_buffer_size = { "latlon": 3.0, # [degrees] - "time": 63.0, # [days] + "time": expedition.instruments_config.argo_float.lifetime.total_seconds() + / (24 * 3600), # [days] } limit_spec = { "spatial": True diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 34b8a2df..dcd061fc 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -43,6 +43,7 @@ instruments_config: min_depth_meter: 0.0 vertical_speed_meter_per_second: -0.1 stationkeeping_time_minutes: 20.0 + lifetime_minutes: 90720.0 ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml index cd3f532a..d983bdc0 100644 --- a/tests/expedition/expedition_dir/expedition.yaml +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -30,6 +30,7 @@ instruments_config: min_depth_meter: 0.0 vertical_speed_meter_per_second: -0.1 stationkeeping_time_minutes: 20.0 + lifetime_minutes: 90720.0 ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 From 7369f46b1d44e9e3053adb5ee1abd51a7f2f0d03 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 9 Dec 2025 10:38:54 +0100 Subject: [PATCH 12/24] add lifetime field to expedition models and _plan --- src/virtualship/cli/_plan.py | 1 + src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/models/expedition.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 3428af8a..0115bd8d 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -138,6 +138,7 @@ def log_exception_to_file( {"name": "cycle_days"}, {"name": "drift_days"}, {"name": "stationkeeping_time", "minutes": True}, + {"name": "lifetime", "minutes": True}, ], }, "drifter_config": { diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index f2a750b8..e035c77a 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -167,7 +167,7 @@ def __init__(self, expedition, from_data): variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} spacetime_buffer_size = { "latlon": 3.0, # [degrees] - "time": expedition.instruments_config.argo_float.lifetime.total_seconds() + "time": expedition.instruments_config.argo_float_config.lifetime.total_seconds() / (24 * 3600), # [days] } limit_spec = { diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 88baa09e..a45f83be 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -213,6 +213,11 @@ class ArgoFloatConfig(pydantic.BaseModel): vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) cycle_days: float = pydantic.Field(gt=0.0) drift_days: float = pydantic.Field(gt=0.0) + lifetime: timedelta = pydantic.Field( + serialization_alias="lifetime_minutes", + validation_alias="lifetime_minutes", + gt=timedelta(), + ) stationkeeping_time: timedelta = pydantic.Field( serialization_alias="stationkeeping_time_minutes", @@ -220,6 +225,14 @@ class ArgoFloatConfig(pydantic.BaseModel): gt=timedelta(), ) + @pydantic.field_serializer("lifetime") + def _serialize_lifetime(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("lifetime", mode="before") + def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + @pydantic.field_serializer("stationkeeping_time") def _serialize_stationkeeping_time(self, value: timedelta, _info): return value.total_seconds() / 60.0 From 15848f32a958f46e95cc96162c4a561ec9c621c3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 9 Dec 2025 10:48:33 +0100 Subject: [PATCH 13/24] update test with argo lifetime --- tests/instruments/test_argo_float.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 0b916b29..66331d64 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -20,6 +20,7 @@ def test_simulate_argo_floats(tmpdir) -> None: VERTICAL_SPEED = -0.10 CYCLE_DAYS = 10 DRIFT_DAYS = 9 + LIFETIME = timedelta(days=1) CONST_TEMPERATURE = 1.0 # constant temperature in fieldset CONST_SALINITY = 1.0 # constant salinity in fieldset @@ -75,6 +76,10 @@ class schedule: ), ] + class instruments_config: + class argo_float_config: + lifetime = LIFETIME + expedition = DummyExpedition() from_data = None From ba3793b3c9763877d27288142ce75c988b1341dd Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 9 Dec 2025 13:16:27 +0100 Subject: [PATCH 14/24] remove temporary debugging tools --- src/virtualship/instruments/base.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 8189c6b7..7186b99b 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -113,26 +113,20 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - TMP = True - - if TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print( - f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... " - ) + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") - + spinner.ok("✅\n") else: + print( + f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... " + ) self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, From 340d940021b049849dc355a0c080496c4f7e0254 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Dec 2025 12:18:33 +0000 Subject: [PATCH 15/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- tests/expedition/test_expedition.py | 2 +- tests/test_utils.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index f35e3b3f..17797a41 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index e035c77a..6289d0d6 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 7186b99b..7e129935 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -9,9 +9,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 1b6269bc..eb780d3e 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 3537b468..221cfa12 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 886b20b4..c96b2d86 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 23d36ec0..8b7ef96d 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 333289ac..2412306f 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index d0a5e06a..64e9887c 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -13,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 13b4b689..90027e8e 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -6,8 +6,8 @@ import pyproj import pytest import xarray as xr - from parcels import FieldSet + from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( Expedition, diff --git a/tests/test_utils.py b/tests/test_utils.py index 6c9398c3..deca66d5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,9 +4,9 @@ import numpy as np import pytest import xarray as xr +from parcels import FieldSet import virtualship.utils -from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, From 7eff191287a27bfe1826618ad46afcaa0ae62e8f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:21:08 +0100 Subject: [PATCH 16/24] update argo endtime and give depth lim in fieldset generation --- src/virtualship/instruments/argo_float.py | 24 +++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index e035c77a..ab3f424e 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -171,8 +171,14 @@ def __init__(self, expedition, from_data): / (24 * 3600), # [days] } limit_spec = { - "spatial": True - } # spatial limits; lat/lon constrained to waypoint locations + buffer + "spatial": True, # spatial limits; lat/lon constrained to waypoint locations + buffer + "depth_min": abs( + expedition.instruments_config.argo_float_config.min_depth_meter + ), # [meters] + "depth_max": abs( + expedition.instruments_config.argo_float_config.max_depth_meter + ), # [meters] + } super().__init__( expedition, @@ -189,7 +195,6 @@ def simulate(self, measurements, out_path) -> None: """Simulate Argo float measurements.""" DT = 10.0 # dt of Argo float simulation integrator OUTPUT_DT = timedelta(minutes=5) - ENDTIME = None if len(measurements) == 0: print( @@ -239,15 +244,8 @@ def simulate(self, measurements, out_path) -> None: chunks=[len(argo_float_particleset), 100], ) - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if ENDTIME is None: - actual_endtime = fieldset_endtime - elif ENDTIME > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(ENDTIME) + # endtime + endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) # execute simulation argo_float_particleset.execute( @@ -257,7 +255,7 @@ def simulate(self, measurements, out_path) -> None: _keep_at_surface, _check_error, ], - endtime=actual_endtime, + endtime=endtime, dt=DT, output_file=out_file, verbose_progress=self.verbose_progress, From fd6a3f69afda26cbc25a1797878b240e685a34a5 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:59:08 +0100 Subject: [PATCH 17/24] remove depth lim on fieldset generation --- src/virtualship/instruments/argo_float.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index ab3f424e..962a1889 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -172,12 +172,6 @@ def __init__(self, expedition, from_data): } limit_spec = { "spatial": True, # spatial limits; lat/lon constrained to waypoint locations + buffer - "depth_min": abs( - expedition.instruments_config.argo_float_config.min_depth_meter - ), # [meters] - "depth_max": abs( - expedition.instruments_config.argo_float_config.max_depth_meter - ), # [meters] } super().__init__( From b00c85ce3e7a991d4dbc55f9f528a5c85b4f1410 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 23 Dec 2025 09:46:29 +0000 Subject: [PATCH 18/24] clean up comment --- src/virtualship/instruments/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 7186b99b..bb9a8138 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -182,8 +182,6 @@ def _generate_fieldset(self) -> FieldSet: time_buffer = self._get_spec_value("buffer", "time", 0.0) - # TODO: also limit from-data to spatial domain? - for key in keys: var = self.variables[key] if self.from_data is not None: # load from local data From ff43127e7ef4ee7f9e34a912b20fd1526b24c3e2 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:38:52 +0000 Subject: [PATCH 19/24] change lifetime in expedition model to be in days units --- src/virtualship/models/expedition.py | 8 ++++---- src/virtualship/static/expedition.yaml | 4 ++-- tests/expedition/expedition_dir/expedition.yaml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index a45f83be..8c6df96e 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -214,8 +214,8 @@ class ArgoFloatConfig(pydantic.BaseModel): cycle_days: float = pydantic.Field(gt=0.0) drift_days: float = pydantic.Field(gt=0.0) lifetime: timedelta = pydantic.Field( - serialization_alias="lifetime_minutes", - validation_alias="lifetime_minutes", + serialization_alias="lifetime_days", + validation_alias="lifetime_days", gt=timedelta(), ) @@ -335,8 +335,8 @@ class DrifterConfig(pydantic.BaseModel): depth_meter: float = pydantic.Field(le=0.0) lifetime: timedelta = pydantic.Field( - serialization_alias="lifetime_minutes", - validation_alias="lifetime_minutes", + serialization_alias="lifetime_days", + validation_alias="lifetime_days", gt=timedelta(), ) stationkeeping_time: timedelta = pydantic.Field( diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index dcd061fc..496adba2 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -43,7 +43,7 @@ instruments_config: min_depth_meter: 0.0 vertical_speed_meter_per_second: -0.1 stationkeeping_time_minutes: 20.0 - lifetime_minutes: 90720.0 + lifetime_days: 90720.0 ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 @@ -54,7 +54,7 @@ instruments_config: stationkeeping_time_minutes: 50.0 drifter_config: depth_meter: -1.0 - lifetime_minutes: 60480.0 + lifetime_days: 60480.0 stationkeeping_time_minutes: 20.0 xbt_config: max_depth_meter: -285.0 diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml index d983bdc0..f7097263 100644 --- a/tests/expedition/expedition_dir/expedition.yaml +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -30,7 +30,7 @@ instruments_config: min_depth_meter: 0.0 vertical_speed_meter_per_second: -0.1 stationkeeping_time_minutes: 20.0 - lifetime_minutes: 90720.0 + lifetime_days: 90720.0 ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 @@ -41,7 +41,7 @@ instruments_config: stationkeeping_time_minutes: 50.0 drifter_config: depth_meter: -1.0 - lifetime_minutes: 40320.0 + lifetime_days: 40320.0 stationkeeping_time_minutes: 20.0 ship_underwater_st_config: period_minutes: 5.0 From 87de2102ef4815c7e473ce858a92ce60378fcc13 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:05:14 +0000 Subject: [PATCH 20/24] update _plan logic (+ more changes to expedition model) to handle lifetime in minutes --- src/virtualship/cli/_plan.py | 29 ++++++++++++++----- src/virtualship/models/expedition.py | 22 +++++++------- src/virtualship/static/expedition.yaml | 4 +-- src/virtualship/utils.py | 13 ++++++--- .../expedition/expedition_dir/expedition.yaml | 4 +-- 5 files changed, 45 insertions(+), 27 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 0115bd8d..81c8f857 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -138,7 +138,7 @@ def log_exception_to_file( {"name": "cycle_days"}, {"name": "drift_days"}, {"name": "stationkeeping_time", "minutes": True}, - {"name": "lifetime", "minutes": True}, + {"name": "lifetime", "days": True}, ], }, "drifter_config": { @@ -146,7 +146,7 @@ def log_exception_to_file( "title": "Drifter", "attributes": [ {"name": "depth_meter"}, - {"name": "lifetime", "minutes": True}, + {"name": "lifetime", "days": True}, {"name": "stationkeeping_time", "minutes": True}, ], }, @@ -264,7 +264,10 @@ def compose(self) -> ComposeResult: with Container(classes="instrument-config"): for attr_meta in attributes: attr = attr_meta["name"] - is_minutes = attr_meta.get("minutes", False) + is_minutes, is_days = ( + attr_meta.get("minutes", False), + attr_meta.get("days", False), + ) validators = group_validators(config_class, attr) if config_instance: raw_value = getattr(config_instance, attr, "") @@ -275,16 +278,23 @@ def compose(self) -> ComposeResult: ) except AttributeError: value = str(raw_value) + elif is_days and raw_value != "": + try: + value = str( + raw_value.total_seconds() / 86400.0 + ) + except AttributeError: + value = str(raw_value) else: value = str(raw_value) else: value = "" label = f"{attr.replace('_', ' ').title()}:" - yield Label( - label - if not is_minutes - else label.replace(":", " Minutes:") - ) + if is_minutes: + label = label.replace(":", " Minutes:") + elif is_days: + label = label.replace(":", " Days:") + yield Label(label) yield Input( id=f"{instrument_name}_{attr}", type=type_to_textual( @@ -392,11 +402,14 @@ def _update_instrument_configs(self): for attr_meta in attributes: attr = attr_meta["name"] is_minutes = attr_meta.get("minutes", False) + is_days = attr_meta.get("days", False) input_id = f"{instrument_name}_{attr}" value = self.query_one(f"#{input_id}").value field_type = get_field_type(config_class, attr) if is_minutes and field_type is datetime.timedelta: value = datetime.timedelta(minutes=float(value)) + elif is_days and field_type is datetime.timedelta: + value = datetime.timedelta(days=float(value)) else: value = field_type(value) kwargs[attr] = value diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 8c6df96e..b8f65558 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -14,7 +14,7 @@ from virtualship.utils import ( _get_bathy_data, _get_waypoint_latlons, - _validate_numeric_mins_to_timedelta, + _validate_numeric_to_timedelta, ) from .location import Location @@ -227,11 +227,11 @@ class ArgoFloatConfig(pydantic.BaseModel): @pydantic.field_serializer("lifetime") def _serialize_lifetime(self, value: timedelta, _info): - return value.total_seconds() / 60.0 + return value.total_seconds() / 86400.0 # [days] @pydantic.field_validator("lifetime", mode="before") def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "days") @pydantic.field_serializer("stationkeeping_time") def _serialize_stationkeeping_time(self, value: timedelta, _info): @@ -239,7 +239,7 @@ def _serialize_stationkeeping_time(self, value: timedelta, _info): @pydantic.field_validator("stationkeeping_time", mode="before") def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "minutes") model_config = pydantic.ConfigDict(populate_by_name=True) @@ -263,7 +263,7 @@ def _serialize_period(self, value: timedelta, _info): @pydantic.field_validator("period", mode="before") def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "minutes") class CTDConfig(pydantic.BaseModel): @@ -285,7 +285,7 @@ def _serialize_stationkeeping_time(self, value: timedelta, _info): @pydantic.field_validator("stationkeeping_time", mode="before") def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "minutes") class CTD_BGCConfig(pydantic.BaseModel): @@ -307,7 +307,7 @@ def _serialize_stationkeeping_time(self, value: timedelta, _info): @pydantic.field_validator("stationkeeping_time", mode="before") def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "minutes") class ShipUnderwaterSTConfig(pydantic.BaseModel): @@ -327,7 +327,7 @@ def _serialize_period(self, value: timedelta, _info): @pydantic.field_validator("period", mode="before") def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "minutes") class DrifterConfig(pydantic.BaseModel): @@ -349,11 +349,11 @@ class DrifterConfig(pydantic.BaseModel): @pydantic.field_serializer("lifetime") def _serialize_lifetime(self, value: timedelta, _info): - return value.total_seconds() / 60.0 + return value.total_seconds() / 86400.0 # [days] @pydantic.field_validator("lifetime", mode="before") def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "days") @pydantic.field_serializer("stationkeeping_time") def _serialize_stationkeeping_time(self, value: timedelta, _info): @@ -361,7 +361,7 @@ def _serialize_stationkeeping_time(self, value: timedelta, _info): @pydantic.field_validator("stationkeeping_time", mode="before") def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) + return _validate_numeric_to_timedelta(value, "minutes") class XBTConfig(pydantic.BaseModel): diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 496adba2..388961c0 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -43,7 +43,7 @@ instruments_config: min_depth_meter: 0.0 vertical_speed_meter_per_second: -0.1 stationkeeping_time_minutes: 20.0 - lifetime_days: 90720.0 + lifetime_days: 63.0 ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 @@ -54,7 +54,7 @@ instruments_config: stationkeeping_time_minutes: 50.0 drifter_config: depth_meter: -1.0 - lifetime_days: 60480.0 + lifetime_days: 42.0 stationkeeping_time_minutes: 20.0 xbt_config: max_depth_meter: -285.0 diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 64e9887c..724c90fd 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -13,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: @@ -186,11 +186,16 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 expedition.to_yaml(yaml_output_path) -def _validate_numeric_mins_to_timedelta(value: int | float | timedelta) -> timedelta: - """Convert minutes to timedelta when reading.""" +def _validate_numeric_to_timedelta( + value: int | float | timedelta, unit: str +) -> timedelta: + """Convert to timedelta when reading.""" if isinstance(value, timedelta): return value - return timedelta(minutes=value) + if unit == "minutes": + return timedelta(minutes=float(value)) + elif unit == "days": + return timedelta(days=float(value)) def _get_expedition(expedition_dir: Path) -> Expedition: diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml index f7097263..65e0b540 100644 --- a/tests/expedition/expedition_dir/expedition.yaml +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -30,7 +30,7 @@ instruments_config: min_depth_meter: 0.0 vertical_speed_meter_per_second: -0.1 stationkeeping_time_minutes: 20.0 - lifetime_days: 90720.0 + lifetime_days: 63.0 ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 @@ -41,7 +41,7 @@ instruments_config: stationkeeping_time_minutes: 50.0 drifter_config: depth_meter: -1.0 - lifetime_days: 40320.0 + lifetime_days: 28.0 stationkeeping_time_minutes: 20.0 ship_underwater_st_config: period_minutes: 5.0 From 593e0f2cf529c4a2701f71d905cdcc3f84d2d3c5 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:10:02 +0000 Subject: [PATCH 21/24] small refactor for readability --- src/virtualship/instruments/base.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index db76c3a7..3b670478 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -9,9 +9,9 @@ import copernicusmarine import xarray as xr -from parcels import FieldSet from yaspin import yaspin +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, @@ -149,20 +149,17 @@ def _get_copernicus_ds( depth_max = self._get_spec_value("limit", "depth_max", None) spatial_constraint = self._get_spec_value("limit", "spatial", True) + min_lon_bound = self.min_lon - latlon_buffer if spatial_constraint else None + max_lon_bound = self.max_lon + latlon_buffer if spatial_constraint else None + min_lat_bound = self.min_lat - latlon_buffer if spatial_constraint else None + max_lat_bound = self.max_lat + latlon_buffer if spatial_constraint else None + return copernicusmarine.open_dataset( dataset_id=product_id, - minimum_longitude=self.min_lon - latlon_buffer - if spatial_constraint - else None, - maximum_longitude=self.max_lon + latlon_buffer - if spatial_constraint - else None, - minimum_latitude=self.min_lat - latlon_buffer - if spatial_constraint - else None, - maximum_latitude=self.max_lat + latlon_buffer - if spatial_constraint - else None, + minimum_longitude=min_lon_bound, + maximum_longitude=max_lon_bound, + minimum_latitude=min_lat_bound, + maximum_latitude=max_lat_bound, variables=[var], start_datetime=self.min_time, end_datetime=self.max_time + timedelta(days=time_buffer), From 6ae13173e1384e12af0345a039a0b878ece74b7b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:11:46 +0000 Subject: [PATCH 22/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/base.py | 2 +- src/virtualship/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3b670478..984e4abf 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -9,9 +9,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 724c90fd..c8493d7f 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -13,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: From 2e8c14a5a13a445fb975d749593af24f6e61cb57 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 19 Dec 2025 11:16:16 +0000 Subject: [PATCH 23/24] remove broken links --- .../assignments/Research_proposal_intro.ipynb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/assignments/Research_proposal_intro.ipynb b/docs/user-guide/assignments/Research_proposal_intro.ipynb index 88f54bc8..45f65947 100644 --- a/docs/user-guide/assignments/Research_proposal_intro.ipynb +++ b/docs/user-guide/assignments/Research_proposal_intro.ipynb @@ -150,9 +150,15 @@ "\n", "Finally, fit your equipment in the 20-foot container. The container will be sent to your port of departure ahead of time with a cargo boat, so make sure you are packed in time for this transfer. Remember there are no shops at sea, so think carefully and plan ahead. \n", "\n", - "![Equipment preparation NIOZ](https://www.nioz.nl/application/files/9116/7500/3457/2023-01-16-packing.jpg) \n", - "![Equipment loading](https://www.nioz.nl/application/files/7416/7810/2265/2023-03-06-container-shifting.jpg) " + "\n", + "" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { From 984d441acbe3d0a3d032f3772d9bc186fd0d7373 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:33:16 +0000 Subject: [PATCH 24/24] update pre-downloaded data documentation to explain argo/drifter lifetime bounds --- docs/user-guide/documentation/pre_download_data.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index fc55f339..f37912e9 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -20,6 +20,10 @@ In addition, all pre-downloaded data must be split into separate files per times **Monthly data**: when using monthly data, ensure that your final .nc file download is for the month *after* your expedition schedule end date. This is to ensure that a Parcels FieldSet can be generated under-the-hood which fully covers the expedition period. For example, if your expedition runs from 1st May to 15th May, your final monthly data file should be in June. Daily data files only need to cover the expedition period exactly. ``` +```{note} +**Argo and Drifter data**: if using Argo floats or Drifters in your expedition, ensure that: 1) the temporal extent of the downloaded data also accounts for the full *lifetime* of the instruments, not just the expedition period, and 2) the spatial bounds of the downloaded data also accounts for the likely drift distance of the instruments over their lifetimes. Otherwise, simulations will end prematurely (out-of-bounds errors) when the data runs out. +``` + Further, VirtualShip expects pre-downloaded data to be organised in a specific directory & filename structure within the specified local data directory. The expected structure is as outlined in the subsequent sections. #### Directory structure