'eof' model fit error

[gkb999]

Describe the bug
A clear and concise description of what the bug is.

Reproducible Minimal Working Example
Provide a concise Python code snippet that demonstrates the issue. To display the code clearly, use GitHub Flavored Markdown for formatting:

My data looks like:

<xarray.Dataset> Size: 5MB
Dimensions:    (time: 420, bnds: 2, plev: 1, lat: 21, lon: 128)
Coordinates:
  * plev       (plev) float64 8B 5e+04
  * lat        (lat) float64 168B -87.86 -85.1 -82.31 ... -37.67 -34.88 -32.09
  * lon        (lon) float64 1kB -177.2 -174.4 -171.6 ... 174.4 177.2 180.0
  * time       (time) object 3kB 2015-01-16 12:00:00 ... 2049-12-16 12:00:00
Dimensions without coordinates: bnds
Data variables:
    time_bnds  (time, bnds) object 7kB ...
    lat_bnds   (lat, bnds) float64 336B -90.0 -86.58 -86.58 ... -33.49 -30.7
    lon_bnds   (lon, bnds) float64 2kB -178.6 -175.8 -175.8 ... 178.6 -178.6
    zg         (time, plev, lat, lon) float32 5MB 5.057e+03 ... 5.836e+03...

Expected behavior
I ran the following expecting spatial patterns and PCs of the variable.

model = xe.single.EOF(use_coslat=True)

Converted lat-lon,

if 'longitude' in H500_f.coords:
    H500_f = H500_f.rename({'lon_bnds': 'lon'})
if 'latitude' in H500_f.coords:
    H500_f = H500_f.rename({'lat_bnds': 'lat'})

Yet, When I fit the model:

model.fit(H500_f, dim="time")

I get the following error:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\dataarray.py:892, in DataArray._getitem_coord(self, key)
    891 try:
--> 892     var = self._coords[key]
    893 except KeyError:

KeyError: 'lat'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_10480\2310289397.py in ?()
----> 1 model.fit(H500_f, dim="time")

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\single\base_model_single_set.py in ?(self, X, dim, weights)
    147 
    148         self.sample_dims = convert_to_dim_type(dim)
    149 
    150         # Preprocess the data & transform to 2D
--> 151         data2D: DataArray = self.preprocessor.fit_transform(
    152             X, self.sample_dims, weights
    153         )
    154 

~\anaconda3\envs\xEOF\Lib\site-packages\sklearn\utils\_set_output.py in ?(self, X, *args, **kwargs)
    317     @wraps(f)
    318     def wrapped(self, X, *args, **kwargs):
--> 319         data_to_wrap = f(self, X, *args, **kwargs)
    320         if isinstance(data_to_wrap, tuple):
    321             # only wrap the first output for cross decomposition
    322             return_tuple = (

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\preprocessor.py in ?(self, X, sample_dims, weights)
    265         weights: list[Data] | Data | None = None,
    266     ) -> DataArray:
    267         # Take advantage of the fact that `.fit()` already transforms the data
    268         # to avoid duplicate computation
--> 269         self, X = self._fit_algorithm(X, sample_dims, weights)
    270         return X

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\preprocessor.py in ?(self, X, sample_dims, weights)
    206         weights = process_parameter("weights", weights, None, self.n_data)
    207 
    208         # 1 | Center, scale and weigh the data
    209         scaler_iterkwargs = {"weights": weights}
--> 210         X = self.scaler.fit_transform(
    211             X=X,
    212             sample_dims=sample_dims,
    213             feature_dims=feature_dims,

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\list_processor.py in ?(self, X, sample_dims, feature_dims, iter_kwargs)
     83         sample_dims: Dims,
     84         feature_dims: DimsList,
     85         iter_kwargs: dict[str, list[Any]] = {},
     86     ) -> list[Data]:
---> 87         return self.fit(X, sample_dims, feature_dims, iter_kwargs).transform(X)  # type: ignore

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\list_processor.py in ?(self, X, sample_dims, feature_dims, iter_kwargs)
     66             # Add transformer specific keyword arguments
     67             # For iterable kwargs, use the i-th element of the iterable
     68             kwargs = {k: v[i] for k, v in self._iter_kwargs.items()}
     69             proc: T = self.transformer_class(**self.init_kwargs)
---> 70             proc.fit(x, sample_dims, feature_dims[i], **kwargs)
     71             self.transformers.append(proc)
     72         return self

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\scaler.py in ?(self, X, sample_dims, feature_dims, weights)
    107                 min=np.finfo(np.float32).eps
    108             )
    109 
    110         if params["with_coslat"]:
--> 111             self.coslat_weights_: DataVar = compute_sqrt_cos_lat_weights(
    112                 data=X, feature_dims=self.feature_dims
    113             )
    114 

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\utils\xarray_utils.py in ?(data, feature_dims)
    127         weights.name = "coslat_weights"
    128         return weights
    129     elif isinstance(data, xr.Dataset):
    130         return xr.Dataset(
--> 131             {
    132                 var: compute_sqrt_cos_lat_weights(da, feature_dims)
    133                 for var, da in data.data_vars.items()
    134             }

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\utils\xarray_utils.py in ?(.0)
    131 def compute_sqrt_cos_lat_weights(data: DataVar, feature_dims: Dims) -> DataVar:
--> 132     """Compute the square root of cosine of latitude weights.
    133 
    134     Parameters
    135     ----------

~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\utils\xarray_utils.py in ?(data, feature_dims)
    119 
    120     if isinstance(data, xr.DataArray):
    121         lat_dim = extract_latitude_dimension(feature_dims)
    122 
--> 123         latitudes = data.coords[lat_dim]
    124         weights = sqrt_cos_lat_weights(latitudes)
    125         # Features that cannot be associated to a latitude receive a weight of 1
    126         # weights = weights.where(weights.notnull(), 1)

~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\coordinates.py in ?(self, key)
    927     def __getitem__(self, key: Hashable) -> T_DataArray:
--> 928         return self._data._getitem_coord(key)

~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\dataarray.py in ?(self, key)
    891         try:
    892             var = self._coords[key]
    893         except KeyError:
    894             dim_sizes = dict(zip(self.dims, self.shape, strict=True))
--> 895             _, key, var = _get_virtual_variable(self._coords, key, dim_sizes)
    896 
    897         return self._replace_maybe_drop_dims(var, name=key)

~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\dataset.py in ?(variables, key, dim_sizes)
    217         raise KeyError(key)
    218 
    219     split_key = key.split(".", 1)
    220     if len(split_key) != 2:
--> 221         raise KeyError(key)
    222 
    223     ref_name, var_name = split_key
    224     ref_var = variables[ref_name]

KeyError: 'lat'

Desktop (please complete the following information):

• OS: Windows
• xeofs version 3.0.4

[nicrie]

hey @gkb999, just checking: are you sure you want to run an EOF model on all variables in your dataset? xeofs wraps everything into a single matrix, but it looks like your dataset includes a few variables that just define spatial and temporal boundaries (which you probably don't want to include).
I'm guessing you're mainly interested in the zgvariable with shape (time, plev, lat, lon), right? If so, does it work if you fit themodel on just this DataArray: da = H500_f.zg?

[gkb999]

Hi @nicrie
I got the issue after you mentione dthis, that dara infact had around 7 dimensions.
Enough cleaning has been done.
Now, I have only 3 dimensions, but the data has nan values.

Here's an overview:
Data looks like:

Dimensions:         (lon: 291, lat: 360, time: 420)
Coordinates:
  * lon             (lon) int32 1kB 0 1 2 3 4 5 6 ... 285 286 287 288 289 290
  * lat             (lat) int32 1kB 0 1 2 3 4 5 6 ... 354 355 356 357 358 359
    latitude        (lon, lat) float64 838kB -78.39 -78.39 ... 50.23 50.01
    longitude       (lon, lat) float64 838kB 73.5 74.5 75.5 ... 72.96 72.99
  * time            (time) object 3kB 2015-01-16 12:00:00 ... 2049-12-16 12:0...
    type            (time) |S7 3kB ...
    member_id       (time) <U8 13kB ...
    dcpp_init_year  (time) float64 3kB ...
    month           (time) int64 3kB ...
Data variables:
    siconc          (time, lon, lat) float32 176MB nan nan nan ... nan nan nan

I renamed lat & lon.

sic = sic.rename({'j':'lat', 'i':'lon'})

Check for nans and amsked them!

has_nans = sic.siconc.isnull().any()
sic_no_nans = sic.siconc.fillna(0)

Now, ran EOF

model = xe.single.EOF(use_coslat=True)
model.fit(sic, dim="time")

It threw no erros and model looked succesful. Although the spatial patterns did not appear like they always do.
print(components) gives

Dimensions:    (lon: 291, lat: 360, mode: 2)
Coordinates:
  * lon        (lon) int32 1kB 0 1 2 3 4 5 6 7 ... 284 285 286 287 288 289 290
  * lat        (lat) int32 1kB 0 1 2 3 4 5 6 7 ... 353 354 355 356 357 358 359
  * mode       (mode) int64 16B 1 2
    latitude   (lon, lat) float64 838kB -78.39 -78.39 -78.39 ... 50.23 50.01
    longitude  (lon, lat) float64 838kB 73.5 74.5 75.5 ... 72.95 72.96 72.99
Data variables:
    siconc     (mode, lon, lat) float64 2MB nan nan nan nan ... nan nan nan nan

components.siconc.sel(mode=1) looks like:

My questions,

1. How can I completely get rid of nans not just while processing/plotting?
2. The spatial patterns are completely not as expected. Where am I going wrong?

[nicrie]

There seems to be a problem in your spatial coordinates. For instance, check your lat coordinate, it reads: 0 1 2 3 4 5 6 ... 354 355 356 357 358 359. What does that mean? I guess that's why your spatial projection seems wrong.

[gkb999]

Thats i and j actually. It is unstructured curvilinear grid with (i, j) and associated latitude, longitude.
Originally data looks like:

How can I use the latitude and longitude?
I renamed them for 'lat' 'lon', but the data array only shows i,j,time. (reference picture below)

So, for obvious reasons, I get this error

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[29], line 2
      1 model = xe.single.EOF(use_coslat=True)
----> 2 model.fit(SIC, dim="time")

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\single\base_model_single_set.py:151, in BaseModelSingleSet.fit(self, X, dim, weights)
    148 self.sample_dims = convert_to_dim_type(dim)
    150 # Preprocess the data & transform to 2D
--> 151 data2D: DataArray = self.preprocessor.fit_transform(
    152     X, self.sample_dims, weights
    153 )
    155 self._fit_algorithm(data2D)
    157 if self._params["compute"]:

File ~\anaconda3\envs\xEOF\Lib\site-packages\sklearn\utils\_set_output.py:319, in _wrap_method_output.<locals>.wrapped(self, X, *args, **kwargs)
    317 @wraps(f)
    318 def wrapped(self, X, *args, **kwargs):
--> 319     data_to_wrap = f(self, X, *args, **kwargs)
    320     if isinstance(data_to_wrap, tuple):
    321         # only wrap the first output for cross decomposition
    322         return_tuple = (
    323             _wrap_data_with_container(method, data_to_wrap[0], X, self),
    324             *data_to_wrap[1:],
    325         )

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\preprocessor.py:269, in Preprocessor.fit_transform(self, X, sample_dims, weights)
    261 def fit_transform(
    262     self,
    263     X: list[Data] | Data,
   (...)
    267     # Take advantage of the fact that `.fit()` already transforms the data
    268     # to avoid duplicate computation
--> 269     self, X = self._fit_algorithm(X, sample_dims, weights)
    270     return X

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\preprocessor.py:210, in Preprocessor._fit_algorithm(self, X, sample_dims, weights)
    208 # 1 | Center, scale and weigh the data
    209 scaler_iterkwargs = {"weights": weights}
--> 210 X = self.scaler.fit_transform(
    211     X=X,
    212     sample_dims=sample_dims,
    213     feature_dims=feature_dims,
    214     iter_kwargs=scaler_iterkwargs,
    215 )
    216 # 2 | Rename dimensions
    217 X = self.renamer.fit_transform(X, sample_dims, feature_dims)

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\list_processor.py:87, in GenericListTransformer.fit_transform(self, X, sample_dims, feature_dims, iter_kwargs)
     80 def fit_transform(
     81     self,
     82     X: list[Data],
   (...)
     85     iter_kwargs: dict[str, list[Any]] = {},
     86 ) -> list[Data]:
---> 87     return self.fit(X, sample_dims, feature_dims, iter_kwargs).transform(X)

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\list_processor.py:70, in GenericListTransformer.fit(self, X, sample_dims, feature_dims, iter_kwargs)
     68     kwargs = {k: v[i] for k, v in self._iter_kwargs.items()}
     69     proc: T = self.transformer_class(**self.init_kwargs)
---> 70     proc.fit(x, sample_dims, feature_dims[i], **kwargs)
     71     self.transformers.append(proc)
     72 return self

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\scaler.py:111, in Scaler.fit(self, X, sample_dims, feature_dims, weights)
    106     self.std_: DataVar = X.std(self.sample_dims).clip(
    107         min=np.finfo(np.float32).eps
    108     )
    110 if params["with_coslat"]:
--> 111     self.coslat_weights_: DataVar = compute_sqrt_cos_lat_weights(
    112         data=X, feature_dims=self.feature_dims
    113     )
    115 # Convert None weights to ones
    116 self.weights_: DataVar = self._process_weights(X, weights)

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\utils\xarray_utils.py:121, in compute_sqrt_cos_lat_weights(data, feature_dims)
    104 """Compute the square root of cosine of latitude weights.
    105 
    106 Parameters
   (...)
    117 
    118 """
    120 if isinstance(data, xr.DataArray):
--> 121     lat_dim = extract_latitude_dimension(feature_dims)
    123     latitudes = data.coords[lat_dim]
    124     weights = sqrt_cos_lat_weights(latitudes)

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\utils\xarray_utils.py:149, in extract_latitude_dimension(feature_dims)
    146 lat_dim = set(feature_dims) & set(VALID_LATITUDE_NAMES)
    148 if len(lat_dim) == 0:
--> 149     raise ValueError(
    150         "No latitude coordinate was found to compute coslat weights. Must be one of the following: {:}".format(
    151             VALID_LATITUDE_NAMES
    152         )
    153     )
    154 elif len(lat_dim) == 1:
    155     return lat_dim.pop()

ValueError: No latitude coordinate was found to compute coslat weights. Must be one of the following: ['latitude', 'lats', 'lat', 'Latitude', 'Lats', 'Lat', 'LATITUDE', 'LATS', 'LAT']

How can I modify the lon,lat now? i and j are 1D, where as latitude and longide are geo2d.

[gkb999]

Hi @nicrie !!!
Thanks for pointing that out.
It is a curvilinear grid as mentioned below.
I regridded to lat-lon one!!! Here's the sample.
Original data:

Regridded data:

Regridded data when plotted:

But now, after regrdding when I try to fit the new regridded data in to model - EOF, I get the following error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[9], line 2
      1 model = xe.single.EOF(use_coslat=True)
----> 2 model.fit(sic.siconc, dim="time")

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\single\base_model_single_set.py:151, in BaseModelSingleSet.fit(self, X, dim, weights)
    148 self.sample_dims = convert_to_dim_type(dim)
    150 # Preprocess the data & transform to 2D
--> 151 data2D: DataArray = self.preprocessor.fit_transform(
    152     X, self.sample_dims, weights
    153 )
    155 self._fit_algorithm(data2D)
    157 if self._params["compute"]:

File ~\anaconda3\envs\xEOF\Lib\site-packages\sklearn\utils\_set_output.py:319, in _wrap_method_output.<locals>.wrapped(self, X, *args, **kwargs)
    317 @wraps(f)
    318 def wrapped(self, X, *args, **kwargs):
--> 319     data_to_wrap = f(self, X, *args, **kwargs)
    320     if isinstance(data_to_wrap, tuple):
    321         # only wrap the first output for cross decomposition
    322         return_tuple = (
    323             _wrap_data_with_container(method, data_to_wrap[0], X, self),
    324             *data_to_wrap[1:],
    325         )

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\preprocessor.py:269, in Preprocessor.fit_transform(self, X, sample_dims, weights)
    261 def fit_transform(
    262     self,
    263     X: list[Data] | Data,
   (...)
    267     # Take advantage of the fact that `.fit()` already transforms the data
    268     # to avoid duplicate computation
--> 269     self, X = self._fit_algorithm(X, sample_dims, weights)
    270     return X

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\preprocessor.py:210, in Preprocessor._fit_algorithm(self, X, sample_dims, weights)
    208 # 1 | Center, scale and weigh the data
    209 scaler_iterkwargs = {"weights": weights}
--> 210 X = self.scaler.fit_transform(
    211     X=X,
    212     sample_dims=sample_dims,
    213     feature_dims=feature_dims,
    214     iter_kwargs=scaler_iterkwargs,
    215 )
    216 # 2 | Rename dimensions
    217 X = self.renamer.fit_transform(X, sample_dims, feature_dims)

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\list_processor.py:87, in GenericListTransformer.fit_transform(self, X, sample_dims, feature_dims, iter_kwargs)
     80 def fit_transform(
     81     self,
     82     X: list[Data],
   (...)
     85     iter_kwargs: dict[str, list[Any]] = {},
     86 ) -> list[Data]:
---> 87     return self.fit(X, sample_dims, feature_dims, iter_kwargs).transform(X)

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\list_processor.py:70, in GenericListTransformer.fit(self, X, sample_dims, feature_dims, iter_kwargs)
     68     kwargs = {k: v[i] for k, v in self._iter_kwargs.items()}
     69     proc: T = self.transformer_class(**self.init_kwargs)
---> 70     proc.fit(x, sample_dims, feature_dims[i], **kwargs)
     71     self.transformers.append(proc)
     72 return self

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\scaler.py:116, in Scaler.fit(self, X, sample_dims, feature_dims, weights)
    111     self.coslat_weights_: DataVar = compute_sqrt_cos_lat_weights(
    112         data=X, feature_dims=self.feature_dims
    113     )
    115 # Convert None weights to ones
--> 116 self.weights_: DataVar = self._process_weights(X, weights)
    118 if self.get_params()["compute"]:
    119     (self.mean_, self.std_, self.coslat_weights_, self.weights_) = dask.compute(
    120         self.mean_,
    121         self.std_,
    122         self.coslat_weights_,
    123         self.weights_,
    124     )

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\preprocessing\scaler.py:63, in Scaler._process_weights(self, X, weights)
     61 def _process_weights(self, X: DataVarBound, weights) -> DataVarBound:
     62     if weights is None:
---> 63         wghts: DataVarBound = feature_ones_like(X, self.feature_dims)
     64     else:
     65         wghts: DataVarBound = weights

File ~\anaconda3\envs\xEOF\Lib\site-packages\xeofs\utils\xarray_utils.py:83, in feature_ones_like(data, feature_dims)
     81     feature_coords = {dim: data[dim] for dim in valid_dims}
     82     shape = tuple(coords.size for coords in feature_coords.values())
---> 83     return xr.DataArray(
     84         np.ones(shape, dtype=float),
     85         dims=tuple(valid_dims),
     86         coords=feature_coords,
     87     )
     88 elif isinstance(data, xr.Dataset):
     89     return xr.Dataset(
     90         {
     91             var: feature_ones_like(da, feature_dims)
     92             for var, da in data.data_vars.items()
     93         }
     94     )

File ~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\dataarray.py:484, in DataArray.__init__(self, data, coords, dims, name, attrs, indexes, fastpath)
    482 data = _check_data_shape(data, coords, dims)
    483 data = as_compatible_data(data)
--> 484 coords, dims = _infer_coords_and_dims(data.shape, coords, dims)
    485 variable = Variable(dims, data, attrs, fastpath=True)
    487 if not isinstance(coords, Coordinates):

File ~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\dataarray.py:214, in _infer_coords_and_dims(shape, coords, dims)
    211             var.dims = (dim,)
    212             new_coords[dim] = var.to_index_variable()
--> 214 _check_coords_dims(shape, new_coords, dims_tuple)
    216 return new_coords, dims_tuple

File ~\anaconda3\envs\xEOF\Lib\site-packages\xarray\core\dataarray.py:144, in _check_coords_dims(shape, coords, dim)
    142 for d, s in v.sizes.items():
    143     if s != sizes[d]:
--> 144         raise ValueError(
    145             f"conflicting sizes for dimension {d!r}: "
    146             f"length {sizes[d]} on the data but length {s} on "
    147             f"coordinate {k!r}"
    148         )

ValueError: conflicting sizes for dimension 'lat': length 4040 on the data but length 40 on coordinate 'lon'