# (C) Crown Copyright, Met Office. All rights reserved.
#
# This file is part of UG-ANTS and is released under the BSD 3-Clause license.
# See LICENSE.txt in the root of the repository for full licensing details.
"""Implementation for the convert to XIOS application.
This application converts a UGrid netCDF file to a UGrid netCDF file with
additional metadata and data layout suitable for loading in to XIOS. This is
intended to be run at the end of a workflow for generating ancillary files.
"""
import warnings
import iris.cube
import iris.exceptions
import numpy as np
import ugants
import ugants.abc
import ugants.utils
from ugants.utils.cube import as_cubelist
[docs]
class ConvertToXIOS(ugants.abc.Application):
"""Translate a UGrid cube to an XIOS compatible cube.
The pipeline is::
+------+ +-----+ +------+
| init | -> | run | -> | save |
+------+ +-----+ +------+
Where:
* :meth:`__init__` sets up the class with the source UGrid cubes to be
translated to XIOS.
* :meth:`run` copies the :attr:`sources` to :attr:`results` and sets some
metadata for XIOS, specifically:
1. If there is no ``online_operation`` attribute, use a default value of
``once``.
2. If there is no ``long_name`` for a variable, copy the ``standard_name``
to the ``long_name``.
3. Arrange data in CF order, such that the last dimensions are 'time',
'vertical levels', 'mesh' in that order, and all other dimensions
(including pseudo level) precede those dimensions.
* :meth:`save` writes the XIOS results to disk with all metadata and data
changes needed for XIOS. Additional changes beyond the :meth:`run`
metadata changes are:
1. Setting the ``_FillValue`` to a specific value that XIOS requires (XIOS
does not honour the ``_FillValue`` attribute).
2. Ensuring the ``online_operation`` is a local attribute.
3. Cast all float type data to single precision, if :attr:`cast_to_single`
is True.
"""
sources: iris.cube.CubeList
"""Cube(s) representing the original UGrid data."""
# Results and output need initial values to be included in the sphinx
# output and to be a valid sphinx cross reference target since these
# attributes are defined in the parent ABC:
results: iris.cube.CubeList = None
"""Cube(s) with the additional metadata needed for XIOS. Generated by the
:meth:`run` method."""
output: str = None
"""Path for writing the :attr:`results` with XIOS compatible metadata and data."""
cast_to_single: bool
"""Cast all double precision (64 bit) float data to single precision (32 bit)."""
[docs]
def __init__(self, sources: iris.cube.CubeList, cast_to_single: bool):
"""Instantiate class with UGrid sources that need to be translated to XIOS.
Parameters
----------
sources: iris.cube.CubeList
Cube(s) representing the original UGrid data. Will be stored in
:attr:`sources`.
cast_to_single: bool
Cast all double precision (64 bit) float data to single precision
(32 bit).
"""
sources = as_cubelist(sources)
self.sources = sources
self.cast_to_single = cast_to_single
self.results = None
self.output = None
[docs]
def run(self):
"""Apply most of the workarounds needed for XIOS.
Does not set the ``_FillValue`` attribute since this needs to be handled
at the save stage.
Sets the :attr:`results` to be a copy of the cubes from
:attr:`sources` with the workarounds needed for XIOS.
"""
self.results = self.sources.copy()
self.results = ugants.utils.cube.as_cubelist(self.results)
self.apply_long_name_workaround()
self.apply_online_operation_workaround()
self.apply_dimension_order_workaround()
[docs]
def apply_long_name_workaround(self):
"""Set a long name on the cube when the cube has a standard name.
XIOS only cares that there is a long name, not what the long name is.
If the cube has a standard name, copy that to the long name. If the
cube doesn't have a standard name, it will have a long name already.
See https://github.com/MetOffice/tcd-XIOS2-extras/discussions/32.
"""
for cube in self.results:
if cube.long_name is None and cube.standard_name is not None:
cube.long_name = cube.standard_name
[docs]
def apply_online_operation_workaround(self):
"""Set online operation if it's not already set.
The value of online operation is important. There are other valid
values, so if the user has manually set one on the cube before saving,
it should be preserved. Otherwise, set the online operation to
``once`` as a default option.
See https://github.com/MetOffice/tcd-XIOS2-extras/discussions/33.
"""
for cube in self.results:
if "online_operation" not in cube.attributes:
cube.attributes["online_operation"] = "once"
[docs]
def apply_dimension_order_workaround(self):
"""Transpose cube to conform to XIOS expected order.
This means we want the last dimension to be the mesh, the penultimate
dimension to be vertical levels (if present), the next last dimension
to be time (if present). All other dimensions (including pseudo
levels) need to be before time, vertical levels and the mesh. The
original order for the other dimensions is preserved.
"""
for cube in self.results:
reorder_cube_dimensions(cube)
[docs]
@staticmethod
def get_fill_values(cubes):
"""Return the fill values to be used when saving each cube to netCDF.
The fill value will be determined according to each cube's data type:
================ ==========
dtype fill value
================ ==========
double (float64) -1.7976931348623157e+308
single (float32) -(32768.0 * 32768.0)
other None
================ ==========
Warning
-------
For data types other than double (64 bit) or single (32 bit) precision float,
the returned fill value will be :obj:`None`. In practice, this means that the
netCDF default fill value will be used for that data type.
Parameters
----------
cubes: iris.cube.CubeList
Cubes to determine the fill values for.
Returns
-------
list
The fill value to be used when saving each cube to netCDF.
"""
fill_values_lookup = {
np.dtype("float32"): -np.float32(32768 * 32768),
np.dtype("float64"): np.nan_to_num(np.NINF),
}
fill_values = []
unrecognised = []
for cube in cubes:
fill_value = fill_values_lookup.get(cube.dtype, None)
fill_values.append(fill_value)
if fill_value is None:
unrecognised.append((cube.name(), cube.dtype.name))
if unrecognised:
dtypes_msg = ", ".join(
f"{name} has dtype {dtype}" for name, dtype in unrecognised
)
warnings.warn(
f"Fill value not known for the following variables: {dtypes_msg}. "
"Using default fill values for these dtypes.",
stacklevel=1,
)
return fill_values
[docs]
def save(self):
"""Save results to XIOS compatible UGrid netcdf.
Overrides parent class to use XIOS specific save arguments:
1. ``online_operation`` is defined as a local rather than global
attribute. See https://github.com/MetOffice/tcd-XIOS2-extras/discussions/33.
2. Fill value is set per-field to a value required by XIOS
(see :meth:`get_fill_values`).
See https://github.com/MetOffice/tcd-XIOS2-extras/discussions/31.
3. If :attr:`cast_to_single` is :obj:`True`, then all double (64 bit) float data
will be cast to single (32 bit) float before saving.
"""
if self.output is None:
raise ValueError("No output file location has been set.")
if self.results is None:
raise ValueError("The application has not yet been run, results is None.")
if self.cast_to_single:
results = cast_to_single_precision(self.results)
else:
results = self.results
fill_values = self.get_fill_values(results)
ugants.io.save.ugrid(
results,
self.output,
local_keys=["online_operation"],
fill_value=fill_values,
)
[docs]
def reorder_cube_dimensions(cube: iris.cube.Cube) -> iris.cube.Cube:
"""Reorder cube dimensions to be consistent with CF and XIOS.
The desired order is: other dimension(s), time, vertical, horizontal (mesh).
The original order of the other dimensions is preserved.
"Other dimensions" includes pseudo levels.
Note that this function modifies the cube in-place, in addition to returning the
modified cube.
Parameters
----------
cube: iris.cube.Cube
Cube with dimensions to be reordered
Returns
-------
iris.cube.Cube
Cube with reordered dimensions
Raises
------
ValueError
If the provided cube has more than one time dimension, or more than one
vertical dimension.
"""
# Move mesh to last dimension
ugants.utils.move_one_dimension(cube, cube.mesh_dim(), -1)
# Move time to penultimate dimension, if it exists. The model_level dimension, if it
# exists, will be inserted as the penultimate dimension, shifting the time
# dimension out by one.
time_coords = cube.coords(axis="t", dim_coords=True)
if len(time_coords) > 1:
time_names = ", ".join(coord.name() for coord in time_coords)
raise ValueError(f"Multiple time dimensions found: {time_names}.")
elif len(time_coords) == 1:
time_dimension = cube.coord_dims(time_coords[0])[0]
ugants.utils.move_one_dimension(cube, time_dimension, -2)
# Now shift vertical level to penultimate dimension, if it exists.
vertical_coords = cube.coords(axis="z", dim_coords=True)
if len(vertical_coords) > 1:
vertical_names = ", ".join(coord.name() for coord in vertical_coords)
raise ValueError(f"Multiple vertical dimensions found: {vertical_names}.")
elif len(vertical_coords) == 1:
vertical_dimension = cube.coord_dims(vertical_coords[0])[0]
ugants.utils.move_one_dimension(cube, vertical_dimension, -2)
return cube
[docs]
def cast_to_single_precision(cubes: iris.cube.CubeList):
"""Cast all double (float64) data to single precision (float32).
Integer data will not be cast.
Parameters
----------
cubes: iris.cube.CubeList
Cubes to be cast
Returns
-------
iris.cube.CubeList
A copy of the provided cubes with all float64 data cast to float32
"""
cubes = cubes.copy()
unrecognised = []
for cube in cubes:
if cube.dtype == np.dtype("float64"):
cube.data = cube.core_data().astype("float32")
else:
unrecognised.append((cube.name(), cube.dtype.name))
if unrecognised:
dtypes_msg = ", ".join(
f"{name} has dtype {dtype}" for name, dtype in unrecognised
)
warnings.warn(
"The following variables are not double precision, and will not be cast to "
f"single precision: {dtypes_msg}.",
stacklevel=1,
)
return cubes