Source code for iris.fileformats.netcdf.ugrid_load

# Copyright Iris contributors
#
# This file is part of Iris and is released under the BSD license.
# See LICENSE in the root of the repository for full licensing details.

r"""Allow the construction of :class:`~iris.mesh.MeshXY`.

.. z_reference:: iris.fileformats.netcdf.ugrid_load
   :tags: topic_load_save;topic_mesh

   API reference

Extension functions for Iris NetCDF loading, to construct
:class:`~iris.mesh.MeshXY` from UGRID data in files.

.. seealso::

    The UGRID Conventions,
    https://ugrid-conventions.github.io/ugrid-conventions/

"""

from contextlib import ExitStack
from dataclasses import dataclass
from functools import partial
from itertools import groupby
from pathlib import Path
from typing import Iterable
import warnings

from iris.common.mixin import CFVariableMixin
from iris.config import get_logger
from iris.coords import AuxCoord
from iris.io import (
    _REMOTE_STORE_SCHEMES,
    _categorise_uri,
    expand_filespecs,
)
from iris.loading import LoadProblems
from iris.mesh.components import Connectivity, MeshXY
from iris.util import guess_coord_axis
from iris.warnings import IrisCfWarning, IrisDefaultingWarning, IrisIgnoringWarning

# NOTE: all imports from iris.fileformats.netcdf must be deferred, to avoid circular
# imports.
# This is needed so that load_mesh/load_meshes can be included in the iris.mesh API.


# Configure the logger.
logger = get_logger(__name__, propagate=True, handler=False)


class _WarnComboCfDefaulting(IrisCfWarning, IrisDefaultingWarning):
    """One-off combination of warning classes - enhances user filtering."""

    pass


class _WarnComboCfDefaultingIgnoring(_WarnComboCfDefaulting, IrisIgnoringWarning):
    """One-off combination of warning classes - enhances user filtering."""

    pass


def _meshes_from_cf(cf_reader):
    """Mesh from cf, common behaviour for extracting meshes from a CFReader.

    Simple now, but expected to increase in complexity as Mesh sharing develops.

    """
    # Mesh instances are shared between file phenomena.
    # TODO: more sophisticated Mesh sharing between files.
    # TODO: access external Mesh cache?
    from iris.fileformats._nc_load_rules.helpers import _add_or_capture

    meshes = {}

    if cf_reader._with_ugrid:
        mesh_vars = cf_reader.cf_group.meshes
        for name, var in mesh_vars.items():
            _ = _add_or_capture(
                build_func=partial(_build_mesh, cf_reader, var),
                add_method=partial(meshes.__setitem__, name),
                cf_var=var,
                # Meshes are 'linked' to zero or more Cubes; they do not have
                #  a destination in the same way as other Cube components.
                destination=LoadProblems.Problem.Destination(
                    iris_class=CFVariableMixin,
                    identifier="NOT_APPLICABLE",
                ),
            )

    return meshes


@dataclass
class _PendingMeshCoords:
    """MeshCoords built ahead of their cube, waiting to be attached to it."""

    coords: list
    dim: int | None
    capture_kwargs: dict


def _prepare_mesh_coords(cf_reader, meshes, cf_var):
    """Build the MeshCoords a data variable needs, before its cube exists.

    Unconventional 'split' usage of ``_add_or_capture``: this half builds and
    captures, :func:`_attach_mesh_coords` adds. The split is forced by
    attribute handling - built after the cube,
    ``helpers._add_unused_attributes`` has already taken the variable's
    ``mesh`` and ``location`` attributes for cube attributes.

    Parameters
    ----------
    cf_reader : iris.fileformats.cf.CFReader
        The reader the mesh variables were found in.
    meshes : dict
        Meshes by name, as returned by :func:`_meshes_from_cf`.
    cf_var : iris.fileformats.cf.CFDataVariable
        The data variable about to be built into a cube.

    Returns
    -------
    _PendingMeshCoords or None
        None when `cf_var` names no mesh, or names one this dataset does not
        hold.

    """
    # NOTE: dynamic imports avoid circularity : see note with module imports
    from iris.cube import Cube
    from iris.fileformats._nc_load_rules.helpers import _add_or_capture

    # Not this module's `logger`, which sits at INFO and would drop a debug
    #  message at source. `TestsMesh.test_missing_mesh` asserts the package
    #  logger, where this message has always been raised.
    from iris.fileformats.netcdf import logger as netcdf_logger

    mesh_name = cf_var.attributes.get("mesh")
    if mesh_name is None:
        return None

    mesh = meshes.get(mesh_name)
    if mesh is None:
        message = (
            f"Mesh '{mesh_name}' - "
            f"referenced by variable: '{cf_var.cf_name}' - "
            "could not be found in file."
        )
        netcdf_logger.debug(message)
        return None

    pending = _PendingMeshCoords(
        coords=[],
        dim=None,
        capture_kwargs=dict(
            cf_var=cf_reader.cf_group.meshes[mesh_name],
            # MeshCoords are an Iris concept; the best fallback we have is to
            #  capture the CF Mesh.
            destination=LoadProblems.Problem.Destination(
                iris_class=Cube,
                identifier=cf_var.cf_name,
            ),
        ),
    )

    def _stash(coords_and_dim):
        pending.coords, pending.dim = coords_and_dim

    _ = _add_or_capture(
        build_func=partial(_build_mesh_coords, mesh, cf_var),
        add_method=partial(_stash),
        **pending.capture_kwargs,
    )
    return pending


def _attach_mesh_coords(pending, cube):
    """Attach MeshCoords prepared by :func:`_prepare_mesh_coords` to their cube.

    Parameters
    ----------
    pending : _PendingMeshCoords
        The return of :func:`_prepare_mesh_coords`. Its `coords` are empty if
        the build failed, in which case the failure is already recorded and
        this call adds nothing.
    cube : iris.cube.Cube
        The cube built from the data variable the MeshCoords were prepared
        for.

    """
    # NOTE: dynamic imports avoid circularity : see note with module imports
    from iris.fileformats._nc_load_rules.helpers import _add_or_capture

    def _add(coords_and_dim):
        coords, dim = coords_and_dim
        for coord in coords:
            cube.add_aux_coord(coord, dim)

    _ = _add_or_capture(
        build_func=partial(lambda: (pending.coords, pending.dim)),
        add_method=partial(_add),
        **pending.capture_kwargs,
    )


[docs] def load_mesh(uris, var_name=None): """Load a single :class:`~iris.mesh.MeshXY` from one or more files or stores. Raises an error if more/less than one :class:`~iris.mesh.MeshXY` is found. Parameters ---------- uris : str or iterable of str One or more filenames/URI's. Filenames can include wildcards. NetCDF files and Zarr stores are read; anything else is ignored. Any URI's must support OpenDAP, NcZarr, or a Zarr remote store scheme. var_name : str, optional Only return a :class:`~iris.mesh.MeshXY` if its var_name matches this value. Returns ------- :class:`iris.mesh.MeshXY` """ meshes_result = load_meshes(uris, var_name) result = set([mesh for file in meshes_result.values() for mesh in file]) mesh_count = len(result) if mesh_count != 1: message = f"Expecting 1 mesh, but input file(s) produced: {mesh_count} ." raise ValueError(message) return result.pop() # Return the single element
[docs] def load_meshes(uris, var_name=None): r"""Load :class:`~iris.mesh.MeshXY` objects from files or Zarr stores. Parameters ---------- uris : str or iterable of str One or more filenames/URI's. Filenames can include wildcards. NetCDF files and Zarr stores are read; anything else is ignored. Any URI's must support OpenDAP, NcZarr, or a Zarr remote store scheme. var_name : str, optional Only return :class:`~iris.mesh.MeshXY` that have var_names matching this value. Returns ------- dict A dictionary mapping each mesh-containing file path/URL in the input ``uris`` to a list of the :class:`~iris.mesh.MeshXY` returned from each. """ # NOTE: no constraints or callbacks supported - these assume they are operating # on a Cube. # NOTE: dynamic imports avoid circularity : see note with module imports from iris.fileformats import FORMAT_AGENT, _load_zarr from iris.fileformats.cf import CFReader import iris.fileformats.netcdf.loader as nc_loader if ( isinstance(uris, str) or hasattr(uris, "fromcdl") or not isinstance(uris, Iterable) ): # Make a string, Dataset, or other single item, into an iterable. uris = [uris] # Group collections of uris by their iris handler # Create list of tuples relating schemes to part names. uri_tuples = sorted(_categorise_uri(uri) for uri in uris) valid_sources = [] for category, groups in groupby(uri_tuples, key=lambda x: x[0]): # Call each scheme handler with the appropriate URIs if category == "file": filenames = [part for _cat, _scheme, part in groups] sources = expand_filespecs(filenames) elif category in {"http", "https", "nczarr"} | _REMOTE_STORE_SCHEMES: sources = [f"{scheme}:{part}" for _cat, scheme, part in groups] else: message = f"Iris cannot handle the URI scheme: {category}" raise ValueError(message) for source in sources: # iris.loading._generate_cubes makes this same file-or-store # distinction independently, for general cube loading - keep # both in step. if category == "file" and not Path(source).is_dir(): with open(source, "rb") as fh: handling_format_spec = FORMAT_AGENT.get_spec(Path(source).name, fh) else: handling_format_spec = FORMAT_AGENT.get_spec(source, None) # TODO: drop this format dispatch once the netCDF and Zarr # cube-building loops are unified - see # https://github.com/SciTools/iris/issues/7353. This copy, not # the mesh code below, is the only reason a netCDF-named module # names Zarr at all. # # `_load_zarr` is compared, never `zarr.load_cubes`: a store's # presence is what should reach zarr-python, not this comparison. if handling_format_spec.handler == nc_loader.load_cubes: valid_sources.append((source, False)) elif handling_format_spec.handler == _load_zarr: valid_sources.append((source, True)) else: message = f"Ignoring file of a format that holds no mesh: {source}" logger.info(msg=message, extra=dict(cls=None)) result = {} for source, is_zarr in valid_sources: with ExitStack() as stack: if is_zarr: # Deferred: a caller that named no Zarr store must not reach # zarr-python at all. Through the package gate, which adds # the version check and the missing-dependency message. from iris.fileformats.zarr import _gated_import ZarrDataset = _gated_import("._dataset").ZarrDataset # CFReader takes an open CFDataset or a filename, so the mesh # machinery below never learns which it was given. target = stack.enter_context(ZarrDataset(source)) else: target = source cf_reader = stack.enter_context(CFReader(target)) meshes_dict = _meshes_from_cf(cf_reader) meshes = list(meshes_dict.values()) if var_name is not None: meshes = list(filter(lambda m: m.var_name == var_name, meshes)) if meshes: result[source] = meshes return result
def _build_aux_coord(coord_var): """Construct a :class:`~iris.coords.AuxCoord`. Construct a :class:`~iris.coords.AuxCoord` from a given :class:`~iris.fileformats.cf.CFUGridAuxiliaryCoordinateVariable`, and guess its mesh axis. """ # NOTE: dynamic imports avoid circularity : see note with module imports from iris.fileformats._nc_load_rules.helpers import get_attr_units, get_names from iris.fileformats.cf import CFUGridAuxiliaryCoordinateVariable from iris.fileformats.cf import loader as cf_loader assert isinstance(coord_var, CFUGridAuxiliaryCoordinateVariable) attributes = {} attr_units = get_attr_units(coord_var, attributes) points_data = cf_loader._get_cf_var_data(coord_var) # Bounds will not be loaded: # Bounds may be present, but the UGRID conventions state this would # always be duplication of the same info provided by the mandatory # connectivities. # Fetch climatological - not allowed for a Mesh, but loading it will # mean an informative error gets raised. climatological = False # TODO: use CF_ATTR_CLIMATOLOGY on re-integration, when no longer # 'experimental'. attr_climatology = coord_var.attributes.get("climatology") if attr_climatology is not None: climatology_vars = coord_var.cf_group.climatology climatological = attr_climatology in climatology_vars standard_name, long_name, var_name = get_names(coord_var, None, attributes) coord = AuxCoord( points_data, standard_name=standard_name, long_name=long_name, var_name=var_name, units=attr_units, attributes=attributes, # TODO: coord_system climatological=climatological, ) axis = guess_coord_axis(coord) if axis is None: if var_name[-2] == "_": # Fall back on UGRID var_name convention. axis = var_name[-1] else: message = f"Cannot guess axis for UGRID coord: {var_name} ." raise ValueError(message) return coord, axis def _build_connectivity(connectivity_var, element_dims): """Construct a :class:`~iris.mesh.Connectivity`. Construct a :class:`~iris.mesh.Connectivity` from a given :class:`~iris.fileformats.cf.CFUGridConnectivityVariable`, and identify the name of its first dimension. """ # NOTE: dynamic imports avoid circularity : see note with module imports from iris.fileformats._nc_load_rules.helpers import get_attr_units, get_names from iris.fileformats.cf import CFUGridConnectivityVariable from iris.fileformats.cf import loader as cf_loader assert isinstance(connectivity_var, CFUGridConnectivityVariable) attributes = {} attr_units = get_attr_units(connectivity_var, attributes) indices_data = cf_loader._get_cf_var_data(connectivity_var) # CFUGridConnectivityVariable.identify never checks the variable's own # cf_role, so a file can omit it. See SciTools/iris#7332. if "cf_role" not in connectivity_var.attributes: message = ( "UGRID connectivity variable has no cf_role attribute: " f"{connectivity_var.cf_name}" ) raise ValueError(message) cf_role = connectivity_var.attributes["cf_role"] # UGRID makes start_index optional, defaulting to 0. start_index = connectivity_var.attributes.get("start_index", 0) dim_names = connectivity_var.dimensions # Connectivity arrays must have two dimensions. assert len(dim_names) == 2 if dim_names[1] in element_dims: location_axis = 1 else: location_axis = 0 standard_name, long_name, var_name = get_names(connectivity_var, None, attributes) connectivity = Connectivity( indices=indices_data, cf_role=cf_role, standard_name=standard_name, long_name=long_name, var_name=var_name, units=attr_units, attributes=attributes, start_index=start_index, location_axis=location_axis, ) return connectivity, dim_names[0] def _build_mesh(cf, mesh_var): """Construct a :class:`~iris.mesh.MeshXY`. Construct a :class:`~iris.mesh.MeshXY` from a given :class:`~iris.fileformats.cf.CFUGridMeshVariable`. """ # NOTE: dynamic imports avoid circularity : see note with module imports from iris.fileformats._nc_load_rules.helpers import get_attr_units, get_names from iris.fileformats.cf import CFUGridMeshVariable from iris.fileformats.cf import loader as cf_loader assert isinstance(mesh_var, CFUGridMeshVariable) attributes = {} attr_units = get_attr_units(mesh_var, attributes) cf_role_message = None if "cf_role" not in mesh_var.attributes: cf_role_message = f"{mesh_var.cf_name} has no cf_role attribute." cf_role = "mesh_topology" else: cf_role = mesh_var.attributes["cf_role"] if cf_role != "mesh_topology": cf_role_message = f"{mesh_var.cf_name} has an inappropriate cf_role: {cf_role}." if cf_role_message: cf_role_message += " Correcting to 'mesh_topology'." warnings.warn( cf_role_message, category=_WarnComboCfDefaulting, ) if "volume_node_connectivity" in mesh_var.attributes: topology_dimension = 3 elif "face_node_connectivity" in mesh_var.attributes: topology_dimension = 2 elif "edge_node_connectivity" in mesh_var.attributes: topology_dimension = 1 else: # Nodes only. We aren't sure yet whether this is a valid option. topology_dimension = 0 if "topology_dimension" not in mesh_var.attributes: msg = ( f"MeshXY variable {mesh_var.cf_name} has no 'topology_dimension'" f" : *Assuming* topology_dimension={topology_dimension}" ", consistent with the attached connectivities." ) warnings.warn(msg, category=_WarnComboCfDefaulting) else: quoted_topology_dimension = mesh_var.attributes["topology_dimension"] if quoted_topology_dimension != topology_dimension: msg = ( f"*Assuming* 'topology_dimension'={topology_dimension}" f", from the attached connectivities of the mesh variable " f"{mesh_var.cf_name}. However, " f"{mesh_var.cf_name}:topology_dimension = " f"{quoted_topology_dimension}" " -- ignoring this as it is inconsistent." ) warnings.warn( msg, category=_WarnComboCfDefaultingIgnoring, ) node_dimension = None edge_dimension = mesh_var.attributes.get("edge_dimension") face_dimension = mesh_var.attributes.get("face_dimension") node_coord_args = [] edge_coord_args = [] face_coord_args = [] for coord_var in mesh_var.cf_group.ugrid_coords.values(): coord_and_axis = _build_aux_coord(coord_var) coord = coord_and_axis[0] if coord.var_name in mesh_var.attributes["node_coordinates"].split(): node_coord_args.append(coord_and_axis) node_dimension = coord_var.dimensions[0] elif coord.var_name in mesh_var.attributes.get("edge_coordinates", "").split(): edge_coord_args.append(coord_and_axis) elif coord.var_name in mesh_var.attributes.get("face_coordinates", "").split(): face_coord_args.append(coord_and_axis) # TODO: support volume_coordinates. else: message = ( f"Invalid UGRID coord: {coord.var_name} . Must be either a" f"node_, edge_ or face_coordinate." ) raise ValueError(message) if node_dimension is None: message = "'node_dimension' could not be identified from mesh node coordinates." raise ValueError(message) # Used for detecting transposed connectivities. element_dims = (edge_dimension, face_dimension) connectivity_args = [] for connectivity_var in mesh_var.cf_group.connectivities.values(): connectivity, first_dim_name = _build_connectivity( connectivity_var, element_dims ) assert connectivity.var_name == mesh_var.attributes[connectivity.cf_role] connectivity_args.append(connectivity) # If the mesh_var has not supplied the dimension name, it is safe to # fall back on the connectivity's first dimension's name. if edge_dimension is None and connectivity.location == "edge": edge_dimension = first_dim_name if face_dimension is None and connectivity.location == "face": face_dimension = first_dim_name standard_name, long_name, var_name = get_names(mesh_var, None, attributes) mesh = MeshXY( topology_dimension=topology_dimension, node_coords_and_axes=node_coord_args, connectivities=connectivity_args, edge_coords_and_axes=edge_coord_args, face_coords_and_axes=face_coord_args, standard_name=standard_name, long_name=long_name, var_name=var_name, units=attr_units, attributes=attributes, node_dimension=node_dimension, edge_dimension=edge_dimension, face_dimension=face_dimension, ) mesh_elements = list(mesh.all_coords) + list(mesh.all_connectivities) + [mesh] mesh_elements = filter(None, mesh_elements) for iris_object in mesh_elements: cf_loader._add_unused_attributes(iris_object, cf.cf_group[iris_object.var_name]) return mesh def _build_mesh_coords(mesh, cf_var): """Construct a tuple of :class:`~iris.mesh.MeshCoord`. Construct a tuple of :class:`~iris.mesh.MeshCoord` using from a given :class:`~iris.mesh.MeshXY` and :class:`~iris.fileformats.cf.CFVariable`. """ # Identify the cube's mesh dimension, for attaching MeshCoords. element_dimensions = { "node": mesh.node_dimension, "edge": mesh.edge_dimension, "face": mesh.face_dimension, } location = cf_var.attributes.get("location", "<empty>") if location is None or location not in element_dimensions: # We should probably issue warnings and recover, but that is too much # work. Raising a more intelligible error is easy to do though. msg = ( f"mesh data variable {cf_var.cf_name!r} has an invalid " f"location={location!r}." ) raise ValueError(msg) mesh_dim_name = element_dimensions.get(location) if mesh_dim_name is None: msg = f"mesh {mesh.name!r} has no {location} dimension." raise ValueError(msg) if mesh_dim_name in cf_var.dimensions: mesh_dim = cf_var.dimensions.index(mesh_dim_name) else: msg = ( f"mesh data variable {cf_var.cf_name!r} does not have the " f"{location} mesh dimension {mesh_dim_name!r}, in its dimensions." ) raise ValueError(msg) mesh_coords = mesh.to_MeshCoords(location=location) return mesh_coords, mesh_dim