# Copyright Iris contributors
#
# This file is part of Iris and is released under the BSD license.
# See LICENSE in the root of the repository for full licensing details.
"""Provides an interface to manage URI scheme support in iris.
.. z_reference:: iris.io
:tags: topic_load_save
API reference
"""
import collections
from collections import OrderedDict
import glob
import pathlib
import re
from urllib.parse import urlsplit
import iris.exceptions
def _is_nczarr_fragment(fragment):
"""Return True if a URI fragment requests NCZarr-compatible mode."""
if not fragment:
return False
# Fragments are ampersand-separated key=value pairs.
for item in fragment.split("&"):
key, sep, value = item.partition("=")
if sep and key.casefold() == "mode":
flags = {
token.strip().casefold() for token in value.split(",") if token.strip()
}
return bool(flags.intersection({"nczarr", "xarray"}))
return False
# URI schemes Iris hands to a store library rather than resolving as a local
# path. Iris depends on none of them: zarr-python raises its own ImportError
# naming what to install.
_REMOTE_STORE_SCHEMES = frozenset({"s3", "gs", "az", "abfs"})
def _is_remote_store_uri(spec):
"""Whether `spec` names a store Iris hands to a store library untouched.
Such a URI must not be resolved as a local path: `.absolute()` turns
``s3://bucket/data.zarr`` into ``/<cwd>/s3:/bucket/data.zarr``.
"""
if not isinstance(spec, str):
return False
scheme, _part, _fragment = decode_uri(spec)
return scheme in _REMOTE_STORE_SCHEMES
# Saving routines, indexed by file extension.
class _SaversDict(dict):
"""A dictionary that can only have string keys with no overlap.
:attr:`cubelist_savers` records which of the registered savers take a
``CubeList`` directly, rather than one cube at a time with
``append=True``. It holds the saver functions themselves, not extensions,
because :func:`save` is also reached with a saver it never looked up here
- passed by the caller, or chosen directly for an NCZarr target.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.cubelist_savers = set()
def __setitem__(self, key, value):
if not isinstance(key, str):
raise ValueError("key is not a string")
if key in self:
raise ValueError("A saver already exists for", key)
for k in self.keys():
if k.endswith(key) or key.endswith(k):
raise ValueError("key %s conflicts with existing key %s" % (key, k))
dict.__setitem__(self, key, value)
_savers = _SaversDict()
[docs]
def run_callback(callback, cube, field, filename):
"""Run the callback mechanism given the appropriate arguments.
Parameters
----------
callback :
A function to add metadata from the originating field and/or URI which
obeys the following rules:
1. Function signature must be: ``(cube, field, filename)``.
2. Modifies the given cube inplace, unless a new cube is
returned by the function.
3. If the cube is to be rejected the callback must raise
an :class:`iris.exceptions.IgnoreCubeException`.
Notes
-----
It is possible that this function returns None for certain callbacks,
the caller of this function should handle this case.
This function maintains laziness when called; it does not realise data.
See more at :doc:`/user_manual/explanation/real_and_lazy_data`.
"""
from iris.cube import Cube
if callback is None:
return cube
# Call the callback function on the cube, generally the function will
# operate on the cube in place, but it is also possible that the function
# will return a completely new cube instance.
try:
result = callback(cube, field, filename)
except iris.exceptions.IgnoreCubeException:
result = None
else:
if result is None:
result = cube
elif not isinstance(result, Cube):
raise TypeError("Callback function returned an unhandled data type.")
return result
[docs]
def decode_uri(uri, default="file"):
r"""Decode a single URI into scheme, scheme-specific part and fragment.
In addition to well-formed URIs, it also supports bare file paths as strings
or :class:`pathlib.PurePath`. Both Windows and UNIX style paths are
accepted.
It also supports 'bare objects', i.e. anything which is not a string.
These are identified with a scheme of 'data', and returned unchanged.
.. testsetup::
from iris.io import *
Examples
--------
>>> from iris.io import decode_uri
>>> print(decode_uri('https://www.thing.com:8080/resource?id=a:b'))
('https', '//www.thing.com:8080/resource?id=a:b', None)
>>> print(decode_uri('file:///data/local/dataZoo/...'))
('file', '///data/local/dataZoo/...', None)
>>> print(decode_uri('file:///data/local/dataZoo/something#mode=nczarr,file'))
('file', '///data/local/dataZoo/something', 'mode=nczarr,file')
>>> print(decode_uri('/data/local/dataZoo/...'))
('file', '/data/local/dataZoo/...', None)
>>> print(decode_uri('file:///C:\data\local\dataZoo\...'))
('file', '///C:\\data\\local\\dataZoo\\...', None)
>>> print(decode_uri('C:\data\local\dataZoo\...'))
('file', 'C:\\data\\local\\dataZoo\\...', None)
>>> print(decode_uri('dataZoo/...'))
('file', 'dataZoo/...', None)
>>> print(decode_uri({}))
('data', {}, None)
"""
if isinstance(uri, pathlib.PurePath):
uri = str(uri)
if isinstance(uri, str):
# make sure scheme has at least 2 letters to avoid windows drives
# put - last in the brackets so it refers to the character, not a range
# reference on valid schemes: https://tools.ietf.org/html/std66#section-3.1
match = re.match(r"^([a-zA-Z][a-zA-Z0-9+.-]+):(.+)", uri)
if match:
scheme = match.group(1)
part = match.group(2)
else:
# Catch bare UNIX and Windows paths
scheme = default
part = uri
fragment = urlsplit(uri).fragment or None
if fragment:
part = part[: -len(fragment) - 1] # remove the fragment from the part
else:
# We can pass things other than strings, like open files.
# These are simply identified as 'data objects'.
scheme = "data"
part = uri
fragment = None
return scheme, part, fragment
def _categorise_uri(uri) -> tuple[str, str, str]:
"""Return `uri` as ``(category, scheme, part)``, for grouping by handler.
The category is the scheme, except that an NCZarr fragment makes it
``"nczarr"``. netCDF-c's Zarr driver is asked for by ``#mode=nczarr,...``
rather than by scheme, so ``file:///s.zarr`` and
``file:///s.zarr#mode=nczarr,file`` need different handlers despite
sharing one scheme. The fragment is folded back into `part`, because a
handler is handed a URI to reopen rather than the decoded pieces.
A native Zarr store has no category of its own. Remotely it is known by
its scheme, and locally it is a directory that only a marker file
distinguishes from any other - see
:func:`iris.fileformats._is_zarr_store`. Deciding it here would put a
filesystem test inside a function that otherwise only reads a string.
"""
scheme, part, fragment = decode_uri(uri)
category = "nczarr" if _is_nczarr_fragment(fragment) else scheme
if fragment:
part = f"{part}#{fragment}"
return category, scheme, part
[docs]
def expand_filespecs(file_specs, files_expected=True):
"""Find all matching file paths from a list of file-specs.
Parameters
----------
file_specs : iterable of str
File paths which may contain ``~`` elements or wildcards.
files_expected : bool, default=True
Whether file is expected to exist (i.e. for load).
Returns
-------
list of str
If files_expected is ``True``:
A well-ordered list of matching absolute file paths.
If any of the file-specs match no existing files, an
exception is raised.
If files_expected is ``False``:
A list of expanded file paths.
"""
# Remove any hostname component - currently unused. A remote store URI is
# passed through: it is not a path, and resolving it mangles it.
filenames = [
fn
if _is_remote_store_uri(fn)
else str(pathlib.Path(fn.removeprefix("//")).expanduser().absolute())
for fn in file_specs
]
if files_expected:
# Try to expand all filenames as globs. A remote URI stands for
# itself; Iris does not list a remote store's neighbours.
glob_expanded = OrderedDict(
[
[fn, [fn] if _is_remote_store_uri(fn) else sorted(glob.glob(fn))]
for fn in filenames
]
)
# If any of the specs expanded to an empty list then raise an error
all_expanded = glob_expanded.values()
if not all(all_expanded):
msg = "One or more of the files specified did not exist:"
for pattern, expanded in glob_expanded.items():
if expanded:
msg += '\n - "{}" matched {} file(s)'.format(
pattern, len(expanded)
)
else:
msg += '\n * "{}" didn\'t match any files'.format(pattern)
raise IOError(msg)
result = [fname for fnames in all_expanded for fname in fnames]
else:
result = filenames
return result
[docs]
def load_files(filenames, callback, constraints=None):
"""Create a generator of Cubes from given files.
Take a list of filenames which may also be globs, and optionally a
constraint set and a callback function, and returns a
generator of Cubes from the given files.
Notes
-----
Typically, this function should not be called directly; instead, the
intended interface for loading is :func:`iris.load`.
"""
yield from _load_files(expand_filespecs(filenames), callback, constraints)
def _load_files(filenames, callback, constraints=None):
"""Create a generator of Cubes from files already expanded by `expand_filespecs`.
`filenames` is not expanded here: a glob matching the same file twice, by
two different patterns, must reach the format handler as two entries, and
:func:`expand_filespecs` keys its own glob cache by the spec string, so
expanding an already-expanded list collapses two identical paths into
one. :func:`load_files` is the public entry point that expands once and
delegates here; :func:`iris.loading._generate_cubes` is the other caller,
expanding once for itself so it can partition files from stores before
either sees the list.
"""
from iris.fileformats import FORMAT_AGENT
# Create default dict mapping iris format handler to its associated filenames
handler_map = collections.defaultdict(list)
for fn in filenames:
with open(fn, "rb") as fh:
handling_format_spec = FORMAT_AGENT.get_spec(pathlib.Path(fn).name, fh)
handler_map[handling_format_spec].append(fn)
# Call each iris format handler with the appropriate filenames
for handling_format_spec in sorted(handler_map):
fnames = handler_map[handling_format_spec]
if handling_format_spec.constraint_aware_handler:
for cube in handling_format_spec.handler(fnames, callback, constraints):
yield cube
else:
for cube in handling_format_spec.handler(fnames, callback):
yield cube
[docs]
def load_stores(targets, callback, constraints=None):
"""Create a generator of Cubes from targets that are stores, not files.
A store is a directory or a URL: something with no single seekable file
to open, so :func:`load_files` cannot reach it and the format agent is
asked without a buffer.
Parameters
----------
targets : iterable of str
Local directory paths or remote URLs, already expanded.
callback : callable
Passed to the format handler.
constraints : optional
Passed to a constraint-aware format handler.
Yields
------
iris.cube.Cube
Notes
-----
Typically, this function should not be called directly; instead, the
intended interface for loading is :func:`iris.load`.
"""
from iris.fileformats import FORMAT_AGENT
handler_map = collections.defaultdict(list)
for target in targets:
handling_format_spec = FORMAT_AGENT.get_spec(target, None)
handler_map[handling_format_spec].append(target)
for handling_format_spec in sorted(handler_map):
names = handler_map[handling_format_spec]
if handling_format_spec.constraint_aware_handler:
for cube in handling_format_spec.handler(names, callback, constraints):
yield cube
else:
for cube in handling_format_spec.handler(names, callback):
yield cube
[docs]
def load_http(urls, callback):
"""Create generator of Cubes from the given OPeNDAP or NcZarr URLs.
Take a list of OPeNDAP or NcZarr URLs and a callback function, and returns a generator
of Cubes from the given URLs.
Notes
-----
Typically, this function should not be called directly; instead, the
intended interface for loading is :func:`iris.load`.
"""
#
# NOTE: this routine is *also* called by "load_data_objects", in which case the
# 'urls' will actually be 'data objects'.
# In principle, however, their scopes are different, so it's just an implementation
# detail that right now the same code will do for both.
# If that changes sometime, the two routines may go their separate ways.
# Create default dict mapping iris format handler to its associated filenames
from iris.fileformats import FORMAT_AGENT
handler_map = collections.defaultdict(list)
for url in urls:
handling_format_spec = FORMAT_AGENT.get_spec(url, None)
handler_map[handling_format_spec].append(url)
# Call each iris format handler with the appropriate filenames
for handling_format_spec in sorted(handler_map):
fnames = handler_map[handling_format_spec]
for cube in handling_format_spec.handler(fnames, callback):
yield cube
[docs]
def load_data_objects(urls, callback):
"""Take a list of data-source objects and a callback function, returns a generator of Cubes.
The 'objects' take the place of 'uris' in the load calls.
The appropriate types of the data-source objects are expected to be
recognised by the handlers : This is done in the usual way by passing the
context to the format picker to get a handler for each.
.. note::
Typically, this function should not be called directly; instead, the
intended interface for loading is :func:`iris.load`.
"""
# NOTE: this operation is currently *identical* to the http one. But it seems
# sensible to provide a distinct handler function for this scheme.
yield from load_http(urls, callback)
def _dot_save(cube, target):
# A simple wrapper for `iris.fileformats.dot.save` which allows the
# saver to be registered without triggering the import of
# `iris.fileformats.dot`.
from iris.fileformats.dot import save
return save(cube, target)
def _dot_save_png(cube, target, **kwargs):
# A simple wrapper for `iris.fileformats.dot.save_png` which allows the
# saver to be registered without triggering the import of
# `iris.fileformats.dot`.
from iris.fileformats.dot import save_png
return save_png(cube, target, **kwargs)
def _zarr_save(cube, target, **kwargs):
# A simple wrapper for `iris.fileformats.zarr.save` which allows the
# saver to be registered without triggering the import of `zarr`, an
# optional dependency. Importing the name from the package rather than
# from `iris.fileformats.zarr.saver` runs the package's `__getattr__`,
# which is what checks the installed zarr version and reports a missing
# or too-old one.
from iris.fileformats.zarr import save
return save(cube, target, **kwargs)
def _grib_save(cube, target, append=False, **kwargs):
# A simple wrapper for the grib save routine, which allows the saver to be
# registered without having the grib implementation installed.
try:
from iris_grib import save_grib2
except ImportError:
raise RuntimeError(
'Unable to save GRIB file - "iris_grib" package is not installed.'
)
save_grib2(cube, target, append, **kwargs)
def _check_init_savers():
from iris.fileformats import netcdf, pp
if "pp" not in _savers:
_savers.update(
{
"pp": pp.save,
"nc": netcdf.save,
"zarr": _zarr_save,
"dot": _dot_save,
"dotpng": _dot_save_png,
"grib2": _grib_save,
}
)
# One line per format that takes a CubeList whole.
_savers.cubelist_savers.add(netcdf.save)
_savers.cubelist_savers.add(_zarr_save)
[docs]
def add_saver(file_extension, new_saver):
"""Add a custom saver to the Iris session.
Parameters
----------
file_extension : str
A string such as "pp" or "my_format".
new_saver : function
A function of the form ``my_saver(cube, target)``.
See Also
--------
iris.io.save :
Save one or more Cubes to file (or other writeable).
"""
# Make sure it's a func with 2+ args
if not hasattr(new_saver, "__call__") or new_saver.__code__.co_argcount < 2:
raise ValueError("Saver routines must be callable with 2+ arguments.")
# Try to add this saver. Invalid keys will be rejected.
_savers[file_extension] = new_saver
[docs]
def find_saver(filespec):
"""Find the saver function appropriate to the given filename or extension.
Parameters
----------
filespec : str
A string such as "my_file.pp" or "PP".
Returns
-------
Save function
Save functions can be passed to :func:`iris.io.save`. Value may also
be None.
"""
_check_init_savers()
# A store is a directory, so "out.zarr/" is a natural thing to type.
candidate = filespec.rstrip("/\\").lower()
matches = [
ext for ext in _savers if candidate.endswith("." + ext) or candidate == ext
]
# Multiple matches could occur if one of the savers included a '.':
# e.g. _savers = {'.dot.png': dot_png_saver, '.png': png_saver}
if len(matches) > 1:
fmt = "Multiple savers found for %r: %s"
matches = ", ".join(map(repr, matches))
raise ValueError(fmt % (filespec, matches))
return _savers[matches[0]] if matches else None
[docs]
def save(source, target, saver=None, **kwargs):
"""Save one or more Cubes to file (or other writeable).
Iris currently supports five file formats for saving. Four are
recognised by filename extension:
* **netCDF** - the Unidata network Common Data Format,
see :func:`iris.fileformats.netcdf.save`
* **GRIB2** - the WMO GRIdded Binary data format,
see :func:`iris_grib.save_grib2`.
* **PP** - the Met Office UM Post Processing Format,
see :func:`iris.fileformats.pp.save`
* **Zarr** - a native Zarr version 3 store, written through zarr-python,
see :func:`iris.fileformats.zarr.saver.save`. See
:doc:`/user_manual/explanation/zarr_io` for more.
The fifth format is recognised differently:
* **Zarr (via NcZarr)** - "The open foundation for chunked, compressed,
N-dimensional
arrays". Saved via the NcZarr utility of the NetCDF library, see
:func:`iris.fileformats.netcdf.save`. NcZarr recognises format via URL
fragments e.g. "file:///path/to/file#mode=nczarr,file". See the NcZarr
docs for more:
https://docs.unidata.ucar.edu/nug/current/nczarr_head.html.
A custom saver can be provided to the function to write to a different
file format.
Parameters
----------
source : :class:`iris.cube.Cube` or :class:`iris.cube.CubeList`
target : str or pathlib.PurePath or io.TextIOWrapper
When given a filename or file, Iris can determine the
file format.
saver : str or function, optional
Specifies the file format to save.
If omitted, Iris will attempt to determine the format.
If a string, this is the recognised filename extension
(where the actual filename may not have it).
Otherwise the value is a saver function, of the form:
``my_saver(cube, target)`` plus any custom keywords. It
is assumed that a saver will accept an ``append`` keyword
if its file format can handle multiple cubes. See also
:func:`iris.io.add_saver`.
**kwargs : dict, optional
All other keywords are passed through to the saver function; see the
relevant saver documentation for more information on keyword arguments.
Warnings
--------
Saving a cube whose data has been loaded lazily
(if `cube.has_lazy_data()` returns `True`) to the same file it expects
to load data from will cause both the data in-memory and the data on
disk to be lost.
.. code-block:: python
cube = iris.load_cube("somefile.nc")
# The next line causes data loss in 'somefile.nc' and the cube.
iris.save(cube, "somefile.nc")
In general, overwriting a file which is the source for any lazily loaded
data can result in corruption. Users should proceed with caution when
attempting to overwrite an existing file.
Notes
-----
This function maintains laziness when called; it does not realise data.
See more at :doc:`/user_manual/explanation/real_and_lazy_data`.
Examples
--------
>>> # Setting up
>>> import iris
>>> my_cube = iris.load_cube(iris.sample_data_path('air_temp.pp'))
>>> my_cube_list = iris.load(iris.sample_data_path('space_weather.nc'))
>>> # Save a cube to PP
>>> iris.save(my_cube, "myfile.pp")
>>> # Save a cube list to a PP file, appending to the contents of the file
>>> # if it already exists
>>> iris.save(my_cube_list, "myfile.pp", append=True)
>>> # Save a cube to netCDF, defaults to NETCDF4 file format
>>> iris.save(my_cube, "myfile.nc")
>>> # Save a cube list to netCDF, using the NETCDF3_CLASSIC storage option
>>> iris.save(my_cube_list, "myfile.nc", netcdf_format="NETCDF3_CLASSIC")
"""
from iris.cube import Cube, CubeList
# Determine format from filename
if isinstance(target, pathlib.PurePath):
target = str(target)
if isinstance(target, str) and saver is None:
if _is_nczarr_fragment(urlsplit(target).fragment or None):
# NCZarr URLs must be passed as-is; do not expand or match by extension.
from iris.fileformats.netcdf.saver import save as _nc_save
saver = _nc_save
else:
# Converts tilde or wildcards to absolute path
(target,) = expand_filespecs([str(target)], False)
saver = find_saver(target)
elif hasattr(target, "name") and saver is None:
saver = find_saver(target.name)
elif isinstance(saver, str):
saver = find_saver(saver)
if saver is None:
raise ValueError("Cannot save; no saver")
# Single cube?
if isinstance(source, Cube):
result = saver(source, target, **kwargs)
# CubeList or sequence of cubes?
elif isinstance(source, CubeList) or (
isinstance(source, (list, tuple)) and all([isinstance(i, Cube) for i in source])
):
# Only allow cubelist saving for those fileformats that are capable.
_check_init_savers()
if saver not in _savers.cubelist_savers:
# Make sure the saver accepts an append keyword
if "append" not in saver.__code__.co_varnames:
raise ValueError(
"Cannot append cubes using saver function "
"'%s' in '%s'"
% (saver.__code__.co_name, saver.__code__.co_filename)
)
# Force append=True for the tail cubes. Don't modify the incoming
# kwargs.
kwargs = kwargs.copy()
for i, cube in enumerate(source):
if i != 0:
kwargs["append"] = True
saver(cube, target, **kwargs)
result = None
# Netcdf saver.
else:
result = saver(source, target, **kwargs)
else:
raise ValueError("Cannot save; non Cube found in source")
return result