Merge pull request #411 from ecmwf/develop

Release 0.9.0

Author: sandorkertesz

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
     with:
       earthkit-data: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
-      python_qa: true
+      python_qa: false
     secrets: inherit
 
 

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.6.0
   hooks:
   - id: trailing-whitespace
   - id: end-of-file-fixer
@@ -10,14 +10,21 @@ repos:
   # - id: check-added-large-files
   - id: debug-statements
   - id: mixed-line-ending
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-merge-conflict # Check for files that contain merge conflict
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.0
   hooks:
   - id: isort
+    args:
+    - -l 110
+    - --force-single-line-imports
+    - --profile black
 - repo: https://github.com/psf/black
   rev: 24.4.2
   hooks:
   - id: black
+    args: [--line-length=110]
 - repo: https://github.com/keewis/blackdoc
   rev: v0.3.8
   hooks:
@@ -27,6 +34,16 @@ repos:
   rev: 6.1.0
   hooks:
   - id: flake8
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.4.6
+  hooks:
+  - id: ruff
+    exclude: '(dev/.*|.*_)\.py$'
+    args:
+    - --line-length=110
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview
 - repo: https://github.com/executablebooks/mdformat
   rev: 0.7.14
   hooks:
@@ -37,11 +54,25 @@ repos:
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --preserve-quotes]
-  - id: pretty-format-toml
-    args: [--autofix]
+  # - id: pretty-format-toml
+  #   args: [--autofix]
+# - repo: https://github.com/b8raoult/pre-commit-docconvert
+#   rev: "0.1.4"
+#   hooks:
+#   - id: docconvert
+#     args: ["numpy"]
 - repo: https://github.com/PyCQA/pydocstyle.git
   rev: 6.1.1
   hooks:
   - id: pydocstyle
     additional_dependencies: [toml]
     exclude: tests|docs
+- repo: https://github.com/b8raoult/optional-dependencies-all
+  rev: "0.0.2"
+  hooks:
+  - id: optional-dependencies-all
+    args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs,test"]
+- repo: https://github.com/tox-dev/pyproject-fmt
+  rev: "2.1.3"
+  hooks:
+  - id: pyproject-fmt

docs/guide/include/settings-set.py

@@ -8,9 +8,7 @@
 
 # Multiple values can be set together. The argument list
 # can be a dictionary:
-earthkit.data.settings.set(
-    {"number-of-download-threads": 7, "url-download-timeout": "1m"}
-)
+earthkit.data.settings.set({"number-of-download-threads": 7, "url-download-timeout": "1m"})
 
 # Alternatively, we can use keyword arguments. However, because
 # the “-” character is not allowed in variable names in Python we have

docs/release_notes/index.rst

@@ -4,6 +4,7 @@ Release notes
 .. toctree::
     :maxdepth: 1
 
+    version_0.9_updates
     version_0.8_updates
     version_0.7_updates
     version_0.6_updates

docs/release_notes/version_0.9_updates.rst

@@ -0,0 +1,18 @@
+Version 0.9 Updates
+/////////////////////////
+
+Version 0.9.1
+===============
+
+New features
+++++++++++++++++
+
+- Renamed the ``constants`` source to ``forcings``
+- Enabled the NetCDF field parser to recognise the following additional coordinates:  "X", "xc", "Y" and "yc"
+- Split encoding from writing in experimental GRIB output generator
+
+Fixes
+++++++
+
+- Fixed issue when NetCDF data containing cftime dates could not be read
+- Fixed issue when NetCDF data containing coordinates called "lat" and "lon" could not be read correctly

pyproject.toml

@@ -1,27 +1,41 @@
 [build-system]
-requires = ["setuptools>=61", "setuptools-scm>=8.0"]
+requires = [
+  "setuptools>=61",
+  "setuptools-scm>=8",
+]
 
 [project]
+name = "earthkit-data"
+description = "A format-agnostic Python interface for geospatial data"
+readme = "README.md"
+license = { text = "Apache License Version 2.0" }
 authors = [
-  {name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int"}
+  { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" },
 ]
+requires-python = ">=3.8"
 classifiers = [
   "Development Status :: 2 - Pre-Alpha",
   "Intended Audience :: Developers",
   "License :: OSI Approved :: Apache Software License",
-  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
-  "Operating System :: OS Independent"
+]
+dynamic = [
+  "version",
 ]
 dependencies = [
   "cfgrib>=0.9.10.1",
-  "eccodes>=1.7.0",
   "dask",
+  "earthkit-geo>=0.2",
+  "earthkit-meteo>=0.0.1",
+  "eccodes>=1.7",
   "entrypoints",
   "filelock",
   "jinja2",
@@ -30,72 +44,112 @@ dependencies = [
   "multiurl",
   "netcdf4",
   "pandas",
-  "pdbufr>=0.11.0",
+  "pdbufr>=0.11",
   "pyyaml",
-  "tqdm>=4.63.0",
-  "xarray>=0.19.0",
-  "earthkit-geo>=0.2.0",
-  "earthkit-meteo>=0.0.1"
+  "tqdm>=4.63",
+  "xarray>=0.19",
 ]
-description = "A format-agnostic Python interface for geospatial data"
-dynamic = ["version"]
-license = {text = "Apache License Version 2.0"}
-name = "earthkit-data"
-readme = "README.md"
-requires-python = ">= 3.8"
-
-[project.optional-dependencies]
-all = [
-  "earthkit-data[mars]",
-  "earthkit-data[cds]",
-  "earthkit-data[ecmwf-opendata]",
-  "earthkit-data[fdb]",
-  "earthkit-data[polytope]",
-  "earthkit-data[wekeo]",
-  "earthkit-data[odb]",
-  "earthkit-data[projection]",
-  "earthkit-data[geopandas]",
-  "earthkit-data[eccovjson]"
-]
-cds = ["cdsapi"]
-eccovjson = ["eccovjson>=0.0.5"]
-ecmwf-opendata = ["ecmwf-opendata>=0.3.3"]
-fdb = ["pyfdb"]
-geopandas = ["geopandas"]
-mars = ["ecmwf-api-client>=1.6.1"]
-odb = ["pyodc"]
-polytope = ["polytope-client>=0.7.4"]
-projection = ["cartopy"]
-test = [
-  "earthkit-data[all]",
+optional-dependencies.all = [
+  "cartopy",
+  "cdsapi",
+  "cfgrib>=0.9.10.1",
+  "dask",
+  "earthkit-geo>=0.2",
+  "earthkit-meteo>=0.0.1",
+  "eccodes>=1.7",
+  "eccovjson>=0.0.5",
+  "ecmwf-api-client>=1.6.1",
+  "ecmwf-opendata>=0.3.3",
+  "entrypoints",
+  "filelock",
+  "geopandas",
+  "hda",
+  "jinja2",
+  "jsonschema",
+  "markdown",
+  "multiurl",
+  "nbconvert",
+  "nbformat",
+  "netcdf4",
+  "pandas",
+  "pdbufr>=0.11",
+  "polytope-client>=0.7.4",
+  "pyfdb",
+  "pyodc",
   "pytest",
   "pytest-cov",
   "pytest-forked",
   "pytest-timeout",
+  "pyyaml",
+  "tqdm>=4.63",
+  "xarray>=0.19",
+]
+optional-dependencies.cds = [
+  "cdsapi",
+]
+optional-dependencies.eccovjson = [
+  "eccovjson>=0.0.5",
+]
+optional-dependencies.ecmwf-opendata = [
+  "ecmwf-opendata>=0.3.3",
+]
+optional-dependencies.fdb = [
+  "pyfdb",
+]
+optional-dependencies.geopandas = [
+  "geopandas",
+]
+optional-dependencies.mars = [
+  "ecmwf-api-client>=1.6.1",
+]
+optional-dependencies.odb = [
+  "pyodc",
+]
+optional-dependencies.polytope = [
+  "polytope-client>=0.7.4",
+]
+optional-dependencies.projection = [
+  "cartopy",
+]
+optional-dependencies.test = [
+  "nbconvert",
   "nbformat",
-  "nbconvert"
+  "pytest",
+  "pytest-cov",
+  "pytest-forked",
+  "pytest-timeout",
+]
+optional-dependencies.wekeo = [
+  "hda",
 ]
-wekeo = ["hda"]
+urls.Documentation = "https://earthkit-data.readthedocs.io/"
+urls.Homepage = "https://github.com/ecmwf/earthkit-data/"
+urls.Issues = "https://github.com/ecmwf/earthkit-data.issues"
+urls.Repository = "https://github.com/ecmwf/earthkit-data/"
 
-[project.urls]
-Documentation = "https://earthkit-data.readthedocs.io/"
-Homepage = "https://github.com/ecmwf/earthkit-data/"
-Issues = "https://github.com/ecmwf/earthkit-data.issues"
-Repository = "https://github.com/ecmwf/earthkit-data/"
+[tool.setuptools.packages.find]
+include = [
+  "earthkit.data",
+]
+where = [
+  "src/",
+]
 
-[tool.coverage.run]
-branch = "true"
+[tool.setuptools_scm]
+version_file = "src/earthkit/data/_version.py"
 
 [tool.isort]
 profile = "black"
 
+[tool.coverage.run]
+branch = "true"
+
 [tool.pydocstyle]
-add_ignore = ["D1", "D200", "D205", "D400", "D401"]
+add_ignore = [
+  "D1",
+  "D200",
+  "D205",
+  "D400",
+  "D401",
+]
 convention = "numpy"
-
-[tool.setuptools.packages.find]
-include = ["earthkit.data"]
-where = ["src/"]
-
-[tool.setuptools_scm]
-version_file = "src/earthkit/data/_version.py"

src/earthkit/data/__init__.py

@@ -23,8 +23,11 @@
 from .core.fieldlist import FieldList
 from .core.settings import SETTINGS as settings
 from .readers.grib.output import new_grib_output
-from .sources import Source, from_source, from_source_lazily
-from .utils.examples import download_example_file, remote_example_file
+from .sources import Source
+from .sources import from_source
+from .sources import from_source_lazily
+from .utils.examples import download_example_file
+from .utils.examples import remote_example_file
 
 __all__ = [
     "ALL",

src/earthkit/data/arguments/argument.py

@@ -9,11 +9,9 @@
 import logging
 
 from earthkit.data.arguments.earthkit_types import infer_type
-from earthkit.data.arguments.transformers import (
-    AliasTransformer,
-    FormatTransformer,
-    TypeTransformer,
-)
+from earthkit.data.arguments.transformers import AliasTransformer
+from earthkit.data.arguments.transformers import FormatTransformer
+from earthkit.data.arguments.transformers import TypeTransformer
 
 LOG = logging.getLogger(__name__)