Python bindings: rework gdal.run() and add gdal.algorithm()

Author: rouault

autotest/gcore/driver_algorithms.py

@@ -12,6 +12,8 @@
 ###############################################################################
 
 
+import pytest
+
 from osgeo import gdal
 
 
@@ -28,6 +30,8 @@ def check(alg, expected_name):
 
 def test_driver_algorithms():
 
-    alg = gdal.GetGlobalAlgorithmRegistry()["driver"]
-    if alg:
-        check(alg, "driver")
+    try:
+        alg = gdal.GetGlobalAlgorithmRegistry()["driver"]
+    except Exception:
+        pytest.skip("no drivers")
+    check(alg, "driver")

autotest/utilities/test_gdal.py

@@ -457,52 +457,104 @@ def test_gdal_algorithm_getter_setter():
         alg["no-mask"] = "bar"
 
 
-def test_gdal_run():
+def test_gdal_algorithm():
+
+    with pytest.raises(RuntimeError, match="'i_do_not_exist' is not a valid algorithm"):
+        gdal.Algorithm("i_do_not_exist")
+
+    with pytest.raises(RuntimeError, match="'i_do_not_exist' is not a valid algorithm"):
+        gdal.Algorithm(["i_do_not_exist"])
 
     with pytest.raises(
-        Exception, match="i_do_not_exist is not a valid sub-algorithm of gdal"
+        RuntimeError, match="'i_do_not_exist' is not a valid sub-algorithm"
     ):
-        with gdal.run("i_do_not_exist"):
-            pass
+        gdal.Algorithm(["raster", "i_do_not_exist"])
+
+    with pytest.raises(
+        RuntimeError,
+        match="Wrong type for algorithm path. Expected string or list of strings",
+    ):
+        gdal.Algorithm(None)
 
     with pytest.raises(
-        Exception, match="i_do_not_exist is not a valid sub-algorithm of gdal"
+        RuntimeError,
+        match="Wrong type for algorithm path. Expected string or list of strings",
     ):
-        with gdal.run(["i_do_not_exist"]):
+        gdal.Algorithm()
+
+    alg = gdal.Algorithm(["raster", "info"])
+    assert alg.GetName() == "info"
+
+    alg = gdal.Algorithm("raster info")
+    assert alg.GetName() == "info"
+
+    alg = gdal.Algorithm("raster", "info")
+    assert alg.GetName() == "info"
+
+    with pytest.raises(RuntimeError, match=r"Algorithm.Run\(\) must be called before"):
+        alg.Output()
+
+    with pytest.raises(RuntimeError, match=r"Algorithm.Run\(\) must be called before"):
+        alg.Output()
+
+    with gdal.Algorithm("raster info") as alg:
+        assert alg.GetName() == "info"
+
+
+def test_gdal_run():
+
+    with pytest.raises(RuntimeError, match="'i_do_not_exist' is not a valid algorithm"):
+        with gdal.Run("i_do_not_exist"):
+            pass
+
+    with pytest.raises(RuntimeError, match="'i_do_not_exist' is not a valid algorithm"):
+        with gdal.Run(["i_do_not_exist"]):
             pass
 
     with pytest.raises(
-        Exception, match="Wrong type for algorithm. Expected string or list of string"
+        RuntimeError, match="'i_do_not_exist' is not a valid sub-algorithm"
     ):
-        with gdal.run(None):
+        with gdal.Run(["raster", "i_do_not_exist"]):
             pass
 
-    with gdal.run("raster info", {"input": "../gcore/data/byte.tif"}) as res:
-        assert len(res["bands"]) == 1
+    with pytest.raises(
+        RuntimeError,
+        match="Wrong type for alg. Expected string, list of strings or Algorithm",
+    ):
+        gdal.Run(None)
+
+    with gdal.Run("raster", "info", {"input": "../gcore/data/byte.tif"}) as alg:
+        assert len(alg.Output()["bands"]) == 1
+
+    with gdal.Run("raster info", {"input": "../gcore/data/byte.tif"}) as alg:
+        assert len(alg.Output()["bands"]) == 1
+
+    with gdal.Run("gdal raster info", input="../gcore/data/byte.tif") as alg:
+        assert len(alg.Output()["bands"]) == 1
 
-    with gdal.run("gdal raster info", input="../gcore/data/byte.tif") as res:
-        assert len(res["bands"]) == 1
+    with gdal.Run(
+        gdal.Algorithm("raster info"), {"input": "../gcore/data/byte.tif"}
+    ) as alg:
+        assert len(alg.Output()["bands"]) == 1
 
-    with gdal.run(
-        "raster info", {"input": "../gcore/data/byte.tif"}, optimize_single_output=False
-    ) as res:
-        assert len(res) == 1
-        assert res["output-string"].startswith("{")
+    alg = gdal.Run("raster info", {"input": "../gcore/data/byte.tif"})
+    assert len(alg.Outputs()) == 1
+    assert alg.Outputs(parse_json=False)["output-string"].startswith("{")
 
-    with gdal.run(
+    with gdal.Run(
         ["gdal", "raster", "reproject"],
         input="../gcore/data/byte.tif",
         output_format="MEM",
         dst_crs="EPSG:4326",
-    ) as ds:
-        assert ds.GetSpatialRef().GetAuthorityCode(None) == "4326"
+    ) as alg:
+        assert alg.Output().GetSpatialRef().GetAuthorityCode(None) == "4326"
 
-    with gdal.run(
+    with gdal.Run(
         ["raster", "reproject"],
         {
             "input": "../gcore/data/byte.tif",
             "output-format": "MEM",
             "dst-crs": "EPSG:4326",
         },
-    ) as ds:
-        assert ds.GetSpatialRef().GetAuthorityCode(None) == "4326"
+    ) as alg:
+        assert alg.Output().GetSpatialRef().GetAuthorityCode(None) == "4326"

doc/source/programs/gdal_cli_from_python.rst

@@ -4,83 +4,152 @@
 How to use "gdal" CLI algorithms from Python
 ================================================================================
 
-Raster commands
----------------
+Principles
+----------
 
-* Getting information on a raster dataset as JSON
+"gdal" CLI algorithms are available as :py:class:`osgeo.gdal.Algorithm` instances.
 
-.. code-block:: python
+A convenient way to access an algorithm and run it is to use the :py:func:`osgeo.gdal.Run`
+function.
+
+.. py:function:: Run(alg, arguments={}, progress=None, **kwargs)
+
+   Run a GDAL algorithm and return it.
+
+   .. versionadded: 3.11
 
-    from osgeo import gdal
+   :param alg: Path to the algorithm or algorithm instance itself. For example "raster info", or ["raster", "info"] or "raster", "info".
+   :type alg: str, list[str], tuple[str], Algorithm
+   :param arguments: Input arguments of the algorithm. For example {"format": "json", "input": "byte.tif"}
+   :type arguments: dict
+   :param progress: Progress function whose arguments are a progress ratio, a string and a user data
+   :type progress: callable
+   :param kwargs: Instead of using the ``arguments`` parameter, it is possible to pass
+                  algorithm arguments directly as named parameters of gdal.Run().
+                  If the named argument has dash characters in it, the corresponding
+                  parameter must replace them with an underscore character.
+                  For example ``dst_crs`` as a a parameter of gdal.Run(), instead of
+                  ``dst-crs`` which is the name to use on the command line.
+   :rtype: a :py:class:`osgeo.gdal.Algorithm` instance
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "info"], {"input": "byte.tif"}) as info:
-        print(info)
 
+If you do not need to access output value(s) of the algorithm, you can call
+``Run`` directly:
+``gdal.Run(["raster", "convert"], input="in.tif", output="out.tif")``
 
-* Converting a georeferenced netCDF file to cloud-optimized GeoTIFF
+The name of algorithm arguments are the long names as documented in the
+documentation page of each algorithm. They can also be obtained with
+:py:meth:`osgeo.gdal.Algorithm.GetArgNames`.
 
 .. code-block:: python
 
-    from osgeo import gdal
+    >>> gdal.Algorithm("raster", "convert").GetArgNames()
+    ['help', 'help-doc', 'version', 'json-usage', 'drivers', 'config', 'progress', 'output-format', 'open-option', 'input-format', 'input', 'output', 'creation-option', 'overwrite', 'append']
+
+For a command such as :ref:`gdal_raster_info_subcommand` that returns a JSON
+output, you can get the return value of :py:func:`osgeo.gdal.Run` and call the
+:py:meth:`osgeo.gdal.Algorithm.Output` method.
+
+.. code-block:: python
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "convert"], {"input": "in.tif", "output": "out.tif", "output-format": "COG", "overwrite": True}):
-        pass
+    alg = gdal.Run("raster", "info", input="byte.tif")
+    info_as_dict = alg.Output()
 
-or
+If the return value is a dataset, :py:func:`osgeo.gdal.Run` can be used
+within a context manager, in which case :py:meth:`osgeo.gdal.Algorithm.Finalize`
+will be called at the exit of the context manager.
 
 .. code-block:: python
 
-    from osgeo import gdal
+    with gdal.Run("raster reproject", input=src_ds, output_format="MEM",
+                  dst_crs="EPSG:4326"}) as alg:
+        values = alg.Output().ReadAsArray()
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "convert"], input="in.tif", output="out.tif", output_format="COG", overwrite=True):
-        pass
 
+Raster commands examples
+------------------------
 
-* Reprojecting a GeoTIFF file to a Deflate compressed tiled GeoTIFF file
+.. example::
+   :title: Getting information on a raster dataset as JSON
 
-.. code-block:: python
+   .. code-block:: python
 
-    from osgeo import gdal
+        from osgeo import gdal
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "reproject"], {"input": "in.tif", "output": "out.tif", "dst-crs": "EPSG:4326", "creation-options": { "TILED": "YES", "COMPRESS": "DEFLATE"} }):
-        pass
+        gdal.UseExceptions()
+        alg = gdal.Run("raster", "info", input="byte.tif")
+        info_as_dict = alg.Output()
 
 
-* Reprojecting a (possibly in-memory) dataset to a in-memory dataset
+.. example::
+   :title: Converting a georeferenced netCDF file to cloud-optimized GeoTIFF
 
-.. code-block:: python
+   .. code-block:: python
 
-    from osgeo import gdal
+        from osgeo import gdal
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "reproject"], {"input": "in.tif", "output-format": "MEM", "dst-crs": "EPSG:4326"}) as ds:
-        print(ds.ReadAsArray())
+        gdal.UseExceptions()
+        gdal.Run("raster", "convert", input="in.tif", output="out.tif",
+                 output_format="COG", overwrite=True)
 
+   or
 
-Vector commands
----------------
+   .. code-block:: python
 
-* Getting information on a vector dataset as JSON
+        from osgeo import gdal
 
-.. code-block:: python
+        gdal.UseExceptions()
+        gdal.Run(["raster", "convert"], {"input": "in.tif", "output": "out.tif", "output-format": "COG", "overwrite": True})
 
-    from osgeo import gdal
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "info"], {"input": "poly.gpkg"}) as info:
-        print(info)
+.. example::
+   :title: Reprojecting a GeoTIFF file to a Deflate-compressed tiled GeoTIFF file
 
+   .. code-block:: python
 
-* Converting a shapefile to a GeoPackage
+        from osgeo import gdal
 
-.. code-block:: python
+        gdal.UseExceptions()
+        gdal.Run("raster", "reproject", input="in.tif", output="out.tif",
+                  dst_crs="EPSG:4326",
+                  creation_options={ "TILED": "YES", "COMPRESS": "DEFLATE"})
+
+
+.. example::
+   :title: Reprojecting a (possibly in-memory) dataset to a in-memory dataset
+
+   .. code-block:: python
+
+        from osgeo import gdal
+
+        gdal.UseExceptions()
+        with gdal.Run("raster", "reproject", input=src_ds, output_format="MEM",
+                      dst_crs="EPSG:4326"}) as alg:
+            values = alg.Output().ReadAsArray()
+
+
+Vector commands examples
+------------------------
+
+
+.. example::
+   :title: Getting information on a vector dataset as JSON
+
+   .. code-block:: python
+
+        from osgeo import gdal
+
+        gdal.UseExceptions()
+        alg = gdal.Run("raster", "info", input="poly.gpkg"})
+        info_as_dict = alg.Output()
+
+
+.. example::
+   :title: Converting a shapefile to a GeoPackage
+
+   .. code-block:: python
 
-    from osgeo import gdal
+        from osgeo import gdal
 
-    gdal.UseExceptions()
-    with gdal.run(["raster", "convert"], {"input": "in.shp", "output": "out.gpkg", "overwrite": True}):
-        pass
+        gdal.UseExceptions()
+        gdal.Run("raster", "convert", input="in.shp", output="out.gpkg", overwrite=True)

doc/source/spelling_wordlist.txt

@@ -46,6 +46,7 @@ Alamos
 Albers
 Alessandro
 alg
+AlgorithmArg
 Alibaba
 Alicante
 allCountries