Python bindings: add gdal.run() context manager to run GDAL algorithms

Fixes #11832

Author: rouault

autotest/utilities/test_gdal.py

@@ -439,3 +439,54 @@ def test_gdal_algorithm_getter_setter():
 
     with pytest.raises(Exception):
         alg["no-mask"] = "bar"
+
+
+def test_gdal_run():
+
+    with pytest.raises(
+        Exception, match="i_do_not_exist is not a valid sub-algorithm of gdal"
+    ):
+        with gdal.run("i_do_not_exist"):
+            pass
+
+    with pytest.raises(
+        Exception, match="i_do_not_exist is not a valid sub-algorithm of gdal"
+    ):
+        with gdal.run(["i_do_not_exist"]):
+            pass
+
+    with pytest.raises(
+        Exception, match="Wrong type for algorithm. Expected string or list of string"
+    ):
+        with gdal.run(None):
+            pass
+
+    with gdal.run("raster info", {"input": "../gcore/data/byte.tif"}) as res:
+        assert len(res["bands"]) == 1
+
+    with gdal.run("gdal raster info", input="../gcore/data/byte.tif") as res:
+        assert len(res["bands"]) == 1
+
+    with gdal.run(
+        "raster info", {"input": "../gcore/data/byte.tif"}, optimize_single_output=False
+    ) as res:
+        assert len(res) == 1
+        assert res["output-string"].startswith("{")
+
+    with gdal.run(
+        ["gdal", "raster", "reproject"],
+        input="../gcore/data/byte.tif",
+        output_format="MEM",
+        dst_crs="EPSG:4326",
+    ) as ds:
+        assert ds.GetSpatialRef().GetAuthorityCode(None) == "4326"
+
+    with gdal.run(
+        ["raster", "reproject"],
+        {
+            "input": "../gcore/data/byte.tif",
+            "output-format": "MEM",
+            "dst-crs": "EPSG:4326",
+        },
+    ) as ds:
+        assert ds.GetSpatialRef().GetAuthorityCode(None) == "4326"

doc/source/programs/gdal_cli_from_python.rst

@@ -12,79 +12,53 @@ Raster commands
 .. code-block:: python
 
     from osgeo import gdal
-    import json
 
     gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["raster"]["info"]
-    alg["input"] = "byte.tif"
-    alg.Run()
-    info = json.loads(alg["output-string"])
-    print(info)
+    with gdal.run(["raster", "info"], {"input": "byte.tif"}) as info:
+        print(info)
 
 
 * Converting a georeferenced netCDF file to cloud-optimized GeoTIFF
 
 .. code-block:: python
 
     from osgeo import gdal
+
     gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["raster"]["convert"]
-    alg["input"] = "in.nc"
-    alg["output"] = "out.tif"
-    alg["output-format"] = "COG"
-    alg["overwrite"] = True  # if the output file may exist
-    alg.Run()
-    alg.Finalize() # ensure output dataset is closed
+    with gdal.run(["raster", "convert"], {"input": "in.tif", "output": "out.tif", "output-format": "COG", "overwrite": True}):
+        pass
 
 or
 
 .. code-block:: python
 
     from osgeo import gdal
+
     gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["raster"]["convert"]
-    alg.ParseRunAndFinalize(["--input=in.nc", "--output-format=COG", "--output=out.tif", "--overwrite"])
+    with gdal.run(["raster", "convert"], input="in.tif", output="out.tif", output_format="COG", overwrite=True):
+        pass
 
 
 * Reprojecting a GeoTIFF file to a Deflate compressed tiled GeoTIFF file
 
 .. code-block:: python
 
     from osgeo import gdal
-    gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["raster"]["reproject"]
-    alg["input"] = "in.tif"
-    alg["output"] = "out.tif"
-    alg["dst-crs"] = "EPSG:4326"
-    alg["creation-options"] = ["TILED=YES", "COMPRESS=DEFLATE"]
-    alg["overwrite"] = True  # if the output file may exist
-    alg.Run()
-    alg.Finalize() # ensure output dataset is closed
-
-or
-
-.. code-block:: python
 
-    from osgeo import gdal
     gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["raster"]["reproject"]
-    alg.ParseRunAndFinalize(["--input=in.tif", "--output=out.tif", "--dst-crs=EPSG:4326", "--co=TILED=YES,COMPRESS=DEFLATE", "--overwrite"])
+    with gdal.run(["raster", "reproject"], {"input": "in.tif", "output": "out.tif", "dst-crs": "EPSG:4326", "creation-options": { "TILED": "YES", "COMPRESS": "DEFLATE"} }):
+        pass
 
 
 * Reprojecting a (possibly in-memory) dataset to a in-memory dataset
 
 .. code-block:: python
 
     from osgeo import gdal
-    gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["raster"]["reproject"]
-    alg["input"] = input_ds
-    alg["output"] = "dummy_name"
-    alg["output-format"] = "MEM"
-    alg["dst-crs"] = "EPSG:4326"
-    alg.Run()
-    output_ds = alg["output"].GetDataset()
 
+    gdal.UseExceptions()
+    with gdal.run(["raster", "reproject"], {"input": "in.tif", "output-format": "MEM", "dst-crs": "EPSG:4326"}) as ds:
+        print(ds.ReadAsArray())
 
 
 Vector commands
@@ -95,34 +69,18 @@ Vector commands
 .. code-block:: python
 
     from osgeo import gdal
-    import json
 
     gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["vector"]["info"]
-    alg["input"] = "poly.gpkg"
-    alg.Run()
-    info = json.loads(alg["output-string"])
-    print(info)
+    with gdal.run(["raster", "info"], {"input": "poly.gpkg"}) as info:
+        print(info)
 
 
 * Converting a shapefile to a GeoPackage
 
 .. code-block:: python
 
     from osgeo import gdal
-    gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["vector"]["convert"]
-    alg["input"] = "in.shp"
-    alg["output"] = "out.gpkg"
-    alg["overwrite"] = True  # if the output file may exist
-    alg.Run()
-    alg.Finalize() # ensure output dataset is closed
-
-or
-
-.. code-block:: python
 
-    from osgeo import gdal
     gdal.UseExceptions()
-    alg = gdal.GetGlobalAlgorithmRegistry()["vector"]["convert"]
-    alg.ParseRunAndFinalize(["--input=in.shp", "--output=out.gpkg", "--overwrite"])
+    with gdal.run(["raster", "convert"], {"input": "in.shp", "output": "out.gpkg", "overwrite": True}):
+        pass

gcore/gdalalgorithm.cpp

@@ -1941,6 +1941,20 @@ bool GDALAlgorithm::ValidateArguments()
     if (m_specialActionRequested)
         return true;
 
+    // If only --output=format=MEM is specified and not --output,
+    // then set empty name for --output.
+    auto outputArg = GetArg(GDAL_ARG_NAME_OUTPUT);
+    auto outputFormatArg = GetArg(GDAL_ARG_NAME_OUTPUT_FORMAT);
+    if (outputArg && outputFormatArg && outputFormatArg->IsExplicitlySet() &&
+        !outputArg->IsExplicitlySet() &&
+        outputFormatArg->GetType() == GAAT_STRING &&
+        EQUAL(outputFormatArg->Get<std::string>().c_str(), "MEM") &&
+        outputArg->GetType() == GAAT_DATASET &&
+        (outputArg->Get<GDALArgDatasetValue>().GetInputFlags() & GADV_NAME))
+    {
+        outputArg->Get<GDALArgDatasetValue>().Set("");
+    }
+
     // The method may emit several errors if several constraints are not met.
     bool ret = true;
     std::map<std::string, std::string> mutualExclusionGroupUsed;

swig/include/python/gdal_python.i

@@ -5165,6 +5165,117 @@ def quiet_warnings():
     finally:
         PopErrorHandler()
 
+
+@contextlib.contextmanager
+def run(algorithm=None, arguments={}, progress=None, optimize_single_output=True, finalize=True, **kwargs):
+    """Run a GDAL algorithm as a context manager
+
+       .. versionadded: 3.11
+
+       Parameters
+       ----------
+       algorithm: str or list[str]
+            Path to the algorithm. For example "raster info", or ["raster", "info"].
+       arguments: dict
+            Input arguments of the algorithm. For example {"format": "json", "input": "byte.tif"}
+       progress: callable
+            Progress function whose arguments are a progress ratio, a string and a user data
+       optimize_single_output: bool
+            Whether to return a single value when there is a single output argument, and to
+            deserialize automatically JSON responses as a dict, and return a osgeo.gdal.Dataset
+            when possible.
+       finalize: bool
+            Whether to run :py:func:`gdal.Algorithm.Finalize` when the context is released.
+       kwargs:
+            Instead of using the ``arguments`` parameter, it is possible to pass
+            algorithm arguments directly as named parameters of gdal.run().
+            If the named argument has dash characters in it, the corresponding
+            parameter must replace them with an underscore character.
+            For example ``dst_crs`` as a a parameter of gdal.run(), instead of
+            ``dst-crs`` which is the name to use on the command line.
+
+       Returns
+       -------
+            A context manager with the output arguments of the algorithm
+
+       Example
+       -------
+
+       >>> with gdal.run(["raster", "info"], {"input": "byte.tif"}) as res:
+       ...     print(res["bands"])
+
+       >>> with gdal.run("raster reproject", input="byte.tif", output_format="MEM", dst_crs="EPSG:4326") as ds
+       ...     print(ds.ReadAsArray())
+    """
+
+    if isinstance(algorithm, list):
+        alg = GetGlobalAlgorithmRegistry()
+        parent_name = "gdal"
+        for i, v in enumerate(algorithm):
+            if i == 0 and v == "gdal":
+                continue
+            subalg = alg[v]
+            if not subalg:
+                raise Exception(f"{v} is not a valid sub-algorithm of {parent_name}")
+            alg = subalg
+            parent_name = alg.GetName()
+    elif isinstance(algorithm, str):
+        if algorithm.startswith("gdal "):
+            algorithm = algorithm[len("gdal "):]
+        alg = GetGlobalAlgorithmRegistry()
+        parent_name = "gdal"
+        for i, v in enumerate(algorithm.split(' ')):
+            if i == 0 and v == "gdal":
+                continue
+            subalg = alg[v]
+            if not subalg:
+                raise Exception(f"{v} is not a valid sub-algorithm of {parent_name}")
+            alg = subalg
+            parent_name = alg.GetName()
+    else:
+        raise Exception("Wrong type for algorithm. Expected string or list of string")
+
+    for k in arguments:
+        alg[k.replace('_', '-')] = arguments[k]
+
+    for k in kwargs:
+        alg[k.replace('_', '-')] = kwargs[k]
+
+    assert alg.Run(progress)
+
+    count_output = 0
+    if optimize_single_output:
+        for name in alg.GetArgNames():
+            arg = alg.GetArg(name)
+            if arg.IsOutput():
+                count_output += 1
+
+    res = {}
+    for name in alg.GetArgNames():
+        arg = alg.GetArg(name)
+        if arg.IsOutput():
+            val = alg[name]
+            if name == "output-string" and count_output == 1:
+                if (val.startswith('{') and (val.endswith('}') or val.endswith('}\n'))) or \
+                   (val.startswith('[') and (val.endswith(']') or val.endswith(']\n'))):
+                    import json
+                    res = json.loads(val)
+                else:
+                    res = val
+            elif count_output == 1:
+                if arg.GetType() == GAAT_DATASET:
+                    res = val.GetDataset()
+                else:
+                    res = val
+            else:
+                res[name] = val
+
+    yield res
+
+    if finalize:
+        assert alg.Finalize()
+
+
 %}