Skip to content

Commit 26081e9

Browse files
rouaultrouault
authored
Merge pull request OSGeo#11962 from rouault/gdalg
Add GDALG (GDAL Streamed Algorithm Format) driver
2 parents 08f6aed + d69bcc4 commit 26081e9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1891
-189
lines changed

‎.github/workflows/ubuntu_24.04/expected_gdalinfo_formats.txt

+1
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
130130
STACTA -raster- (rovs): Spatio-Temporal Asset Catalog Tiled Assets (*.json)
131131
STACIT -raster- (rovs): Spatio-Temporal Asset Catalog Items
132132
JPEGXL -raster- (rwv): JPEG-XL (*.jxl)
133+
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
133134
GPKG -raster,vector- (rw+uvs): GeoPackage (*.gpkg, *.gpkg.zip)
134135
SQLite -raster,vector- (rw+uv): SQLite / Spatialite / RasterLite2 (*.sqlite, *.db)
135136
OpenFileGDB -raster,vector- (rw+uv): ESRI FileGeodatabase (using OpenFileGDB) (*.gdb)

‎.github/workflows/ubuntu_24.04/expected_ogrinfo_formats.txt

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
1010
BAG -raster,multidimensional raster,vector- (rw+v): Bathymetry Attributed Grid (*.bag)
1111
EEDA -vector- (ro): Earth Engine Data API
1212
OGCAPI -raster,vector- (rov): OGCAPI
13+
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
1314
ESRI Shapefile -vector- (rw+uv): ESRI Shapefile (*.shp, *.dbf, *.shz, *.shp.zip)
1415
MapInfo File -vector- (rw+uv): MapInfo File (*.tab, *.mif, *.mid)
1516
LVBAG -vector- (rov): Kadaster LV BAG Extract 2.0 (*.xml)

‎.github/workflows/windows_conda_expected_gdalinfo_formats.txt

+1
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
129129
OGCAPI -raster,vector- (rov): OGCAPI
130130
STACTA -raster- (rovs): Spatio-Temporal Asset Catalog Tiled Assets (*.json)
131131
STACIT -raster- (rovs): Spatio-Temporal Asset Catalog Items
132+
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
132133
NSIDCbin -raster- (rov): NSIDC Sea Ice Concentrations binary (.bin) (*.bin)
133134
GPKG -raster,vector- (rw+uvs): GeoPackage (*.gpkg, *.gpkg.zip)
134135
OpenFileGDB -raster,vector- (rw+uv): ESRI FileGeodatabase (using OpenFileGDB) (*.gdb)

‎.github/workflows/windows_conda_expected_ogrinfo_formats.txt

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Supported Formats: (ro:read-only, rw:read-write, +:write from scratch, u:update,
1111
BAG -raster,multidimensional raster,vector- (rw+v): Bathymetry Attributed Grid (*.bag)
1212
EEDA -vector- (ro): Earth Engine Data API
1313
OGCAPI -raster,vector- (rov): OGCAPI
14+
GDALG -raster,vector- (rov): GDAL Streamed Algorithm driver (*.gdalg.json)
1415
ESRI Shapefile -vector- (rw+uv): ESRI Shapefile (*.shp, *.dbf, *.shz, *.shp.zip)
1516
MapInfo File -vector- (rw+uv): MapInfo File (*.tab, *.mif, *.mid)
1617
LVBAG -vector- (rov): Kadaster LV BAG Extract 2.0 (*.xml)

‎apps/data/gdal_algorithm.schema.json

+4
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@
7171
"$ref": "#/definitions/algorithm"
7272
},
7373
"$comment": "For pipeline algorithm, description of the accepted step algorithms. Only present for such pipeline algorithms"
74+
},
75+
"supports_streamed_output": {
76+
"type": "boolean",
77+
"$comment": "Whether the algorithm supports a streamed output dataset"
7478
}
7579
},
7680
"required": [

‎apps/gdal.cpp

-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,6 @@ MAIN_START(argc, argv)
121121
if (argc < 1)
122122
return (-argc);
123123

124-
alg->SetCallPath(std::vector<std::string>{argv[0]});
125124
std::vector<std::string> args;
126125
for (int i = 1; i < argc; ++i)
127126
args.push_back(argv[i]);

‎apps/gdalalg_abstract_pipeline.h

+88
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,94 @@ bool GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep(
160160
return false;
161161
}
162162

163+
// Handle output to GDALG file
164+
if (!m_steps.empty() && m_steps.back()->GetName() == "write")
165+
{
166+
if (m_steps.back()->IsGDALGOutput())
167+
{
168+
const auto outputArg = m_steps.back()->GetArg(GDAL_ARG_NAME_OUTPUT);
169+
const auto &filename =
170+
outputArg->GDALAlgorithmArg::template Get<GDALArgDatasetValue>()
171+
.GetName();
172+
VSIStatBufL sStat;
173+
if (VSIStatL(filename.c_str(), &sStat) == 0)
174+
{
175+
const auto overwriteArg =
176+
m_steps.back()->GetArg(GDAL_ARG_NAME_OVERWRITE);
177+
if (overwriteArg && overwriteArg->GetType() == GAAT_BOOLEAN)
178+
{
179+
if (!overwriteArg->GDALAlgorithmArg::template Get<bool>())
180+
{
181+
CPLError(CE_Failure, CPLE_AppDefined,
182+
"File '%s' already exists. Specify the "
183+
"--overwrite option to overwrite it.",
184+
filename.c_str());
185+
return false;
186+
}
187+
}
188+
}
189+
190+
std::string osCommandLine;
191+
192+
for (const auto &path : GDALAlgorithm::m_callPath)
193+
{
194+
if (!osCommandLine.empty())
195+
osCommandLine += ' ';
196+
osCommandLine += path;
197+
}
198+
199+
// Do not include the last step
200+
for (size_t i = 0; i + 1 < m_steps.size(); ++i)
201+
{
202+
const auto &step = m_steps[i];
203+
if (i > 0)
204+
osCommandLine += " !";
205+
osCommandLine += ' ';
206+
osCommandLine += step->GetName();
207+
208+
for (const auto &arg : step->GetArgs())
209+
{
210+
if (arg->IsExplicitlySet())
211+
{
212+
osCommandLine += ' ';
213+
std::string strArg;
214+
if (!arg->Serialize(strArg))
215+
{
216+
CPLError(CE_Failure, CPLE_AppDefined,
217+
"Cannot serialize argument %s",
218+
arg->GetName().c_str());
219+
return false;
220+
}
221+
osCommandLine += strArg;
222+
}
223+
}
224+
}
225+
226+
CPLJSONDocument oDoc;
227+
oDoc.GetRoot().Add("type", "gdal_streamed_alg");
228+
oDoc.GetRoot().Add("command_line", osCommandLine);
229+
230+
return oDoc.Save(filename);
231+
}
232+
}
233+
234+
if (GDALAlgorithm::m_executionForStreamOutput)
235+
{
236+
// For security reasons, to avoid that reading a .gdalg.json file writes
237+
// a file on the file system.
238+
for (const auto &step : m_steps)
239+
{
240+
if (step->GetName() == "write" &&
241+
!EQUAL(step->m_format.c_str(), "stream"))
242+
{
243+
StepAlgorithm::ReportError(CE_Failure, CPLE_AppDefined,
244+
"in streamed execution, --format "
245+
"stream should be used");
246+
return false;
247+
}
248+
}
249+
}
250+
163251
GDALDataset *poCurDS = nullptr;
164252
for (size_t i = 0; i < m_steps.size(); ++i)
165253
{

‎apps/gdalalg_main.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ GDALMainAlgorithm::GDALMainAlgorithm()
3030
RegisterSubAlgorithm(*pInfo);
3131
}
3232

33+
SetCallPath({NAME});
34+
3335
m_longDescription = "'gdal <FILENAME>' can also be used as a shortcut for "
3436
"'gdal info <FILENAME>'.\n"
3537
"And 'gdal read <FILENAME> ! ...' as a shortcut for "

‎apps/gdalalg_raster_mosaic.cpp

+16-3
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,11 @@
3131
GDALRasterMosaicAlgorithm::GDALRasterMosaicAlgorithm()
3232
: GDALAlgorithm(NAME, DESCRIPTION, HELP_URL)
3333
{
34+
m_supportsStreamedOutput = true;
35+
3436
AddProgressArg();
35-
AddOutputFormatArg(&m_format);
37+
AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
38+
/* bGDALGAllowed = */ true);
3639
AddArg(GDAL_ARG_NAME_INPUT, 'i',
3740
_("Input raster datasets (or specify a @<filename> to point to a "
3841
"file containing filenames)"),
@@ -157,7 +160,14 @@ bool GDALRasterMosaicAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
157160
}
158161
else
159162
{
160-
aosInputDatasetNames.push_back(ds.GetName().c_str());
163+
std::string osDatasetName = ds.GetName();
164+
if (!GetReferencePathForRelativePaths().empty())
165+
{
166+
osDatasetName = GDALDataset::BuildFilename(
167+
osDatasetName.c_str(),
168+
GetReferencePathForRelativePaths().c_str(), true);
169+
}
170+
aosInputDatasetNames.push_back(osDatasetName.c_str());
161171
}
162172
}
163173
}
@@ -184,6 +194,7 @@ bool GDALRasterMosaicAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
184194

185195
const bool bVRTOutput =
186196
m_outputDataset.GetName().empty() || EQUAL(m_format.c_str(), "VRT") ||
197+
EQUAL(m_format.c_str(), "stream") ||
187198
EQUAL(CPLGetExtensionSafe(m_outputDataset.GetName().c_str()).c_str(),
188199
"VRT");
189200

@@ -273,7 +284,9 @@ bool GDALRasterMosaicAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
273284
}
274285

275286
auto poOutDS = std::unique_ptr<GDALDataset>(GDALDataset::FromHandle(
276-
GDALBuildVRT(bVRTOutput ? m_outputDataset.GetName().c_str() : "",
287+
GDALBuildVRT(EQUAL(m_format.c_str(), "stream") ? ""
288+
: bVRTOutput ? m_outputDataset.GetName().c_str()
289+
: "",
277290
foundByName ? aosInputDatasetNames.size()
278291
: static_cast<int>(m_inputDatasets.size()),
279292
ahInputDatasets.empty() ? nullptr : ahInputDatasets.data(),

‎apps/gdalalg_raster_pipeline.cpp

+84-5
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ GDALRasterPipelineStepAlgorithm::GDALRasterPipelineStepAlgorithm(
4141
{
4242
if (m_standaloneStep)
4343
{
44+
m_supportsStreamedOutput = true;
45+
4446
AddInputArgs(false, false);
4547
AddProgressArg();
4648
AddOutputArgs(false);
@@ -76,7 +78,8 @@ void GDALRasterPipelineStepAlgorithm::AddInputArgs(
7678

7779
void GDALRasterPipelineStepAlgorithm::AddOutputArgs(bool hiddenForCLI)
7880
{
79-
AddOutputFormatArg(&m_format)
81+
AddOutputFormatArg(&m_format, /* bStreamAllowed = */ true,
82+
/* bGDALGAllowed = */ true)
8083
.AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
8184
{GDAL_DCAP_RASTER, GDAL_DCAP_CREATECOPY})
8285
.SetHiddenForCLI(hiddenForCLI);
@@ -119,20 +122,38 @@ bool GDALRasterPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
119122
}
120123
}
121124

125+
if (m_executionForStreamOutput && !EQUAL(m_format.c_str(), "stream"))
126+
{
127+
// For security reasons, to avoid that reading a .gdalg.json file
128+
// writes a file on the file system.
129+
ReportError(CE_Failure, CPLE_NotSupported,
130+
"gdal raster pipeline in streamed execution should use "
131+
"--format stream");
132+
return false;
133+
}
134+
122135
bool ret = false;
123136
if (readAlg.Run())
124137
{
125138
m_inputDataset.Set(readAlg.m_outputDataset.GetDatasetRef());
126139
m_outputDataset.Set(nullptr);
127140
if (RunStep(nullptr, nullptr))
128141
{
129-
writeAlg.m_inputDataset.Set(m_outputDataset.GetDatasetRef());
130-
if (writeAlg.Run(pfnProgress, pProgressData))
142+
if (m_format == "stream")
131143
{
132-
m_outputDataset.Set(
133-
writeAlg.m_outputDataset.GetDatasetRef());
134144
ret = true;
135145
}
146+
else
147+
{
148+
writeAlg.m_inputDataset.Set(
149+
m_outputDataset.GetDatasetRef());
150+
if (writeAlg.Run(pfnProgress, pProgressData))
151+
{
152+
m_outputDataset.Set(
153+
writeAlg.m_outputDataset.GetDatasetRef());
154+
ret = true;
155+
}
156+
}
136157
}
137158
}
138159

@@ -144,6 +165,44 @@ bool GDALRasterPipelineStepAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
144165
}
145166
}
146167

168+
/************************************************************************/
169+
/* ProcessGDALGOutput() */
170+
/************************************************************************/
171+
172+
GDALAlgorithm::ProcessGDALGOutputRet
173+
GDALRasterPipelineStepAlgorithm::ProcessGDALGOutput()
174+
{
175+
if (m_standaloneStep)
176+
{
177+
return GDALAlgorithm::ProcessGDALGOutput();
178+
}
179+
else
180+
{
181+
// GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep() might
182+
// actually detect a GDALG output request and process it.
183+
return GDALAlgorithm::ProcessGDALGOutputRet::NOT_GDALG;
184+
}
185+
}
186+
187+
/************************************************************************/
188+
/* GDALRasterPipelineStepAlgorithm::CheckSafeForStreamOutput() */
189+
/************************************************************************/
190+
191+
bool GDALRasterPipelineStepAlgorithm::CheckSafeForStreamOutput()
192+
{
193+
if (m_standaloneStep)
194+
{
195+
return GDALAlgorithm::CheckSafeForStreamOutput();
196+
}
197+
else
198+
{
199+
// The check is actually done in
200+
// GDALAbstractPipelineAlgorithm<StepAlgorithm>::RunStep()
201+
// so return true for now.
202+
return true;
203+
}
204+
}
205+
147206
/************************************************************************/
148207
/* GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm() */
149208
/************************************************************************/
@@ -154,6 +213,8 @@ GDALRasterPipelineAlgorithm::GDALRasterPipelineAlgorithm(
154213
NAME, DESCRIPTION, HELP_URL,
155214
/*standaloneStep=*/false)
156215
{
216+
m_supportsStreamedOutput = true;
217+
157218
AddInputArgs(openForMixedRasterVector, /* hiddenForCLI = */ true);
158219
AddProgressArg();
159220
AddArg("pipeline", 0, _("Pipeline string"), &m_pipeline)
@@ -285,6 +346,18 @@ bool GDALRasterPipelineAlgorithm::ParseCommandLineArguments(
285346
if (!steps.back().alg)
286347
steps.pop_back();
287348

349+
// Automatically add a final write step if none in m_executionForStreamOutput
350+
// mode
351+
if (m_executionForStreamOutput && !steps.empty() &&
352+
steps.back().alg->GetName() != GDALRasterWriteAlgorithm::NAME)
353+
{
354+
steps.resize(steps.size() + 1);
355+
steps.back().alg = GetStepAlg(GDALRasterWriteAlgorithm::NAME);
356+
steps.back().args.push_back("--output-format");
357+
steps.back().args.push_back("stream");
358+
steps.back().args.push_back("streamed_dataset");
359+
}
360+
288361
if (steps.size() < 2)
289362
{
290363
ReportError(CE_Failure, CPLE_AppDefined,
@@ -325,6 +398,12 @@ bool GDALRasterPipelineAlgorithm::ParseCommandLineArguments(
325398
}
326399
}
327400

401+
for (auto &step : steps)
402+
{
403+
step.alg->SetReferencePathForRelativePaths(
404+
GetReferencePathForRelativePaths());
405+
}
406+
328407
if (!m_pipeline.empty())
329408
{
330409
// Propagate input parameters set at the pipeline level to the

‎apps/gdalalg_raster_pipeline.h

+2
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ class GDALRasterPipelineStepAlgorithm /* non final */ : public GDALAlgorithm
5555

5656
private:
5757
bool RunImpl(GDALProgressFunc pfnProgress, void *pProgressData) override;
58+
GDALAlgorithm::ProcessGDALGOutputRet ProcessGDALGOutput() override;
59+
bool CheckSafeForStreamOutput() override;
5860
};
5961

6062
/************************************************************************/

0 commit comments

Comments
 (0)