From bcc37563819fd4e8e952a07e6ea262def1d3976d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 17 Dec 2024 17:32:32 +0100 Subject: [PATCH 01/40] /vsicurl?: accept header.= to specify HTTP request headers Fixes #11503 --- autotest/gcore/vsicurl.py | 24 ++++++++++++++++++++++++ doc/source/user/virtual_file_systems.rst | 1 + port/cpl_vsil_curl.cpp | 11 +++++++++++ 3 files changed, 36 insertions(+) diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index 2bc0f2f2b4b4..43c7de4bb761 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -1609,3 +1609,27 @@ def test_vsicurl_VSICURL_QUERY_STRING(server, filename, query_string): assert statres.size == 3 finally: gdal.SetPathSpecificOption(full_filename, "VSICURL_QUERY_STRING", None) + + +############################################################################### +# Test /vsicurl?header.foo=bar& + + +@gdaltest.enable_exceptions() +def test_vsicurl_header_option(server): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add( + "HEAD", + "/test_vsicurl_header_option.bin", + 200, + {"Content-Length": "3"}, + expected_headers={"foo": "bar", "Accept": "application/json"}, + ) + + with webserver.install_http_handler(handler): + full_filename = f"/vsicurl?header.foo=bar&header.Accept=application%2Fjson&url=http%3A%2F%2Flocalhost%3A{server.port}%2Ftest_vsicurl_header_option.bin" + statres = gdal.VSIStatL(full_filename) + assert statres.size == 3 diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst index 1b1eb0526dd2..ddacadfc5b0c 100644 --- a/doc/source/user/virtual_file_systems.rst +++ b/doc/source/user/virtual_file_systems.rst @@ -387,6 +387,7 @@ Starting with GDAL 2.3, options can be passed in the filename with the following - referer=value: HTTP Referer header - cookie=value: HTTP Cookie header - header_file=value: Filename that contains one or several "Header: Value" lines +- header.=: HTTP request header of name and value . (GDAL >= 3.11). e.g. ``header.Accept=application%2Fjson`` - unsafessl=yes/no - low_speed_time=value - low_speed_limit=value diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp index e259b4078aa0..9ea17cabf2c0 100644 --- a/port/cpl_vsil_curl.cpp +++ b/port/cpl_vsil_curl.cpp @@ -352,6 +352,7 @@ static std::string VSICurlGetURLFromFilename( } std::string osURL; + std::string osHeaders; for (int i = 0; papszTokens[i]; i++) { char *pszKey = nullptr; @@ -433,6 +434,13 @@ static std::string VSICurlGetURLFromFilename( *ppszPlanetaryComputerCollection = CPLStrdup(pszValue); } } + else if (STARTS_WITH(pszKey, "header.")) + { + osHeaders += (pszKey + strlen("header.")); + osHeaders += ':'; + osHeaders += pszValue; + osHeaders += "\r\n"; + } else { CPLError(CE_Warning, CPLE_NotSupported, @@ -442,6 +450,9 @@ static std::string VSICurlGetURLFromFilename( CPLFree(pszKey); } + if (paosHTTPOptions && !osHeaders.empty()) + paosHTTPOptions->SetNameValue("HEADERS", osHeaders.c_str()); + CSLDestroy(papszTokens); if (osURL.empty()) { From e4fe405de90d81a9f9b1facf8c83b14f5486e600 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2024 21:26:42 +0100 Subject: [PATCH 02/40] GeoJSON: add a FOREIGN_MEMBERS=AUTO/ALL/NONE/STAC open option ``` - .. oo:: FOREIGN_MEMBERS :choices: AUTO, ALL, NONE, STAC :default: AUTO :since: 3.11.0 Whether and how foreign members at the feature level should be processed as OGR fields: - ``AUTO`` mode behaves like ``STAC`` mode if a ``stac_version`` member is found at the Feature level, otherwise it behaves as ``NONE`` mode. - In ``ALL`` mode, all foreign members at the feature level are added. Whether to recursively explore nested objects and produce flatten OGR attributes or not is decided by the ``FLATTEN_NESTED_ATTRIBUTES`` open option. - In ``NONE`` mode, no foreign members at the feature level are added. - ``STAC`` mode (Spatio-Temporal Asset Catalog) behaves the same as ``ALL``, except content under the ``assets`` member is by default flattened as ``assets.{asset_name}.{asset_property}`` fields. ``` --- autotest/ogr/data/geojson/stac_item.json | 109 ++++++++++++ autotest/ogr/ogr_geojson.py | 36 ++++ autotest/ogr/ogr_miramon_vector.py | 7 +- doc/source/drivers/vector/geojson.rst | 21 +++ .../geojson/ogrgeojsondatasource.cpp | 23 +++ ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp | 8 + ogr/ogrsf_frmts/geojson/ogrgeojsonreader.cpp | 165 ++++++++++++++++++ ogr/ogrsf_frmts/geojson/ogrgeojsonreader.h | 16 ++ 8 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 autotest/ogr/data/geojson/stac_item.json diff --git a/autotest/ogr/data/geojson/stac_item.json b/autotest/ogr/data/geojson/stac_item.json new file mode 100644 index 000000000000..05821ff9c8c7 --- /dev/null +++ b/autotest/ogr/data/geojson/stac_item.json @@ -0,0 +1,109 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + ], + "id": "my_id", + "description": "Landsat Collection 2 Level-2 Surface Reflectance Product", + "bbox": [ + -155.80483200278817, + 17.736060531368373, + -153.68026753986817, + 19.82593799656933 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -155.41188228421524, + 19.82593799656933 + ], + [ + -155.80483200278817, + 18.09429339392015 + ], + [ + -154.0866457786547, + 17.736060531368373 + ], + [ + -153.68026753986817, + 19.470851166420736 + ], + [ + -155.41188228421524, + 19.82593799656933 + ] + ] + ] + }, + "properties": { + "datetime": "2024-12-16T20:42:39.252121Z", + "eo:cloud_cover": 58.93, + "view:sun_azimuth": 150.92260609, + "view:sun_elevation": 42.2078599, + "platform": "LANDSAT_9", + "instruments": [ + "OLI", + "TIRS" + ], + "view:off_nadir": 0, + "landsat:cloud_cover_land": 43.14, + "landsat:wrs_type": "2", + "landsat:wrs_path": "062", + "landsat:wrs_row": "047", + "landsat:scene_id": "LC90620472024351LGN00", + "landsat:collection_category": "A1", + "landsat:collection_number": "02", + "landsat:correction": "L2SP", + "accuracy:geometric_x_bias": 0, + "accuracy:geometric_y_bias": 0, + "accuracy:geometric_x_stddev": 5.048, + "accuracy:geometric_y_stddev": 4.77, + "accuracy:geometric_rmse": 6.946, + "proj:epsg": null, + "proj:shape": [ + 7701, + 7571 + ], + "proj:transform": [ + 30, + 0, + 124185, + 0, + -30, + 1862415 + ], + "proj:wkt2": "PROJCS[\"AEA WGS84\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Albers_Conic_Equal_Area\"],PARAMETER[\"latitude_of_center\",3],PARAMETER[\"longitude_of_center\",-157],PARAMETER[\"standard_parallel_1\",8],PARAMETER[\"standard_parallel_2\",18],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH]]", + "card4l:specification": "SR", + "card4l:specification_version": "5.0", + "created": "2024-12-17T13:13:26.984Z", + "updated": "2024-12-18T04:59:46.657Z" + }, + "assets": { + "thumbnail": { + "title": "Thumbnail image", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://exmaple.com/thumb_small.jpeg", + "alternate": { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": true, + "href": "s3://example/thumb_small.jpeg" + } + } + } + }, + "links": [ + { + "rel": "self", + "href": "https://example.com/items/my_id" + } + ], + "collection": "landsat-c2l2alb-sr" +} diff --git a/autotest/ogr/ogr_geojson.py b/autotest/ogr/ogr_geojson.py index 61e25654c9a9..ee83ba27c291 100755 --- a/autotest/ogr/ogr_geojson.py +++ b/autotest/ogr/ogr_geojson.py @@ -5784,3 +5784,39 @@ def test_ogr_geojson_schema_override( assert ( gdal.GetLastErrorMsg().find(expected_warning) != -1 ), f"Warning {expected_warning} not found, got {gdal.GetLastErrorMsg()} instead" + + +############################################################################### +# Test FOREIGN_MEMBERS open option + + +@pytest.mark.parametrize( + "foreign_members_option", [None, "AUTO", "ALL", "NONE", "STAC"] +) +def test_ogr_geojson_foreign_members(foreign_members_option): + + open_options = {} + if foreign_members_option: + open_options["FOREIGN_MEMBERS"] = foreign_members_option + ds = gdal.OpenEx( + "data/geojson/stac_item.json", gdal.OF_VECTOR, open_options=open_options + ) + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + if foreign_members_option is None or foreign_members_option in ("AUTO", "STAC"): + assert lyr.GetLayerDefn().GetFieldCount() == 39 + assert f["stac_version"] == "1.0.0" + assert f["assets.thumbnail.title"] == "Thumbnail image" + assert json.loads(f["assets.thumbnail.alternate"]) == { + "s3": { + "storage:platform": "AWS", + "storage:requester_pays": True, + "href": "s3://example/thumb_small.jpeg", + } + } + elif foreign_members_option == "ALL": + assert lyr.GetLayerDefn().GetFieldCount() == 35 + assert f["stac_version"] == "1.0.0" + assert f["assets"] != "" + else: + assert lyr.GetLayerDefn().GetFieldCount() == 29 diff --git a/autotest/ogr/ogr_miramon_vector.py b/autotest/ogr/ogr_miramon_vector.py index 3cfebb9b3ba6..ecf4a647c9bb 100644 --- a/autotest/ogr/ogr_miramon_vector.py +++ b/autotest/ogr/ogr_miramon_vector.py @@ -1437,9 +1437,14 @@ def test_ogr_miramon_create_field_after_feature(tmp_path): def test_ogr_miramon_json_import_not_failing(tmp_vsimem): out_filename = str(tmp_vsimem / "out/json_layer_to_mm.pol") + src_ds = gdal.OpenEx( + "data/miramon_inputs/LT05_L2SP_038037_20120505_20200820_02_T1_ST_stac_minimal.json", + gdal.OF_VECTOR, + open_options=["FOREIGN_MEMBERS=NONE"], + ) gdal.VectorTranslate( out_filename, - "data/miramon_inputs/LT05_L2SP_038037_20120505_20200820_02_T1_ST_stac_minimal.json", + src_ds, format="MiraMonVector", ) diff --git a/doc/source/drivers/vector/geojson.rst b/doc/source/drivers/vector/geojson.rst index 4bc519abb7d6..dab99b5ec820 100644 --- a/doc/source/drivers/vector/geojson.rst +++ b/doc/source/drivers/vector/geojson.rst @@ -239,6 +239,27 @@ This driver supports the following open options: Can also be set with the :config:`OGR_GEOJSON_DATE_AS_STRING` configuration option. +- .. oo:: FOREIGN_MEMBERS + :choices: AUTO, ALL, NONE, STAC + :default: AUTO + :since: 3.11.0 + + Whether and how foreign members at the feature level should be processed + as OGR fields: + + - ``AUTO`` mode behaves like ``STAC`` mode if a ``stac_version`` member is found at + the Feature level, otherwise it behaves as ``NONE`` mode. + + - In ``ALL`` mode, all foreign members at the feature level are added. + Whether to recursively explore nested objects and produce flatten OGR attributes + or not is decided by the ``FLATTEN_NESTED_ATTRIBUTES`` open option. + + - In ``NONE`` mode, no foreign members at the feature level are added. + + - ``STAC`` mode (Spatio-Temporal Asset Catalog) behaves the same as ``ALL``, + except content under the ``assets`` member is by default flattened + as ``assets.{asset_name}.{asset_property}`` fields. + - .. oo:: OGR_SCHEMA :choices: | :since: 3.11.0 diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp index 123b83b7bef2..7fe16f97b864 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp @@ -1225,6 +1225,29 @@ void OGRGeoJSONDataSource::SetOptionsOnReader(GDALOpenInfo *poOpenInfo, poReader->SetDateAsString(CPLTestBool(CSLFetchNameValueDef( poOpenInfo->papszOpenOptions, "DATE_AS_STRING", CPLGetConfigOption("OGR_GEOJSON_DATE_AS_STRING", "NO")))); + + const char *pszForeignMembers = CSLFetchNameValueDef( + poOpenInfo->papszOpenOptions, "FOREIGN_MEMBERS", "AUTO"); + if (EQUAL(pszForeignMembers, "AUTO")) + { + poReader->SetForeignMemberProcessing( + OGRGeoJSONBaseReader::ForeignMemberProcessing::AUTO); + } + else if (EQUAL(pszForeignMembers, "ALL")) + { + poReader->SetForeignMemberProcessing( + OGRGeoJSONBaseReader::ForeignMemberProcessing::ALL); + } + else if (EQUAL(pszForeignMembers, "NONE")) + { + poReader->SetForeignMemberProcessing( + OGRGeoJSONBaseReader::ForeignMemberProcessing::NONE); + } + else if (EQUAL(pszForeignMembers, "STAC")) + { + poReader->SetForeignMemberProcessing( + OGRGeoJSONBaseReader::ForeignMemberProcessing::STAC); + } } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp index 93755c74f981..1b993844a311 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp @@ -709,6 +709,14 @@ void RegisterOGRGeoJSON() " " " " "