Skip to content

Commit a12a5f6

Browse files
authored
VRT Pixel Functions: Add function to evaluate arbitrary expression (OSGeo#11209)
Adds a C++ pixel function called "expression" that can evaluate an arbitrary expression (or indeed, a mini-program) using either: - the [exprtk library](https://www.partow.net/programming/exprtk/index.html). This is a single MIT-licensed header, that appears to be quite widely used. But which causes a significant increase in size of libgdal (8 MB) - or [muparser](https://github.com/beltoforion/muparser), that supports a reasonable variety of functions including a ternary conditional operator muparser is the deault when the "dialect" is not specified. An example VRT that is allowed is: ``` <VRTDataset rasterXSize="1" rasterYSize="1"> <VRTRasterBand dataType="Float64" band="1" subClass="VRTDerivedRasterBand"> <PixelFunctionType>expression</PixelFunctionType> <PixelFunctionArguments expression="(NIR-R)/(NIR+R)" /> <SimpleSource name="NIR"> <SourceFilename relativeToVRT="0">source_0.tif</SourceFilename> <SourceBand>1</SourceBand> </SimpleSource> <SimpleSource name="R"> <SourceFilename relativeToVRT="0">source_1.tif</SourceFilename> <SourceBand>1</SourceBand> </SimpleSource> </VRTRasterBand> </VRTDataset> ```
1 parent 07a271a commit a12a5f6

29 files changed

+2272
-223
lines changed

.github/workflows/alpine/Dockerfile.ci

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ RUN apk add \
4141
lz4-dev \
4242
make \
4343
mariadb-connector-c-dev \
44+
muparser-dev \
4445
netcdf-dev \
4546
odbc-cpp-wrapper-dev \
4647
ogdi-dev \

.github/workflows/cmake_builds.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ jobs:
328328
base-devel git mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache
329329
mingw-w64-x86_64-pcre mingw-w64-x86_64-xerces-c mingw-w64-x86_64-zstd mingw-w64-x86_64-libarchive
330330
mingw-w64-x86_64-geos mingw-w64-x86_64-libspatialite mingw-w64-x86_64-proj
331-
mingw-w64-x86_64-cgal mingw-w64-x86_64-libfreexl mingw-w64-x86_64-hdf5 mingw-w64-x86_64-netcdf mingw-w64-x86_64-poppler mingw-w64-x86_64-podofo mingw-w64-x86_64-postgresql
331+
mingw-w64-x86_64-cgal mingw-w64-x86_64-libfreexl mingw-w64-x86_64-hdf5 mingw-w64-x86_64-muparser mingw-w64-x86_64-netcdf mingw-w64-x86_64-poppler mingw-w64-x86_64-podofo mingw-w64-x86_64-postgresql
332332
mingw-w64-x86_64-libgeotiff mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw-w64-x86_64-openjpeg2
333333
mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil mingw-w64-x86_64-blosc mingw-w64-x86_64-libavif
334334
- name: Setup cache
@@ -434,7 +434,7 @@ jobs:
434434
cfitsio freexl geotiff libjpeg-turbo libpq libspatialite libwebp-base pcre pcre2 postgresql \
435435
sqlite tiledb zstd cryptopp cgal doxygen librttopo openssl liblzma-devel \
436436
openjdk ant qhull armadillo blas blas-devel libblas libcblas liblapack liblapacke blosc libarchive \
437-
libarrow pyarrow libaec libheif libavif cmake fsspec
437+
libarrow pyarrow libaec libheif libavif muparser cmake fsspec
438438
- name: Check CMake version
439439
shell: bash -l {0}
440440
run: |

.github/workflows/fedora_rawhide/Dockerfile.ci

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ RUN dnf install -y clang make diffutils ccache cmake \
1919
mdbtools-devel mdbtools-odbc unixODBC-devel \
2020
armadillo-devel qhull-devel \
2121
hdf-devel hdf5-devel netcdf-devel \
22+
muParser-devel \
2223
libpq-devel \
2324
libavif-devel \
2425
python3-setuptools python3-pip python3-devel python3-lxml swig \

.github/workflows/ubuntu_20.04/Dockerfile.ci

+6
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ RUN apt-get update -y \
4949
liblz4-dev \
5050
liblzma-dev \
5151
libmono-system-drawing4.0-cil \
52+
libmuparser-dev \
5253
libmysqlclient-dev \
5354
libnetcdf-dev \
5455
libogdi-dev \
@@ -274,6 +275,11 @@ RUN if test "${OPENDRIVE_VERSION}" != ""; then ( \
274275
&& rm -rf libOpenDRIVE-${OPENDRIVE_VERSION} \
275276
); fi
276277

278+
# Install exprtk
279+
RUN wget -q https://www.partow.net/downloads/exprtk.zip && \
280+
unzip -j -d /usr/local/include exprtk.zip exprtk/exprtk.hpp && \
281+
rm exprtk.zip
282+
277283
RUN ldconfig
278284

279285
COPY requirements.txt /tmp/

.github/workflows/ubuntu_20.04/build.sh

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ cmake "${GDAL_SOURCE_DIR:=..}" \
1515
-DCMAKE_INSTALL_PREFIX=/tmp/install-gdal \
1616
-DGDAL_USE_TIFF_INTERNAL=OFF \
1717
-DGDAL_USE_GEOTIFF_INTERNAL=OFF \
18+
-DGDAL_USE_EXPRTK=ON \
1819
-DECW_ROOT=/opt/libecwj2-3.3 \
1920
-DMRSID_ROOT=/usr/local \
2021
-DFileGDB_ROOT=/usr/local/FileGDB_API \

.github/workflows/ubuntu_24.04/Dockerfile.ci

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ RUN apt-get update && \
3737
liblcms2-2 \
3838
liblz4-dev \
3939
liblzma-dev \
40+
libmuparser-dev \
4041
libmysqlclient-dev \
4142
libnetcdf-dev \
4243
libogdi-dev \

.pre-commit-config.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ repos:
4242
autotest/cpp/data/|
4343
autotest/gdrivers/data/|
4444
swig/|
45+
third_party/exprtk/|
4546
third_party/fast_float/|
47+
third_party/muparser/|
4648
third_party/LercLib/|
4749
autotest/ogr/data/|
4850
alg/internal_libqhull/|
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<VRTDataset rasterXSize="20" rasterYSize="20">
2+
<VRTRasterBand dataType="Float64" band="1" subClass="VRTDerivedRasterBand">
3+
<PixelFunctionType>expression</PixelFunctionType>
4+
<PixelFunctionArguments expression="B1 + 5" dialect="muparser"/>
5+
<ArraySource>
6+
<Array name="test">
7+
<DataType>Float64</DataType>
8+
<Dimension name="Y" size="20"/>
9+
<Dimension name="X" size="20"/>
10+
<ConstantValue>10</ConstantValue>
11+
</Array>
12+
</ArraySource>
13+
</VRTRasterBand>
14+
</VRTDataset>

autotest/gdrivers/vrtderived.py

+219
Original file line numberDiff line numberDiff line change
@@ -1055,6 +1055,225 @@ def identity(in_ar, out_ar, *args, **kwargs):
10551055
assert vrt_ds.GetRasterBand(1).DataType == dtype
10561056

10571057

1058+
###############################################################################
1059+
# Test arbitrary expression pixel functions
1060+
1061+
1062+
def vrt_expression_xml(tmpdir, expression, dialect, sources):
1063+
1064+
drv = gdal.GetDriverByName("GTiff")
1065+
1066+
nx = 1
1067+
ny = 1
1068+
1069+
expression = expression.replace("<", "&lt;").replace(">", "&gt;")
1070+
1071+
xml = f"""<VRTDataset rasterXSize="{nx}" rasterYSize="{ny}">
1072+
<VRTRasterBand dataType="Float64" band="1" subClass="VRTDerivedRasterBand">
1073+
<PixelFunctionType>expression</PixelFunctionType>
1074+
<PixelFunctionArguments expression="{expression}" dialect="{dialect}" />"""
1075+
1076+
for i, source in enumerate(sources):
1077+
if type(source) is tuple:
1078+
source_name, source_value = source
1079+
else:
1080+
source_name = ""
1081+
source_value = source
1082+
1083+
src_fname = tmpdir / f"source_{i}.tif"
1084+
1085+
with drv.Create(src_fname, 1, 1, 1, gdal.GDT_Float64) as ds:
1086+
ds.GetRasterBand(1).Fill(source_value)
1087+
1088+
xml += f"""<SimpleSource name="{source_name}">
1089+
<SourceFilename relativeToVRT="0">{src_fname}</SourceFilename>
1090+
<SourceBand>1</SourceBand>
1091+
</SimpleSource>"""
1092+
1093+
xml += "</VRTRasterBand></VRTDataset>"
1094+
1095+
return xml
1096+
1097+
1098+
@pytest.mark.parametrize(
1099+
"expression,sources,result,dialects",
1100+
[
1101+
pytest.param("A", [("A", 77)], 77, None, id="identity"),
1102+
pytest.param(
1103+
"(NIR-R)/(NIR+R)",
1104+
[("NIR", 77), ("R", 63)],
1105+
(77 - 63) / (77 + 63),
1106+
None,
1107+
id="simple expression",
1108+
),
1109+
pytest.param(
1110+
"if (A > B) 1.5*C ; else A",
1111+
[("A", 77), ("B", 63), ("C", 18)],
1112+
27,
1113+
["exprtk"],
1114+
id="exprtk conditional (explicit)",
1115+
),
1116+
pytest.param(
1117+
"(A > B) ? 1.5*C : A",
1118+
[("A", 77), ("B", 63), ("C", 18)],
1119+
27,
1120+
["muparser"],
1121+
id="muparser conditional (explicit)",
1122+
),
1123+
pytest.param(
1124+
"(A > B)*(1.5*C) + (A <= B)*(A)",
1125+
[("A", 77), ("B", 63), ("C", 18)],
1126+
27,
1127+
None,
1128+
id="conditional (implicit)",
1129+
),
1130+
pytest.param(
1131+
"B2 * PopDensity",
1132+
[("PopDensity", 3), ("", 7)],
1133+
21,
1134+
None,
1135+
id="implicit source name",
1136+
),
1137+
pytest.param(
1138+
"B1 / sum(BANDS)",
1139+
[("", 3), ("", 5), ("", 31)],
1140+
3 / (3 + 5 + 31),
1141+
None,
1142+
id="use of BANDS variable",
1143+
),
1144+
pytest.param(
1145+
"B1 / sum(B2, B3) ",
1146+
[("", 3), ("", 5), ("", 31)],
1147+
3 / (5 + 31),
1148+
None,
1149+
id="aggregate specified inputs",
1150+
),
1151+
pytest.param(
1152+
"var q[2] := {B2, B3}; B1 * q",
1153+
[("", 3), ("", 5), ("", 31)],
1154+
15, # First value in returned vector. This behavior doesn't seem desirable
1155+
# but I haven't figured out how to detect a vector return.
1156+
["exprtk"],
1157+
id="return vector",
1158+
),
1159+
pytest.param(
1160+
"B1 + B2 + B3",
1161+
(5, 9, float("nan")),
1162+
float("nan"),
1163+
None,
1164+
id="nan propagated via arithmetic",
1165+
),
1166+
pytest.param(
1167+
"if (B3) B1 ; else B2",
1168+
(5, 9, float("nan")),
1169+
5,
1170+
["exprtk"],
1171+
id="exprtk nan = truth in conditional?",
1172+
),
1173+
pytest.param(
1174+
"B3 ? B1 : B2",
1175+
(5, 9, float("nan")),
1176+
5,
1177+
["muparser"],
1178+
id="muparser nan = truth in conditional?",
1179+
),
1180+
pytest.param(
1181+
"if (B3 > 0) B1 ; else B2",
1182+
(5, 9, float("nan")),
1183+
9,
1184+
["exprtk"],
1185+
id="exprtk nan comparison is false in conditional",
1186+
),
1187+
pytest.param(
1188+
"(B3 > 0) ? B1 : B2",
1189+
(5, 9, float("nan")),
1190+
9,
1191+
["muparser"],
1192+
id="muparser nan comparison is false in conditional",
1193+
),
1194+
pytest.param(
1195+
"if (B1 > 5) B1",
1196+
(1,),
1197+
float("nan"),
1198+
["exprtk"],
1199+
id="expression returns nodata",
1200+
),
1201+
],
1202+
)
1203+
@pytest.mark.parametrize("dialect", ("exprtk", "muparser"))
1204+
def test_vrt_pixelfn_expression(
1205+
tmp_vsimem, expression, sources, result, dialect, dialects
1206+
):
1207+
pytest.importorskip("numpy")
1208+
1209+
if not gdaltest.gdal_has_vrt_expression_dialect(dialect):
1210+
pytest.skip(f"Expression dialect {dialect} is not available")
1211+
1212+
if dialects and dialect not in dialects:
1213+
pytest.skip(f"Expression not supported for dialect {dialect}")
1214+
1215+
xml = vrt_expression_xml(tmp_vsimem, expression, dialect, sources)
1216+
1217+
with gdal.Open(xml) as ds:
1218+
assert pytest.approx(ds.ReadAsArray()[0][0], nan_ok=True) == result
1219+
1220+
1221+
@pytest.mark.parametrize(
1222+
"expression,sources,dialect,exception",
1223+
[
1224+
pytest.param(
1225+
"A*B + C",
1226+
[("A", 77), ("B", 63)],
1227+
"exprtk",
1228+
"Undefined symbol",
1229+
id="exprtk undefined variable",
1230+
),
1231+
pytest.param(
1232+
"A*B + C",
1233+
[("A", 77), ("B", 63)],
1234+
"muparser",
1235+
"Unexpected token",
1236+
id="muparser undefined variable",
1237+
),
1238+
pytest.param(
1239+
"(".join(["asin", "sin", "acos", "cos"] * 100) + "(X" + 100 * 4 * ")",
1240+
[("X", 0.5)],
1241+
"exprtk",
1242+
"exceeds maximum allowed stack depth",
1243+
id="expression is too complex",
1244+
),
1245+
pytest.param(
1246+
" ".join(["sin(x) + cos(x)"] * 10000),
1247+
[("x", 0.5)],
1248+
"exprtk",
1249+
"exceeds maximum of 100000 set by GDAL_EXPRTK_MAX_EXPRESSION_LENGTH",
1250+
id="expression is too long",
1251+
),
1252+
],
1253+
)
1254+
def test_vrt_pixelfn_expression_invalid(
1255+
tmp_vsimem, expression, sources, dialect, exception
1256+
):
1257+
pytest.importorskip("numpy")
1258+
1259+
if not gdaltest.gdal_has_vrt_expression_dialect(dialect):
1260+
pytest.skip(f"Expression dialect {dialect} is not available")
1261+
1262+
messages = []
1263+
1264+
def handle(ecls, ecode, emsg):
1265+
messages.append(emsg)
1266+
1267+
xml = vrt_expression_xml(tmp_vsimem, expression, dialect, sources)
1268+
1269+
with gdaltest.error_handler(handle):
1270+
ds = gdal.Open(xml)
1271+
if ds:
1272+
assert ds.ReadAsArray() is None
1273+
1274+
assert exception in "".join(messages)
1275+
1276+
10581277
###############################################################################
10591278
# Cleanup.
10601279

0 commit comments

Comments
 (0)