Skip to content

Commit b73b8f5

Browse files
rouaultrouault
authored
Merge pull request #11113 from dbaston/cachemax-units
Use CPLParseMemorySize for GDAL_CACHEMAX and similar config options
2 parents 9ad3aff + cab4e4f commit b73b8f5

File tree

7 files changed

+125
-79
lines changed

7 files changed

+125
-79
lines changed

‎autotest/cpp/test_cpl.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1777,6 +1777,16 @@ TEST_F(test_cpl, CPLParseMemorySize)
17771777
EXPECT_GT(nValue, 100 * 1024 * 1024);
17781778
EXPECT_TRUE(bUnitSpecified);
17791779

1780+
result = CPLParseMemorySize("0", &nValue, &bUnitSpecified);
1781+
EXPECT_EQ(result, CE_None);
1782+
EXPECT_EQ(nValue, 0);
1783+
EXPECT_FALSE(bUnitSpecified);
1784+
1785+
result = CPLParseMemorySize("0MB", &nValue, &bUnitSpecified);
1786+
EXPECT_EQ(result, CE_None);
1787+
EXPECT_EQ(nValue, 0);
1788+
EXPECT_TRUE(bUnitSpecified);
1789+
17801790
result = CPLParseMemorySize(" 802 ", &nValue, &bUnitSpecified);
17811791
EXPECT_EQ(result, CE_None);
17821792
EXPECT_EQ(nValue, 802);

‎autotest/gcore/vsifile.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -237,16 +237,16 @@ def test_vsifile_4():
237237
# Test vsicache
238238

239239

240-
@pytest.mark.parametrize("cache_size", ("0", "65536", None))
241-
def test_vsifile_5(cache_size):
240+
@pytest.mark.parametrize("cache_size", ("0", "64kb", None))
241+
def test_vsifile_5(tmp_path, cache_size):
242242

243-
fp = gdal.VSIFOpenL("tmp/vsifile_5.bin", "wb")
243+
fp = gdal.VSIFOpenL(tmp_path / "vsifile_5.bin", "wb")
244244
ref_data = "".join(["%08X" % i for i in range(5 * 32768)])
245245
gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp)
246246
gdal.VSIFCloseL(fp)
247247

248248
with gdal.config_options({"VSI_CACHE": "YES", "VSI_CACHE_SIZE": cache_size}):
249-
fp = gdal.VSIFOpenL("tmp/vsifile_5.bin", "rb")
249+
fp = gdal.VSIFOpenL(tmp_path / "vsifile_5.bin", "rb")
250250

251251
gdal.VSIFSeekL(fp, 50000, 0)
252252
if gdal.VSIFTellL(fp) != 50000:
@@ -277,8 +277,6 @@ def test_vsifile_5(cache_size):
277277

278278
gdal.VSIFCloseL(fp)
279279

280-
gdal.Unlink("tmp/vsifile_5.bin")
281-
282280

283281
###############################################################################
284282
# Test vsicache an read errors (https://github.com/qgis/QGIS/issues/45293)

‎doc/source/user/configoptions.rst

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,10 @@ Performance and caching
222222
:program:`gdalwarp`.
223223
If its value is small (less than 100000), it is assumed to be measured in megabytes,
224224
otherwise in bytes. Alternatively, the value can be set to "X%" to mean X%
225-
of the usable physical RAM. Note that this value is only consulted the first
225+
of the usable physical RAM.
226+
Since GDAL 3.11, the value of :config:`GDAL_CACHEMAX` may specify the
227+
units directly (e.g., "500MB", "2GB").
228+
Note that this value is only consulted the first
226229
time the cache size is requested. To change this value programmatically
227230
during operation of the program it is better to use
228231
:cpp:func:`GDALSetCacheMax` (always in bytes) or or
@@ -318,6 +321,9 @@ Performance and caching
318321
Set the size of the VSI cache. Be wary of large values for
319322
``VSI_CACHE_SIZE`` when opening VRT datasources containing many source
320323
rasters, as this is a per-file cache.
324+
Since GDAL 3.11, the value of ``VSI_CACHE_SIZE`` may be specified using
325+
memory units (e.g., "25 MB").
326+
321327

322328
Driver management
323329
^^^^^^^^^^^^^^^^^
@@ -421,7 +427,7 @@ General options
421427
option.
422428

423429
- .. config:: CPL_VSIL_DEFLATE_CHUNK_SIZE
424-
:default: 1 M
430+
:default: 1M
425431

426432
- .. config:: GDAL_DISABLE_CPLLOCALEC
427433
:choices: YES, NO
@@ -610,7 +616,8 @@ Networking options
610616
:since: 2.3
611617

612618
Size of global least-recently-used (LRU) cache shared among all downloaded
613-
content.
619+
content. Value is assumed to represent bytes unless memory units are
620+
specified (since GDAL 3.11).
614621

615622
- .. config:: CPL_VSIL_CURL_USE_HEAD
616623
:choices: YES, NO
@@ -658,6 +665,9 @@ Networking options
658665
:choices: <bytes>
659666
:since: 2.3
660667

668+
Value is assumed to represent bytes unless memory units are
669+
specified (since GDAL 3.11).
670+
661671
- .. config:: GDAL_INGESTED_BYTES_AT_OPEN
662672
:since: 2.3
663673

‎gcore/gdalrasterblock.cpp

Lines changed: 19 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -207,11 +207,12 @@ int CPL_STDCALL GDALGetCacheMax()
207207
* Gets the maximum amount of memory available to the GDALRasterBlock
208208
* caching system for caching GDAL read/write imagery.
209209
*
210-
* The first type this function is called, it will read the GDAL_CACHEMAX
210+
* The first time this function is called, it will read the GDAL_CACHEMAX
211211
* configuration option to initialize the maximum cache memory.
212212
* Starting with GDAL 2.1, the value can be expressed as x% of the usable
213-
* physical RAM (which may potentially be used by other processes). Otherwise
214-
* it is expected to be a value in MB.
213+
* physical RAM (which may potentially be used by other processes). Starting
214+
* with GDAL 3.11, the value can include units of memory. If not units are
215+
* provided the value is assumed to be in MB.
215216
*
216217
* @return maximum in bytes.
217218
*
@@ -232,60 +233,29 @@ GIntBig CPL_STDCALL GDALGetCacheMax64()
232233
CPLTestBool(CPLGetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "NO"));
233234

234235
const char *pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX", "5%");
235-
236236
GIntBig nNewCacheMax;
237-
if (strchr(pszCacheMax, '%') != nullptr)
237+
bool bUnitSpecified = false;
238+
239+
if (CPLParseMemorySize(pszCacheMax, &nNewCacheMax,
240+
&bUnitSpecified) != CE_None)
238241
{
239-
GIntBig nUsablePhysicalRAM = CPLGetUsablePhysicalRAM();
240-
if (nUsablePhysicalRAM > 0)
242+
CPLError(CE_Failure, CPLE_NotSupported,
243+
"Invalid value for GDAL_CACHEMAX. "
244+
"Using default value.");
245+
if (CPLParseMemorySize("5%", &nNewCacheMax, &bUnitSpecified) !=
246+
CE_None)
241247
{
242-
// For some reason, coverity pretends that this will overflow.
243-
// "Multiply operation overflows on operands
244-
// static_cast<double>( nUsablePhysicalRAM ) and
245-
// CPLAtof(pszCacheMax). Example values for operands: CPLAtof(
246-
// pszCacheMax ) = 2251799813685248,
247-
// static_cast<double>(nUsablePhysicalRAM) =
248-
// -9223372036854775808." coverity[overflow,tainted_data]
249-
double dfCacheMax =
250-
static_cast<double>(nUsablePhysicalRAM) *
251-
CPLAtof(pszCacheMax) / 100.0;
252-
if (dfCacheMax >= 0 && dfCacheMax < 1e15)
253-
nNewCacheMax = static_cast<GIntBig>(dfCacheMax);
254-
else
255-
nNewCacheMax = nCacheMax;
256-
}
257-
else
258-
{
259-
CPLDebug("GDAL", "Cannot determine usable physical RAM.");
248+
// This means that usable physical RAM could not be determined.
260249
nNewCacheMax = nCacheMax;
261250
}
262251
}
263-
else
252+
253+
if (!bUnitSpecified && nNewCacheMax < 100000)
264254
{
265-
nNewCacheMax = CPLAtoGIntBig(pszCacheMax);
266-
if (nNewCacheMax < 100000)
267-
{
268-
if (nNewCacheMax < 0)
269-
{
270-
CPLError(CE_Failure, CPLE_NotSupported,
271-
"Invalid value for GDAL_CACHEMAX. "
272-
"Using default value.");
273-
GIntBig nUsablePhysicalRAM = CPLGetUsablePhysicalRAM();
274-
if (nUsablePhysicalRAM)
275-
nNewCacheMax = nUsablePhysicalRAM / 20;
276-
else
277-
{
278-
CPLDebug("GDAL",
279-
"Cannot determine usable physical RAM.");
280-
nNewCacheMax = nCacheMax;
281-
}
282-
}
283-
else
284-
{
285-
nNewCacheMax *= 1024 * 1024;
286-
}
287-
}
255+
// Assume MB
256+
nNewCacheMax *= (1024 * 1024);
288257
}
258+
289259
nCacheMax = nNewCacheMax;
290260
CPLDebug("GDAL", "GDAL_CACHEMAX = " CPL_FRMT_GIB " MB",
291261
nCacheMax / (1024 * 1024));

‎port/cpl_string.cpp

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1779,10 +1779,10 @@ CPLErr CPLParseMemorySize(const char *pszValue, GIntBig *pnValue,
17791779
return CE_Failure;
17801780
}
17811781

1782-
if (value <= 0 || !std::isfinite(value))
1782+
if (value < 0 || !std::isfinite(value))
17831783
{
17841784
CPLError(CE_Failure, CPLE_IllegalArg,
1785-
"Memory size must be a positive number.");
1785+
"Memory size must be a positive number or zero.");
17861786
return CE_Failure;
17871787
}
17881788

@@ -1800,6 +1800,12 @@ CPLErr CPLParseMemorySize(const char *pszValue, GIntBig *pnValue,
18001800
return CE_Failure;
18011801
}
18021802
auto bytes = CPLGetUsablePhysicalRAM();
1803+
if (bytes == 0)
1804+
{
1805+
CPLError(CE_Failure, CPLE_NotSupported,
1806+
"Cannot determine usable physical RAM");
1807+
return CE_Failure;
1808+
}
18031809
value *= static_cast<double>(bytes / 100);
18041810
unit = c;
18051811
}
@@ -1842,7 +1848,10 @@ CPLErr CPLParseMemorySize(const char *pszValue, GIntBig *pnValue,
18421848
}
18431849

18441850
*pnValue = static_cast<GIntBig>(value);
1845-
*pbUnitSpecified = (unit != nullptr);
1851+
if (pbUnitSpecified)
1852+
{
1853+
*pbUnitSpecified = (unit != nullptr);
1854+
}
18461855
return CE_None;
18471856
}
18481857

‎port/cpl_vsil_cache.cpp

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,13 +112,30 @@ class VSICachedFile final : public VSIVirtualHandle
112112

113113
static size_t GetCacheMax(size_t nCacheSize)
114114
{
115-
return nCacheSize ? nCacheSize
116-
: static_cast<size_t>(std::min(
117-
static_cast<GUIntBig>(
118-
std::numeric_limits<size_t>::max() / 2),
119-
CPLScanUIntBig(CPLGetConfigOption("VSI_CACHE_SIZE",
120-
"25000000"),
121-
40)));
115+
if (nCacheSize)
116+
{
117+
return nCacheSize;
118+
}
119+
120+
const char *pszCacheSize = CPLGetConfigOption("VSI_CACHE_SIZE", "25000000");
121+
GIntBig nMemorySize;
122+
bool bUnitSpecified;
123+
if (CPLParseMemorySize(pszCacheSize, &nMemorySize, &bUnitSpecified) !=
124+
CE_None)
125+
{
126+
CPLError(
127+
CE_Failure, CPLE_IllegalArg,
128+
"Failed to parse value of VSI_CACHE_SIZE. Using default of 25MB");
129+
nMemorySize = 25000000;
130+
}
131+
else if (static_cast<size_t>(nMemorySize) >
132+
std::numeric_limits<size_t>::max() / 2)
133+
{
134+
nMemorySize =
135+
static_cast<GIntBig>(std::numeric_limits<size_t>::max() / 2);
136+
}
137+
138+
return static_cast<size_t>(nMemorySize);
122139
}
123140

124141
/************************************************************************/

‎port/cpl_vsil_curl.cpp

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -137,31 +137,63 @@ static void VSICURLReadGlobalEnvVariables()
137137
Initializer()
138138
{
139139
constexpr int DOWNLOAD_CHUNK_SIZE_DEFAULT = 16384;
140+
const char *pszChunkSize =
141+
CPLGetConfigOption("CPL_VSIL_CURL_CHUNK_SIZE", nullptr);
142+
GIntBig nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT;
143+
144+
if (pszChunkSize)
145+
{
146+
if (CPLParseMemorySize(pszChunkSize, &nChunkSize, nullptr) !=
147+
CE_None)
148+
{
149+
CPLError(
150+
CE_Warning, CPLE_AppDefined,
151+
"Could not parse value for CPL_VSIL_CURL_CHUNK_SIZE. "
152+
"Using default value of %d instead.",
153+
DOWNLOAD_CHUNK_SIZE_DEFAULT);
154+
}
155+
}
140156

141-
DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = atoi(CPLGetConfigOption(
142-
"CPL_VSIL_CURL_CHUNK_SIZE",
143-
CPLSPrintf("%d", DOWNLOAD_CHUNK_SIZE_DEFAULT)));
144157
constexpr int MIN_CHUNK_SIZE = 1024;
145158
constexpr int MAX_CHUNK_SIZE = 10 * 1024 * 1024;
146-
if (DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY < MIN_CHUNK_SIZE ||
147-
DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY > MAX_CHUNK_SIZE)
159+
if (nChunkSize < MIN_CHUNK_SIZE || nChunkSize > MAX_CHUNK_SIZE)
148160
{
149-
DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY =
150-
DOWNLOAD_CHUNK_SIZE_DEFAULT;
161+
nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT;
151162
CPLError(CE_Warning, CPLE_AppDefined,
152163
"Invalid value for CPL_VSIL_CURL_CHUNK_SIZE. "
153164
"Allowed range is [%d, %d]. "
154165
"Using CPL_VSIL_CURL_CHUNK_SIZE=%d instead",
155166
MIN_CHUNK_SIZE, MAX_CHUNK_SIZE,
156-
DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY);
167+
DOWNLOAD_CHUNK_SIZE_DEFAULT);
157168
}
169+
DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY =
170+
static_cast<int>(nChunkSize);
158171

159172
constexpr int N_MAX_REGIONS_DEFAULT = 1000;
160173
constexpr int CACHE_SIZE_DEFAULT =
161174
N_MAX_REGIONS_DEFAULT * DOWNLOAD_CHUNK_SIZE_DEFAULT;
162-
GIntBig nCacheSize = CPLAtoGIntBig(
163-
CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE",
164-
CPLSPrintf("%d", CACHE_SIZE_DEFAULT)));
175+
176+
const char *pszCacheSize =
177+
CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE", nullptr);
178+
GIntBig nCacheSize = CACHE_SIZE_DEFAULT;
179+
180+
if (pszCacheSize)
181+
{
182+
if (CPLParseMemorySize(pszCacheSize, &nCacheSize, nullptr) !=
183+
CE_None)
184+
{
185+
CPLError(
186+
CE_Warning, CPLE_AppDefined,
187+
"Could not parse value for CPL_VSIL_CURL_CACHE_SIZE. "
188+
"Using default value of " CPL_FRMT_GIB " instead.",
189+
nCacheSize);
190+
}
191+
}
192+
else
193+
{
194+
nCacheSize = CACHE_SIZE_DEFAULT;
195+
}
196+
165197
const auto nMaxRAM = CPLGetUsablePhysicalRAM();
166198
const auto nMinVal = DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY;
167199
auto nMaxVal = static_cast<GIntBig>(INT_MAX) *

0 commit comments

Comments
 (0)