Skip to content

Commit 59916e2

Browse files
rouaultrouault
committed
Add (minimum) support for libarrow 18.0.0
Essentially to fix compiler warnings about missing cases in switch() and fix a deprecation warning. Support for DECIMAL32 and DECIMAL64 should work on the read side, but not actually tested. The ogrlayerarrow.cpp generic code isn't ready for them yet. No support on the write side to limit backwards compatibility issues.
1 parent ec578b5 commit 59916e2

File tree

3 files changed

+155
-2
lines changed

3 files changed

+155
-2
lines changed

‎ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,10 @@ inline bool OGRArrowLayer::IsHandledListOrMapType(
244244
itemTypeId == arrow::Type::HALF_FLOAT ||
245245
itemTypeId == arrow::Type::FLOAT ||
246246
itemTypeId == arrow::Type::DOUBLE ||
247+
#if ARROW_VERSION_MAJOR >= 18
248+
itemTypeId == arrow::Type::DECIMAL32 ||
249+
itemTypeId == arrow::Type::DECIMAL64 ||
250+
#endif
247251
itemTypeId == arrow::Type::DECIMAL128 ||
248252
itemTypeId == arrow::Type::DECIMAL256 ||
249253
itemTypeId == arrow::Type::STRING ||
@@ -427,6 +431,10 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR(
427431
// nanosecond accuracy
428432
break;
429433

434+
#if ARROW_VERSION_MAJOR >= 18
435+
case arrow::Type::DECIMAL32:
436+
case arrow::Type::DECIMAL64:
437+
#endif
430438
case arrow::Type::DECIMAL128:
431439
case arrow::Type::DECIMAL256:
432440
{
@@ -473,6 +481,10 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR(
473481
eSubType = OFSTFloat32;
474482
break;
475483
case arrow::Type::DOUBLE:
484+
#if ARROW_VERSION_MAJOR >= 18
485+
case arrow::Type::DECIMAL32:
486+
case arrow::Type::DECIMAL64:
487+
#endif
476488
case arrow::Type::DECIMAL128:
477489
case arrow::Type::DECIMAL256:
478490
eType = OFTRealList;
@@ -1279,6 +1291,23 @@ static void AddToArray(CPLJSONArray &oArray, const arrow::Array *array,
12791291
static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
12801292
break;
12811293
}
1294+
1295+
#if ARROW_VERSION_MAJOR >= 18
1296+
case arrow::Type::DECIMAL32:
1297+
{
1298+
oArray.Add(CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
1299+
->FormatValue(nIdx)
1300+
.c_str()));
1301+
break;
1302+
}
1303+
case arrow::Type::DECIMAL64:
1304+
{
1305+
oArray.Add(CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
1306+
->FormatValue(nIdx)
1307+
.c_str()));
1308+
break;
1309+
}
1310+
#endif
12821311
case arrow::Type::DECIMAL128:
12831312
{
12841313
oArray.Add(
@@ -1449,6 +1478,25 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey,
14491478
static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
14501479
break;
14511480
}
1481+
1482+
#if ARROW_VERSION_MAJOR >= 18
1483+
case arrow::Type::DECIMAL32:
1484+
{
1485+
oDict.Add(osKey,
1486+
CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
1487+
->FormatValue(nIdx)
1488+
.c_str()));
1489+
break;
1490+
}
1491+
case arrow::Type::DECIMAL64:
1492+
{
1493+
oDict.Add(osKey,
1494+
CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
1495+
->FormatValue(nIdx)
1496+
.c_str()));
1497+
break;
1498+
}
1499+
#endif
14521500
case arrow::Type::DECIMAL128:
14531501
{
14541502
oDict.Add(osKey,
@@ -1710,6 +1758,48 @@ static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray,
17101758
break;
17111759
}
17121760

1761+
#if ARROW_VERSION_MAJOR >= 18
1762+
case arrow::Type::DECIMAL32:
1763+
{
1764+
const auto values = std::static_pointer_cast<arrow::Decimal32Array>(
1765+
array->values());
1766+
const auto nIdxStart = array->value_offset(nIdxInArray);
1767+
const int nCount = array->value_length(nIdxInArray);
1768+
std::vector<double> aValues;
1769+
aValues.reserve(nCount);
1770+
for (int k = 0; k < nCount; k++)
1771+
{
1772+
if (values->IsNull(nIdxStart + k))
1773+
aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1774+
else
1775+
aValues.push_back(
1776+
CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1777+
}
1778+
poFeature->SetField(i, nCount, aValues.data());
1779+
break;
1780+
}
1781+
1782+
case arrow::Type::DECIMAL64:
1783+
{
1784+
const auto values = std::static_pointer_cast<arrow::Decimal64Array>(
1785+
array->values());
1786+
const auto nIdxStart = array->value_offset(nIdxInArray);
1787+
const int nCount = array->value_length(nIdxInArray);
1788+
std::vector<double> aValues;
1789+
aValues.reserve(nCount);
1790+
for (int k = 0; k < nCount; k++)
1791+
{
1792+
if (values->IsNull(nIdxStart + k))
1793+
aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1794+
else
1795+
aValues.push_back(
1796+
CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1797+
}
1798+
poFeature->SetField(i, nCount, aValues.data());
1799+
break;
1800+
}
1801+
#endif
1802+
17131803
case arrow::Type::DECIMAL128:
17141804
{
17151805
const auto values =
@@ -2313,6 +2403,26 @@ inline OGRFeature *OGRArrowLayer::ReadFeature(
23132403
break;
23142404
}
23152405

2406+
#if ARROW_VERSION_MAJOR >= 18
2407+
case arrow::Type::DECIMAL32:
2408+
{
2409+
const auto castArray =
2410+
static_cast<const arrow::Decimal32Array *>(array);
2411+
poFeature->SetField(
2412+
i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2413+
break;
2414+
}
2415+
2416+
case arrow::Type::DECIMAL64:
2417+
{
2418+
const auto castArray =
2419+
static_cast<const arrow::Decimal64Array *>(array);
2420+
poFeature->SetField(
2421+
i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2422+
break;
2423+
}
2424+
#endif
2425+
23162426
case arrow::Type::DECIMAL128:
23172427
{
23182428
const auto castArray =
@@ -3803,6 +3913,34 @@ inline bool OGRArrowLayer::SkipToNextFeatureDueToAttributeFilter() const
38033913
break;
38043914
}
38053915

3916+
#if ARROW_VERSION_MAJOR >= 18
3917+
case arrow::Type::DECIMAL32:
3918+
{
3919+
const auto castArray =
3920+
static_cast<const arrow::Decimal32Array *>(array);
3921+
if (!ConstraintEvaluator(
3922+
constraint,
3923+
CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
3924+
{
3925+
return true;
3926+
}
3927+
break;
3928+
}
3929+
3930+
case arrow::Type::DECIMAL64:
3931+
{
3932+
const auto castArray =
3933+
static_cast<const arrow::Decimal64Array *>(array);
3934+
if (!ConstraintEvaluator(
3935+
constraint,
3936+
CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
3937+
{
3938+
return true;
3939+
}
3940+
break;
3941+
}
3942+
#endif
3943+
38063944
case arrow::Type::DECIMAL128:
38073945
{
38083946
const auto castArray =

‎ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,19 @@ inline void OGRArrowWriterLayer::CreateSchemaCommon()
199199
{
200200
const int nPrecision = poFieldDefn->GetPrecision();
201201
if (nWidth != 0 && nPrecision != 0)
202-
dt = arrow::decimal(nWidth, nPrecision);
202+
{
203+
// Since arrow 18.0, we could use arrow::smallest_decimal()
204+
// to return the smallest representation (i.e. possibly
205+
// decimal32 and decimal64). But for now keep decimal128
206+
// as the minimum for backwards compatibility.
207+
// GetValueDecimal() and other functions in
208+
// ogrlayerarrow.cpp would have to be adapted for decimal32
209+
// and decimal64 compatibility.
210+
if (nWidth > 38)
211+
dt = arrow::decimal256(nWidth, nPrecision);
212+
else
213+
dt = arrow::decimal128(nWidth, nPrecision);
214+
}
203215
else if (eSubDT == OFSTFloat32)
204216
dt = arrow::float32();
205217
else

‎ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1404,6 +1404,10 @@ bool OGRParquetLayer::ReadNextBatch()
14041404
{
14051405
m_nIdxInBatch = 0;
14061406

1407+
const int nNumGroups = m_poArrowReader->num_row_groups();
1408+
if (nNumGroups == 0)
1409+
return false;
1410+
14071411
if (m_bSingleBatch)
14081412
{
14091413
CPLAssert(m_iRecordBatch == 0);
@@ -1456,7 +1460,6 @@ bool OGRParquetLayer::ReadNextBatch()
14561460
}
14571461
else
14581462
{
1459-
const int nNumGroups = m_poArrowReader->num_row_groups();
14601463
OGRField sMin;
14611464
OGRField sMax;
14621465
OGR_RawField_SetNull(&sMin);

0 commit comments

Comments
 (0)