Skip to content

Commit 5e90f23

Browse files
25491415192549141519
authoredAug 20, 2024
* Fix bug(oceanbase#2017): oblogminer incorrectly labeled LOB type NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug. * Enhancement(oceanbase#2017): oblogminer incorrectly labeled LOB type NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug. * Enhancement(oceanbase#2017): oblogminer incorrectly labeled NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug. * Enhancement(oceanbase#2017): oblogminer incorrectly labeled NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug. * Enhancement(oceanbase#2017): oblogminer incorrectly labeled NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug. * Enhancement(oceanbase#2017): oblogminer incorrectly labeled NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug. * Enhancement(oceanbase#2017): oblogminer incorrectly labeled NULL values from real logs as potentially inaccurate. Update oblogminer unit tests to adapt to the fixed bug.Check all the format
1 parent c45c83e commit 5e90f23

File tree

2 files changed

+245
-72
lines changed

2 files changed

+245
-72
lines changed
 

‎src/logservice/logminer/ob_log_miner_record.cpp

+73-71
Original file line numberDiff line numberDiff line change
@@ -441,24 +441,25 @@ int ObLogMinerRecord::build_dml_stmt_(ICDCRecord &cdc_rec)
441441
binlogBuf *new_cols = cdc_rec.newCols(new_col_cnt);
442442
binlogBuf *old_cols = cdc_rec.oldCols(old_col_cnt);
443443
ITableMeta *tbl_meta = cdc_rec.getTableMeta();
444-
// When updating or deleting records with lob type,
445-
// the null value of the lob type column may be incorrect
446-
// due to the limitations of obcdc.
447-
bool has_lob_null = false;
448-
// xmltype and sdo_geometry type don't support compare operation.
444+
// When updating or deleting records with value null,
445+
// the null value of the column may be incorrect
446+
// due to the limitations of obcdc and the minimal mode.
447+
// Currently, it only indicates that obcdc has mistakenly identified a NULL value.
448+
bool has_unreliable_null = false;
449+
// xmltype and sdo_geometry type don't support compare operation.
449450
bool has_unsupport_type_compare = false;
450451
if (OB_SUCC(ret)) {
451452
switch(record_type_) {
452-
// Insert records with lob type is accurate. obcdc will output all value of lob type.
453+
// Insert records with null value is accurate. obcdc will output all value accurately.
453454
case EINSERT: {
454455
if (OB_FAIL(build_insert_stmt_(redo_stmt_, new_cols, new_col_cnt, tbl_meta))) {
455456
LOG_ERROR("build insert redo stmt failed", KPC(this));
456457
} else {
457458
if (OB_FAIL(build_delete_stmt_(undo_stmt_, new_cols, new_col_cnt,
458-
tbl_meta, has_lob_null, has_unsupport_type_compare))) {
459+
tbl_meta, has_unreliable_null, has_unsupport_type_compare))) {
459460
LOG_ERROR("build insert undo stmt failed", KPC(this));
460461
} else {
461-
// ignore has_lob_null
462+
// ignore has_unreliable_null
462463
if (has_unsupport_type_compare) {
463464
APPEND_STMT(undo_stmt_, "/* POTENTIALLY INACCURATE */");
464465
}
@@ -467,51 +468,51 @@ int ObLogMinerRecord::build_dml_stmt_(ICDCRecord &cdc_rec)
467468
break;
468469
}
469470

470-
// Update records with lob type maybe inaccurate,
471+
// Update records with null value maybe inaccurate,
471472
// if NULL value appears in the pre/post mirror, the NULL may be incorrect.
472473
case EUPDATE: {
473474
if (OB_FAIL(build_update_stmt_(redo_stmt_, new_cols, new_col_cnt, old_cols,
474-
old_col_cnt, tbl_meta, has_lob_null, has_unsupport_type_compare))) {
475+
old_col_cnt, tbl_meta, has_unreliable_null, has_unsupport_type_compare))) {
475476
LOG_ERROR("build update redo stmt failed", KPC(this));
476477
} else {
477-
if (has_lob_null || has_unsupport_type_compare) {
478+
if (has_unreliable_null || has_unsupport_type_compare) {
478479
APPEND_STMT(redo_stmt_, "/* POTENTIALLY INACCURATE */");
479-
has_lob_null = false;
480+
has_unreliable_null = false;
480481
has_unsupport_type_compare = false;
481482
}
482483
}
483484
if (OB_SUCC(ret)) {
484485
if (OB_FAIL(build_update_stmt_(undo_stmt_, old_cols, old_col_cnt, new_cols,
485-
new_col_cnt, tbl_meta, has_lob_null, has_unsupport_type_compare))) {
486+
new_col_cnt, tbl_meta, has_unreliable_null, has_unsupport_type_compare))) {
486487
LOG_ERROR("build update undo stmt failed", KPC(this));
487488
} else {
488-
if (has_lob_null || has_unsupport_type_compare) {
489+
if (has_unreliable_null || has_unsupport_type_compare) {
489490
APPEND_STMT(undo_stmt_, "/* POTENTIALLY INACCURATE */");
490491
}
491492
}
492493
}
493494
break;
494495
}
495496

496-
// Delete records with lob type maybe inaccurate,
497+
// Delete records with null value maybe inaccurate,
497498
// if NULL value appears in the pre mirror, the NULL may be incorrect.
498499
case EDELETE: {
499500
if (OB_FAIL(build_delete_stmt_(redo_stmt_, old_cols, old_col_cnt,
500-
tbl_meta, has_lob_null, has_unsupport_type_compare))) {
501+
tbl_meta, has_unreliable_null, has_unsupport_type_compare))) {
501502
LOG_ERROR("build delete redo stmt failed", KPC(this));
502503
} else {
503-
if (has_lob_null || has_unsupport_type_compare) {
504+
if (has_unreliable_null || has_unsupport_type_compare) {
504505
APPEND_STMT(redo_stmt_, "/* POTENTIALLY INACCURATE */");
505-
has_lob_null = false;
506+
has_unreliable_null = false;
506507
has_unsupport_type_compare = false;
507508
}
508509
}
509510
if (OB_SUCC(ret)) {
510511
if (OB_FAIL(build_insert_stmt_(undo_stmt_, old_cols,
511-
old_col_cnt, tbl_meta, has_lob_null))) {
512+
old_col_cnt, tbl_meta, has_unreliable_null))) {
512513
LOG_ERROR("build delete undo stmt failed", KPC(this));
513514
} else {
514-
if (has_lob_null) {
515+
if (has_unreliable_null) {
515516
APPEND_STMT(undo_stmt_, "/* POTENTIALLY INACCURATE */");
516517
}
517518
}
@@ -536,9 +537,9 @@ int ObLogMinerRecord::build_insert_stmt_(ObStringBuffer &stmt,
536537
ITableMeta *tbl_meta)
537538
{
538539
int ret = OB_SUCCESS;
539-
// ignore has_lob_null
540-
bool has_lob_null = false;
541-
if (OB_FAIL(build_insert_stmt_(stmt, new_cols, new_col_cnt, tbl_meta, has_lob_null))) {
540+
// ignore has_unreliable_null
541+
bool has_unreliable_null = false;
542+
if (OB_FAIL(build_insert_stmt_(stmt, new_cols, new_col_cnt, tbl_meta, has_unreliable_null))) {
542543
LOG_ERROR("build insert stmt failed", KPC(this));
543544
}
544545
return ret;
@@ -547,7 +548,7 @@ int ObLogMinerRecord::build_insert_stmt_(ObStringBuffer &stmt,
547548
binlogBuf *new_cols,
548549
const unsigned int new_col_cnt,
549550
ITableMeta *tbl_meta,
550-
bool &has_lob_null)
551+
bool &has_unreliable_null)
551552
{
552553
int ret = OB_SUCCESS;
553554
if (IS_NOT_INIT) {
@@ -594,8 +595,8 @@ int ObLogMinerRecord::build_insert_stmt_(ObStringBuffer &stmt,
594595
"col_idx", i);
595596
}
596597
if (OB_SUCC(ret)) {
597-
if (is_lob_type_(col_meta) && nullptr == new_cols[i].buf) {
598-
has_lob_null = true;
598+
if (nullptr == new_cols[i].buf && new_cols[i].m_origin == VALUE_ORIGIN::PADDING) {
599+
has_unreliable_null = true;
599600
}
600601
}
601602
}
@@ -615,8 +616,8 @@ int ObLogMinerRecord::build_update_stmt_(ObStringBuffer &stmt,
615616
binlogBuf *old_cols,
616617
const unsigned int old_col_cnt,
617618
ITableMeta *tbl_meta,
618-
bool &has_lob_null,
619-
bool &has_unsupport_type_compare)
619+
bool &has_unreliable_null,
620+
bool &has_unsupport_type_compare)
620621
{
621622
int ret = OB_SUCCESS;
622623
if (IS_NOT_INIT) {
@@ -653,15 +654,16 @@ int ObLogMinerRecord::build_update_stmt_(ObStringBuffer &stmt,
653654
"col_idx", i);
654655
}
655656
if (OB_SUCC(ret)) {
656-
if (is_lob_type_(col_meta) && nullptr == new_cols[i].buf) {
657-
has_lob_null = true;
657+
if ((nullptr == new_cols[i].buf && new_cols[i].m_origin == VALUE_ORIGIN::PADDING)
658+
|| (nullptr != old_cols[i].buf && old_cols[i].m_origin == VALUE_ORIGIN::PADDING)) {
659+
has_unreliable_null = true;
658660
}
659661
}
660662
}
661663
APPEND_STMT(stmt, " WHERE ");
662664

663665
if (OB_SUCC(ret) && OB_FAIL(build_where_conds_(stmt, old_cols, old_col_cnt,
664-
tbl_meta, has_lob_null, has_unsupport_type_compare))) {
666+
tbl_meta, has_unreliable_null, has_unsupport_type_compare))) {
665667
LOG_ERROR("build where conds failed",);
666668
}
667669
if (lib::Worker::CompatMode::MYSQL == compat_mode_) {
@@ -686,8 +688,8 @@ int ObLogMinerRecord::build_delete_stmt_(ObStringBuffer &stmt,
686688
binlogBuf *old_cols,
687689
const unsigned int old_col_cnt,
688690
ITableMeta *tbl_meta,
689-
bool &has_lob_null,
690-
bool &has_unsupport_type_compare)
691+
bool &has_unreliable_null,
692+
bool &has_unsupport_type_compare)
691693
{
692694
int ret = OB_SUCCESS;
693695
if (IS_NOT_INIT) {
@@ -705,7 +707,7 @@ int ObLogMinerRecord::build_delete_stmt_(ObStringBuffer &stmt,
705707
APPEND_STMT(stmt, " WHERE ");
706708

707709
if (OB_SUCC(ret) && OB_FAIL(build_where_conds_(stmt, old_cols, old_col_cnt,
708-
tbl_meta, has_lob_null, has_unsupport_type_compare))) {
710+
tbl_meta, has_unreliable_null, has_unsupport_type_compare))) {
709711
LOG_ERROR("build where conds failed",);
710712
}
711713
if (lib::Worker::CompatMode::MYSQL == compat_mode_) {
@@ -861,21 +863,21 @@ int ObLogMinerRecord::build_column_value_(ObStringBuffer &stmt,
861863
}
862864

863865
int ObLogMinerRecord::build_where_conds_(ObStringBuffer &stmt,
864-
binlogBuf *cols,
865-
const unsigned int col_cnt,
866-
ITableMeta *tbl_meta,
867-
bool &has_lob_null,
868-
bool &has_unsupport_type_compare)
866+
binlogBuf *cols,
867+
const unsigned int col_cnt,
868+
ITableMeta *tbl_meta,
869+
bool &has_unreliable_null,
870+
bool &has_unsupport_type_compare)
869871
{
870872
int ret = OB_SUCCESS;
871873
if (!unique_keys_.empty()) {
872874
if (OB_FAIL(build_key_conds_(stmt, cols, col_cnt, tbl_meta,
873-
unique_keys_, has_lob_null, has_unsupport_type_compare))) {
875+
unique_keys_, has_unreliable_null, has_unsupport_type_compare))) {
874876
LOG_ERROR("build unique keys failed", K(stmt), K(unique_keys_));
875877
}
876878
} else if (!primary_keys_.empty()) {
877879
if (OB_FAIL(build_key_conds_(stmt, cols, col_cnt, tbl_meta,
878-
primary_keys_, has_lob_null, has_unsupport_type_compare))) {
880+
primary_keys_, has_unreliable_null, has_unsupport_type_compare))) {
879881
LOG_ERROR("build primary keys failed", K(stmt), K(primary_keys_));
880882
}
881883
} else {
@@ -885,7 +887,7 @@ int ObLogMinerRecord::build_where_conds_(ObStringBuffer &stmt,
885887
APPEND_STMT(stmt, " AND ");
886888
}
887889
if (OB_SUCC(ret)) {
888-
if (OB_FAIL(build_cond_(stmt, cols, i, tbl_meta, col_meta, has_lob_null, has_unsupport_type_compare))) {
890+
if (OB_FAIL(build_cond_(stmt, cols, i, tbl_meta, col_meta, has_unreliable_null, has_unsupport_type_compare))) {
889891
LOG_ERROR("build cond failed", "table_name", tbl_meta->getName());
890892
}
891893
}
@@ -895,12 +897,12 @@ int ObLogMinerRecord::build_where_conds_(ObStringBuffer &stmt,
895897
}
896898

897899
int ObLogMinerRecord::build_key_conds_(ObStringBuffer &stmt,
898-
binlogBuf *cols,
899-
const unsigned int col_cnt,
900-
ITableMeta *tbl_meta,
901-
const KeyArray &key,
902-
bool &has_lob_null,
903-
bool &has_unsupport_type_compare)
900+
binlogBuf *cols,
901+
const unsigned int col_cnt,
902+
ITableMeta *tbl_meta,
903+
const KeyArray &key,
904+
bool &has_unreliable_null,
905+
bool &has_unsupport_type_compare)
904906
{
905907
int ret = OB_SUCCESS;
906908
for (int i = 0; OB_SUCC(ret) && i < key.count(); i++) {
@@ -915,7 +917,7 @@ int ObLogMinerRecord::build_key_conds_(ObStringBuffer &stmt,
915917
}
916918
if (OB_SUCC(ret)) {
917919
if (OB_FAIL(build_cond_(stmt, cols, col_idx, tbl_meta, col_meta,
918-
has_lob_null, has_unsupport_type_compare))) {
920+
has_unreliable_null, has_unsupport_type_compare))) {
919921
LOG_ERROR("build cond failed", "table_name", tbl_meta->getName());
920922
}
921923
}
@@ -925,12 +927,12 @@ int ObLogMinerRecord::build_key_conds_(ObStringBuffer &stmt,
925927
}
926928

927929
int ObLogMinerRecord::build_cond_(ObStringBuffer &stmt,
928-
binlogBuf *cols,
929-
const unsigned int col_idx,
930-
ITableMeta *tbl_meta,
931-
IColMeta *col_meta,
932-
bool &has_lob_null,
933-
bool &has_unsupport_type_compare)
930+
binlogBuf *cols,
931+
const unsigned int col_idx,
932+
ITableMeta *tbl_meta,
933+
IColMeta *col_meta,
934+
bool &has_unreliable_null,
935+
bool &has_unsupport_type_compare)
934936
{
935937
int ret = OB_SUCCESS;
936938
if (OB_ISNULL(col_meta)) {
@@ -940,7 +942,7 @@ int ObLogMinerRecord::build_cond_(ObStringBuffer &stmt,
940942
if (is_lob_type_(col_meta) && nullptr != cols[col_idx].buf) {
941943
// build lob type compare condition, excluding null value condition
942944
if (OB_FAIL(build_lob_cond_(stmt, cols, col_idx, tbl_meta,
943-
col_meta, has_lob_null, has_unsupport_type_compare))) {
945+
col_meta, has_unreliable_null, has_unsupport_type_compare))) {
944946
LOG_ERROR("build lob condition failed", "table_name", tbl_meta->getName());
945947
}
946948
} else {
@@ -949,21 +951,21 @@ int ObLogMinerRecord::build_cond_(ObStringBuffer &stmt,
949951
}
950952
}
951953
if (OB_SUCC(ret)) {
952-
if (is_lob_type_(col_meta) && nullptr == cols[col_idx].buf) {
953-
has_lob_null = true;
954+
if (nullptr == cols[col_idx].buf && cols[col_idx].m_origin == VALUE_ORIGIN::PADDING) {
955+
has_unreliable_null = true;
954956
}
955957
}
956958
}
957959
return ret;
958960
}
959961

960962
int ObLogMinerRecord::build_lob_cond_(ObStringBuffer &stmt,
961-
binlogBuf *cols,
962-
const unsigned int col_idx,
963-
ITableMeta *tbl_meta,
964-
IColMeta *col_meta,
965-
bool &has_lob_null,
966-
bool &has_unsupport_type_compare)
963+
binlogBuf *cols,
964+
const unsigned int col_idx,
965+
ITableMeta *tbl_meta,
966+
IColMeta *col_meta,
967+
bool &has_unreliable_null,
968+
bool &has_unsupport_type_compare)
967969
{
968970
int ret = OB_SUCCESS;
969971
if (OB_UNLIKELY(!is_lob_type_(col_meta) || nullptr == cols[col_idx].buf)) {
@@ -1020,10 +1022,10 @@ int ObLogMinerRecord::build_lob_cond_(ObStringBuffer &stmt,
10201022
}
10211023

10221024
int ObLogMinerRecord::build_func_cond_(ObStringBuffer &stmt,
1023-
binlogBuf *cols,
1024-
const unsigned int col_idx,
1025+
binlogBuf *cols,
1026+
const unsigned int col_idx,
10251027
ITableMeta *tbl_meta,
1026-
IColMeta *col_meta,
1028+
IColMeta *col_meta,
10271029
const char *func_name)
10281030
{
10291031
int ret = OB_SUCCESS;
@@ -1042,10 +1044,10 @@ int ObLogMinerRecord::build_func_cond_(ObStringBuffer &stmt,
10421044
}
10431045

10441046
int ObLogMinerRecord::build_normal_cond_(ObStringBuffer &stmt,
1045-
binlogBuf *cols,
1046-
const unsigned int col_idx,
1047-
ITableMeta *tbl_meta,
1048-
IColMeta *col_meta)
1047+
binlogBuf *cols,
1048+
const unsigned int col_idx,
1049+
ITableMeta *tbl_meta,
1050+
IColMeta *col_meta)
10491051
{
10501052
int ret = OB_SUCCESS;
10511053
APPEND_ESCAPE_CHAR(stmt);

0 commit comments

Comments
 (0)