Skip to content

Commit cfe6012

Browse files
dongwook-chansean-k1
dongwook-chan
and
sean-k1
authored
Add VisibilityBitmap to TableMapEvent in replication (#813)
* Add `VisibilityBitmap` to `TableMapEvent` for MySQL 8.0.23+ Invisible Columns Changes: - `TableMapEvent.VisibilityBitmap` `VisibilityBitmap` is a bitmap where each bit represents the visibility of a corresponding column in a table. If a bit is set, it indicates that the corresponding column is NOT an invinsible column. Invisible column was introduced in MySQL version 8.0.23. - `TableMapEvent.VisibilityMap` `VisibilityMap` lists boolean values of which is true if column of the same index is NOT an invisible column. Co-authored-by: sean <sean.k1@kakaoent.com> * Refactor bitmap iteration to align with MySQL source code Changes: - Refactpred `UnsignedMap` - Refactored `VisibilityMap` Suggested by: #813 (comment) * Add test for column visibility in table map Changes: - Added data for MySQL 8.0 only (Only MySQL 8.0.23+ supports invisible columns) * Add test case 2 for column visibility in table map Changes: - Add test case 2 where invisible columns does not exists at all --------- Co-authored-by: sean <sean.k1@kakaoent.com>
1 parent dc97dfa commit cfe6012

File tree

3 files changed

+160
-6
lines changed

3 files changed

+160
-6
lines changed

‎replication/const.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ const (
240240
TABLE_MAP_OPT_META_PRIMARY_KEY_WITH_PREFIX
241241
TABLE_MAP_OPT_META_ENUM_AND_SET_DEFAULT_CHARSET
242242
TABLE_MAP_OPT_META_ENUM_AND_SET_COLUMN_CHARSET
243+
TABLE_MAP_OPT_META_COLUMN_VISIBILITY
243244
)
244245

245246
type IntVarEventType byte

‎replication/row_event.go

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ type TableMapEvent struct {
8181
// EnumSetDefaultCharset/EnumSetColumnCharset is similar to DefaultCharset/ColumnCharset but for enum/set columns.
8282
EnumSetDefaultCharset []uint64
8383
EnumSetColumnCharset []uint64
84+
85+
// VisibilityBitmap stores bits that are set if corresponding column is not invisible (MySQL 8.0.23+)
86+
VisibilityBitmap []byte
8487
}
8588

8689
func (e *TableMapEvent) Decode(data []byte) error {
@@ -312,6 +315,9 @@ func (e *TableMapEvent) decodeOptionalMeta(data []byte) (err error) {
312315
return err
313316
}
314317

318+
case TABLE_MAP_OPT_META_COLUMN_VISIBILITY:
319+
e.VisibilityBitmap = v
320+
315321
default:
316322
// Ignore for future extension
317323
}
@@ -421,6 +427,7 @@ func (e *TableMapEvent) Dump(w io.Writer) {
421427
fmt.Fprintf(w, "Primary key prefix: %v\n", e.PrimaryKeyPrefix)
422428
fmt.Fprintf(w, "Enum/set default charset: %v\n", e.EnumSetDefaultCharset)
423429
fmt.Fprintf(w, "Enum/set column charset: %v\n", e.EnumSetColumnCharset)
430+
fmt.Fprintf(w, "Invisible Column bitmap: \n%s", hex.Dump(e.VisibilityBitmap))
424431

425432
unsignedMap := e.UnsignedMap()
426433
fmt.Fprintf(w, "UnsignedMap: %#v\n", unsignedMap)
@@ -440,6 +447,9 @@ func (e *TableMapEvent) Dump(w io.Writer) {
440447
geometryTypeMap := e.GeometryTypeMap()
441448
fmt.Fprintf(w, "GeometryTypeMap: %#v\n", geometryTypeMap)
442449

450+
visibilityMap := e.VisibilityMap()
451+
fmt.Fprintf(w, "VisibilityMap: %#v\n", visibilityMap)
452+
443453
nameMaxLen := 0
444454
for _, name := range e.ColumnName {
445455
if len(name) > nameMaxLen {
@@ -608,14 +618,19 @@ func (e *TableMapEvent) UnsignedMap() map[int]bool {
608618
if len(e.SignednessBitmap) == 0 {
609619
return nil
610620
}
611-
p := 0
612621
ret := make(map[int]bool)
613-
for i := 0; i < int(e.ColumnCount); i++ {
614-
if !e.IsNumericColumn(i) {
615-
continue
622+
i := 0
623+
for _, field := range e.SignednessBitmap {
624+
for c := 0x80; c != 0; {
625+
if e.IsNumericColumn(i) {
626+
ret[i] = field&byte(c) != 0
627+
c >>= 1
628+
}
629+
i++
630+
if i >= int(e.ColumnCount) {
631+
return ret
632+
}
616633
}
617-
ret[i] = e.SignednessBitmap[p/8]&(1<<uint(7-p%8)) != 0
618-
p++
619634
}
620635
return ret
621636
}
@@ -730,6 +745,27 @@ func (e *TableMapEvent) GeometryTypeMap() map[int]uint64 {
730745
return ret
731746
}
732747

748+
// VisibilityMap returns a map: column index -> visiblity.
749+
// Invisible column was introduced in MySQL 8.0.23
750+
// nil is returned if not available.
751+
func (e *TableMapEvent) VisibilityMap() map[int]bool {
752+
if len(e.VisibilityBitmap) == 0 {
753+
return nil
754+
}
755+
ret := make(map[int]bool)
756+
i := 0
757+
for _, field := range e.VisibilityBitmap {
758+
for c := 0x80; c != 0; c >>= 1 {
759+
ret[i] = field&byte(c) != 0
760+
i++
761+
if uint64(i) >= e.ColumnCount {
762+
return ret
763+
}
764+
}
765+
}
766+
return ret
767+
}
768+
733769
// Below realType and IsXXXColumn are base from:
734770
// table_def::type in sql/rpl_utility.h
735771
// Table_map_log_event::print_columns in mysql-8.0/sql/log_event.cc and mariadb-10.5/sql/log_event_client.cc

‎replication/row_event_test.go

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -939,6 +939,123 @@ func TestTableMapOptMetaPrimaryKey(t *testing.T) {
939939
}
940940
}
941941

942+
func TestTableMapOptMetaVisibility(t *testing.T) {
943+
/*
944+
SET GLOBAL binlog_row_image = FULL;
945+
SET GLOBAL binlog_row_metadata = FULL; -- if applicable
946+
947+
CREATE DATABASE test;
948+
USE test;
949+
*/
950+
951+
/*
952+
CREATE TABLE _visibility(
953+
`col0` INT INVISIBLE,
954+
`col1` INT,
955+
`col2` INT INVISIBLE,
956+
`col3` INT,
957+
`col4` INT,
958+
`col5` INT INVISIBLE,
959+
`col6` INT INVISIBLE,
960+
`col7` INT INVISIBLE,
961+
`col8` INT,
962+
`col9` INT INVISIBLE,
963+
`col10` INT INVISIBLE
964+
);
965+
*/
966+
case1VisibilityBitmap := []byte{0x58, 0x80}
967+
case1VisibilityMap := map[int]bool{
968+
0: false,
969+
1: true,
970+
2: false,
971+
3: true,
972+
4: true,
973+
5: false,
974+
6: false,
975+
7: false,
976+
8: true,
977+
9: false,
978+
10: false,
979+
}
980+
981+
/*
982+
CREATE TABLE _visibility(
983+
`col0` INT,
984+
`col1` INT,
985+
`col2` INT,
986+
`col3` INT,
987+
`col4` INT,
988+
`col5` INT,
989+
`col6` INT,
990+
`col7` INT,
991+
`col8` INT,
992+
`col9` INT,
993+
`col10` INT
994+
);
995+
*/
996+
case2VisibilityBitmap := []byte{0xff, 0xe0}
997+
case2VisibilityMap := map[int]bool{
998+
0: true,
999+
1: true,
1000+
2: true,
1001+
3: true,
1002+
4: true,
1003+
5: true,
1004+
6: true,
1005+
7: true,
1006+
8: true,
1007+
9: true,
1008+
10: true,
1009+
}
1010+
1011+
// Invisible column and INVISIBLE keyword is available only on MySQL 8.0.23+
1012+
testcases := []struct {
1013+
data []byte
1014+
expectedVisibilityBitmap []byte
1015+
expectedVisibilityMap map[int]bool
1016+
}{
1017+
{
1018+
// mysql 8.0, case1
1019+
data: []byte("^\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07\x01\x02\x00\x00\x048\x04col0\x04col1\x04col2\x04col3\x04col4\x04col5\x04col6\x04col7\x04col8\x04col9\x05col10\x0c\x02X\x80"),
1020+
expectedVisibilityBitmap: case1VisibilityBitmap,
1021+
expectedVisibilityMap: case1VisibilityMap,
1022+
},
1023+
{
1024+
// mysql 5.7, case2
1025+
data: []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07"),
1026+
expectedVisibilityBitmap: []byte(nil),
1027+
expectedVisibilityMap: map[int]bool(nil),
1028+
},
1029+
{
1030+
// mysql 8.0, case2
1031+
data: []byte("^\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07\x01\x02\x00\x00\x048\x04col0\x04col1\x04col2\x04col3\x04col4\x04col5\x04col6\x04col7\x04col8\x04col9\x05col10\x0c\x02\xff\xe0"),
1032+
expectedVisibilityBitmap: case2VisibilityBitmap,
1033+
expectedVisibilityMap: case2VisibilityMap,
1034+
},
1035+
{
1036+
// mariadb 10.4, case2
1037+
data: []byte("\x12\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07"),
1038+
expectedVisibilityBitmap: []byte(nil),
1039+
expectedVisibilityMap: map[int]bool(nil),
1040+
},
1041+
{
1042+
// mariadb 10.5, case2
1043+
data: []byte("\x12\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x0b_visibility\x00\x0b\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x03\x00\xff\x07\x01\x02\x00\x00\x048\x04col0\x04col1\x04col2\x04col3\x04col4\x04col5\x04col6\x04col7\x04col8\x04col9\x05col10"),
1044+
expectedVisibilityBitmap: []byte(nil),
1045+
expectedVisibilityMap: map[int]bool(nil),
1046+
},
1047+
}
1048+
1049+
for _, tc := range testcases {
1050+
tableMapEvent := new(TableMapEvent)
1051+
tableMapEvent.tableIDSize = 6
1052+
err := tableMapEvent.Decode(tc.data)
1053+
require.NoError(t, err)
1054+
require.Equal(t, tc.expectedVisibilityBitmap, tableMapEvent.VisibilityBitmap)
1055+
require.Equal(t, tc.expectedVisibilityMap, tableMapEvent.VisibilityMap())
1056+
}
1057+
}
1058+
9421059
func TestTableMapHelperMaps(t *testing.T) {
9431060
/*
9441061
CREATE TABLE `_types` (

0 commit comments

Comments
 (0)