From d8bc28fc04c102fb0712b3310398b9035444cfac Mon Sep 17 00:00:00 2001 From: tserakhau Date: Sun, 30 Jun 2024 15:25:05 +0200 Subject: [PATCH] Initial commit --- .mapping.json | 519 - .../provideradapter/airbyte/adapter.go | 35 - .../provideradapter/clickhouse/adapter.go | 53 - .../provideradapter/elastic/adapter.go | 55 - .../provideradapter/eventhub/adapter.go | 24 - .../provideradapter/kafka/adapter.go | 70 - .../provideradapter/logbroker/adapter.go | 108 - .../provideradapter/mongo/adapter.go | 34 - .../provideradapter/mysql/adapter.go | 57 - .../provideradapter/postgres/adapter.go | 50 - pkg/dataplane/provideradapter/s3/adapter.go | 32 - pkg/dataplane/provideradapter/ydb/adapter.go | 142 - pkg/dataplane/provideradapter/yds/adapter.go | 72 - pkg/dataplane/provideradapter/yt/adapter.go | 48 - pkg/dataplane/providers.go | 14 - recipe/mongo/README.md | 36 - recipe/mongo/cmd/binurl/README.md | 21 - recipe/mongo/cmd/binurl/binary_fetcher.go | 215 - recipe/mongo/example/configs/auth.yaml | 16 - recipe/mongo/example/launch_cluster/README.md | 14 - recipe/mongo/example/launch_cluster/main.go | 77 - recipe/mongo/example/recipe_usage/README.md | 44 - .../mongo/example/recipe_usage/sample_test.go | 11 - recipe/mongo/pkg/binurl/binary_links.go | 256 - recipe/mongo/pkg/cluster/cluster.go | 190 - .../mongo/pkg/cluster/config_replica_set.go | 96 - recipe/mongo/pkg/cluster/environment_info.go | 30 - recipe/mongo/pkg/cluster/mongod.go | 290 - recipe/mongo/pkg/cluster/mongos.go | 186 - recipe/mongo/pkg/cluster/shard_replica_set.go | 98 - recipe/mongo/pkg/config/config.go | 193 - recipe/mongo/pkg/tar/tar.go | 90 - recipe/mongo/pkg/util/test_common.go | 69 - recipe/mongo/pkg/util/yatest.go | 21 - recipe/mongo/recipe.go | 310 - recipe/mongo/test/4.4/cluster_test.go | 11 - recipe/mongo/test/4.4/mongocluster.yaml | 17 - recipe/mongo/test/5.0/cluster_test.go | 11 - recipe/mongo/test/5.0/mongocluster.yaml | 17 - recipe/mongo/test/6.0/cluster_test.go | 11 - recipe/mongo/test/6.0/mongocluster.yaml | 17 - tests/canon/all_db_test.go | 33 - tests/canon/clickhouse/README.md | 19 - tests/canon/clickhouse/canon_test.go | 87 - tests/canon/clickhouse/snapshot/data.sql | 162 - tests/canon/gotest/canondata/result.json | 15055 ---------------- tests/canon/mongo/README.md | 19 - tests/canon/mongo/canon_docs.go | 138 - tests/canon/mongo/canon_test.go | 83 - .../canon/mongo/gotest/canondata/result.json | 1368 -- tests/canon/mysql/canon_sql.go | 45 - tests/canon/mysql/canon_test.go | 116 - tests/canon/mysql/canondata/result.json | 70 - tests/canon/mysql/dump/date_types.sql | 77 - tests/canon/mysql/dump/initial_data.sql | 249 - tests/canon/mysql/dump/json_types.sql | 18 - tests/canon/mysql/dump/numeric_types.sql | 114 - tests/canon/mysql/dump/numeric_types_bit.sql | 20 - .../mysql/dump/numeric_types_boolean.sql | 16 - .../mysql/dump/numeric_types_decimal.sql | 34 - .../canon/mysql/dump/numeric_types_float.sql | 34 - tests/canon/mysql/dump/numeric_types_int.sql | 41 - tests/canon/mysql/dump/spatial_types.sql | 0 tests/canon/mysql/dump/string_types.sql | 80 - tests/canon/mysql/dump/string_types_emoji.sql | 15 - tests/canon/parser/README.md | 30 - .../canon/parser/canon_static_generic_test.go | 42 - .../extracted | 439 - .../extracted | 1022 -- .../extracted | 563 - .../extracted | 1430 -- .../extracted | 308 - .../extracted | 2266 --- .../extracted | 308 - .../extracted | 954 - .../extracted | 1190 -- .../extracted | 512 - .../extracted | 223 - .../canon/parser/gotest/canondata/result.json | 35 - .../sample_proto/sample_proto/README.MD | 4 - .../sample_proto/sample_proto.proto | 35 - .../samples/dynamic/sample_proto/test_case.go | 111 - .../samples/static/generic/json.config.json | 120 - .../parser/samples/static/generic/json.sample | 4 - .../samples/static/generic/mdb.config.json | 354 - .../parser/samples/static/generic/mdb.sample | 1 - .../static/generic/metrika.config.json | 211 - .../samples/static/generic/metrika.sample | 4 - .../generic/metrika_complex.config.json | 172 - .../static/generic/metrika_complex.sample | 1 - .../samples/static/generic/taxi.config.json | 357 - .../parser/samples/static/generic/taxi.sample | 4 - .../static/generic/tm-5249.config.json | 55 - .../samples/static/generic/tm-5249.sample | 2 - .../samples/static/generic/tskv.config.json | 107 - .../parser/samples/static/generic/tskv.sample | 4 - .../static/logfeller/_type_check_rules.yaml | 0 .../static/logfeller/kikimr-log-2.yaml | 0 .../samples/static/logfeller/kikimr-log.yaml | 0 .../static/logfeller/kikimr-new-log.yaml | 0 .../static/logfeller/kikimr.config.json | 120 - .../samples/static/logfeller/kikimr.sample | 4 - .../static/logfeller/kikimr_new.config.json | 72 - .../static/logfeller/kikimr_new.sample | 3 - .../static/logfeller/sensitive.config.json | 107 - .../samples/static/logfeller/sensitive.sample | 1 - tests/canon/parser/testcase/test_case.go | 85 - tests/canon/postgres/canon_sql.go | 27 - tests/canon/postgres/canon_test.go | 92 - tests/canon/postgres/dump/array_types.sql | 183 - tests/canon/postgres/dump/date_types.sql | 52 - tests/canon/postgres/dump/geom_types.sql | 28 - tests/canon/postgres/dump/numeric_types.sql | 150 - tests/canon/postgres/dump/text_types.sql | 36 - tests/canon/postgres/dump/wtf_types.sql | 36 - .../postgres/gotest/canondata/result.json | 38 - tests/canon/s3/csv/canon_test.go | 338 - .../extracted | 461 - .../extracted | 461 - tests/canon/s3/csv/canondata/result.json | 462 - tests/canon/s3/jsonline/canon_test.go | 94 - tests/canon/s3/jsonline/canondata/result.json | 355 - tests/canon/s3/parquet/canon_test.go | 131 - .../extracted | 580 - .../extracted | 869 - .../extracted | 580 - .../extracted | 359 - .../extracted | 359 - .../extracted | 359 - .../extracted | 596 - .../extracted | 1175 -- .../extracted | 1175 -- .../extracted | 2858 --- .../extracted | 359 - .../extracted | 2858 --- .../extracted | 359 - .../extracted | 359 - .../extracted | 359 - .../extracted | 2858 --- .../extracted | 359 - .../extracted | 557 - .../extracted | 461 - .../extracted | 830 - .../extracted | 569 - .../extracted | 1616 -- .../extracted | 450 - .../extracted | 126 - .../extracted | 1669 -- .../extracted | 374 - .../extracted | 3266 ---- .../extracted | 415 - .../extracted | 359 - .../extracted | 121 - tests/canon/s3/parquet/canondata/result.json | 117 - tests/canon/sequences/README.md | 16 - tests/canon/sequences/canondata/result.json | 11 - .../dump/00_insert_update_delete.sql | 3 - tests/canon/sequences/dump/01_updatepk.sql | 4 - .../dump/02_insert_update_insert.sql | 12 - .../dump/init.insert_update_delete.sql | 3 - tests/canon/sequences/sequences_test.go | 111 - tests/canon/ydb/canon_test.go | 133 - tests/canon/ydb/canondata/result.json | 5 - tests/canon/yt/canon_test.go | 262 - tests/canon/yt/canondata/result.json | 11 - tests/e2e/ch2ch/db_alt_name/check_db_test.go | 57 - tests/e2e/ch2ch/db_alt_name/dump/dst.sql | 1 - tests/e2e/ch2ch/db_alt_name/dump/src.sql | 16 - .../ch2ch/db_complex_name/check_db_test.go | 40 - tests/e2e/ch2ch/db_complex_name/dump/dst.sql | 1 - tests/e2e/ch2ch/db_complex_name/dump/src.sql | 185 - .../incremental_many_shards/check_db_test.go | 104 - .../incremental_many_shards/dump/dst.sql | 18 - .../incremental_many_shards/dump/src.sql | 18 - .../incremental_one_shard/check_db_test.go | 97 - .../ch2ch/incremental_one_shard/dump/dst.sql | 18 - .../ch2ch/incremental_one_shard/dump/src.sql | 18 - tests/e2e/ch2ch/snapshot/check_db_test.go | 89 - tests/e2e/ch2ch/snapshot/dump/dst.sql | 1 - tests/e2e/ch2ch/snapshot/dump/src.sql | 225 - .../check_db_test.go | 85 - .../dump/dst.sql | 1 - .../dump/src.sql | 59 - tests/e2e/ch2yt/static_table/check_db_test.go | 60 - tests/e2e/ch2yt/static_table/dump/src.sql | 36 - tests/e2e/kafka2ch/blank_parser/ch_init.sql | 1 - .../kafka2ch/blank_parser/check_db_test.go | 77 - .../extracted | 55 - .../replication/canondata/result.json | 5 - .../e2e/kafka2ch/replication/check_db_test.go | 127 - .../e2e/kafka2ch/replication/dump/ch/dump.sql | 1 - .../extracted | 55 - .../replication_mv/canondata/result.json | 5 - .../kafka2ch/replication_mv/check_db_test.go | 137 - .../kafka2ch/replication_mv/dump/ch/dump.sql | 35 - .../metering/canondata/result.json | 44 - .../e2e/kafka2kafka/metering/metering_test.go | 191 - tests/e2e/kafka2kafka/mirror/mirror_test.go | 120 - .../multi_topic/canondata/result.json | 12 - .../kafka2kafka/multi_topic/mirror_test.go | 102 - .../kafka2mongo/replication/check_db_test.go | 115 - .../replication/dump/date_time.sql | 0 .../kafka2mysql/filter_rows/check_db_test.go | 181 - .../filter_rows/dump/date_time.sql | 0 .../kafka2mysql/replication/check_db_test.go | 107 - .../replication/dump/date_time.sql | 0 tests/e2e/mongo2ch/snapshot/check_db_test.go | 81 - tests/e2e/mongo2ch/snapshot/dump.sql | 1 - .../snapshot_flatten/canondata/result.json | 5 - .../extracted | 40 - .../snapshot_flatten/check_db_test.go | 126 - tests/e2e/mongo2ch/snapshot_flatten/dump.sql | 1 - tests/e2e/mongo2mock/slots/slot_test.go | 362 - .../tech_db_permission/permission_test.go | 240 - .../add_db_on_snapshot/check_db_test.go | 180 - .../bson_obj_too_large/check_db_test.go | 382 - .../mongo2mongo/bson_order/reorder_test.go | 327 - .../mongo2mongo/db_rename/check_db_test.go | 176 - .../db_rename_rep/check_db_test.go | 242 - .../mongo2mongo/replication/check_db_test.go | 385 - .../replication_filter_test/check_db_test.go | 246 - .../replication_update_model/check_db_test.go | 172 - .../rps/replication_source/rps_test.go | 382 - tests/e2e/mongo2mongo/rps/rps.go | 309 - .../to_sharded/document_key_updates/db1.yaml | 12 - .../to_sharded/document_key_updates/db2.yaml | 12 - .../document_key_updates/rps_test.go | 356 - .../to_sharded/nested_shard_key/db1.yaml | 12 - .../to_sharded/nested_shard_key/db2.yaml | 12 - .../nested_shard_key/nested_shard_key_test.go | 274 - .../e2e/mongo2mongo/snapshot/check_db_test.go | 131 - tests/e2e/mysql2ch/comparators.go | 69 - .../e2e/mysql2ch/replication/check_db_test.go | 85 - .../e2e/mysql2ch/replication/dump/ch/dump.sql | 1 - .../mysql2ch/replication/dump/mysql/dump.sql | 15 - .../replication_minimal/check_db_test.go | 107 - .../replication_minimal/dump/ch/dump.sql | 1 - .../replication_minimal/dump/mysql/dump.sql | 9 - tests/e2e/mysql2ch/snapshot/check_db_test.go | 61 - tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql | 1 - .../e2e/mysql2ch/snapshot/dump/mysql/dump.sql | 101 - .../snapshot_empty_table/check_db_test.go | 66 - .../snapshot_empty_table/dump/ch/dump.sql | 1 - .../snapshot_empty_table/dump/mysql/dump.sql | 24 - .../extracted | 1 - .../extracted.0 | 1 - .../extracted.1 | 1 - .../extracted.2 | 1 - .../extracted.3 | 1 - .../extracted.4 | 1 - .../replication/canondata/result.json | 28 - .../debezium/replication/check_db_test.go | 144 - .../debezium/replication/init_source/dump.sql | 114 - .../debezium/replication/testdata/insert.sql | 123 - .../replication/testdata/update_string.sql | 1 - .../debezium/snapshot/canondata/result.json | 5 - .../snapshot.snapshot.TestSnapshot/extracted | 1 - .../debezium/snapshot/check_db_test.go | 101 - .../debezium/snapshot/init_source/dump.sql | 235 - .../debezium_replication/check_db_test.go | 460 - .../debezium_replication/dump/dump.sql | 125 - .../testdata/debezium_msg_0_key.txt | 1 - .../testdata/debezium_msg_0_val.txt | 1 - .../testdata/debezium_msg_1_key.txt | 1 - .../testdata/debezium_msg_1_val.txt | 1 - .../testdata/debezium_msg_2_key.txt | 1 - .../testdata/debezium_msg_2_val.txt | 1 - .../testdata/debezium_msg_3_key.txt | 1 - .../testdata/debezium_msg_3_val.txt | 1 - .../testdata/debezium_msg_4_key.txt | 1 - .../testdata/debezium_msg_5_key.txt | 1 - .../testdata/debezium_msg_5_val.txt | 1 - .../testdata/debezium_msg_6_key.txt | 1 - .../testdata/debezium_msg_6_val.txt | 1 - .../testdata/debezium_msg_7_key.txt | 1 - .../debezium_snapshot/check_db_test.go | 73 - .../debezium/debezium_snapshot/dump/dump.sql | 253 - .../testdata/change_item_key.txt | 1 - .../testdata/change_item_val.txt | 1 - .../non_utf8_charset/check_db_test.go | 147 - .../mysql2mock/non_utf8_charset/dump/dump.sql | 11 - .../mysql2mock/timezone/canondata/result.json | 20 - .../e2e/mysql2mock/timezone/check_db_test.go | 147 - tests/e2e/mysql2mock/timezone/dump/dump.sql | 11 - tests/e2e/mysql2mock/views/check_db_test.go | 130 - tests/e2e/mysql2mock/views/dump/dump.sql | 21 - tests/e2e/mysql2mysql/alters/check_db_test.go | 215 - .../mysql2mysql/alters/dump/type_check.sql | 23 - tests/e2e/mysql2mysql/binary/check_db_test.go | 101 - .../mysql2mysql/binary/dump/type_check.sql | 12 - .../cascade_deletes/common/test.go | 120 - .../cascade_deletes/dump/type_check.sql | 39 - .../test_per_table/check_db_test.go | 24 - .../test_per_transaction/check_db_test.go | 25 - .../cleanup_tables/cleanup_test.go | 87 - .../cleanup_tables/source/dump.sql | 38 - .../cleanup_tables/target/dump.sql | 46 - .../e2e/mysql2mysql/comment/check_db_test.go | 61 - .../e2e/mysql2mysql/comment/dump/comment.sql | 8 - .../consistent_snapshot/check_db_test.go | 151 - .../dump/consistent_snapshot.sql | 44 - .../mysql2mysql/date_time/check_db_test.go | 151 - .../mysql2mysql/date_time/dump/date_time.sql | 34 - .../debezium/all_datatypes/check_db_test.go | 171 - .../all_datatypes/dump/type_check.sql | 231 - .../all_datatypes_nohomo/check_db_test.go | 173 - .../all_datatypes_nohomo/dump/type_check.sql | 231 - .../all_datatypes_serde/check_db_test.go | 209 - .../all_datatypes_serde/dump/type_check.sql | 231 - .../check_db_test.go | 189 - .../dump/type_check.sql | 231 - .../check_db_test.go | 72 - .../dump/type_check.sql | 118 - .../check_db_test.go | 273 - .../dump/type_check.sql | 231 - .../check_db_test.go | 189 - .../dump/type_check.sql | 231 - .../check_db_test.go | 113 - .../dump/type_check.sql | 59 - .../canondata/float.float.TestFloat/extracted | 181 - .../float.float.TestFloat/extracted.0 | 181 - .../mysql2mysql/float/canondata/result.json | 10 - tests/e2e/mysql2mysql/float/check_db_test.go | 38 - tests/e2e/mysql2mysql/float/dump/dump.sql | 73 - tests/e2e/mysql2mysql/float/increment.sql | 61 - .../e2e/mysql2mysql/geometry/check_db_test.go | 141 - .../mysql2mysql/geometry/dump/geometry.sql | 23 - tests/e2e/mysql2mysql/light/check_db_test.go | 53 - .../e2e/mysql2mysql/light/dump/type_check.sql | 155 - .../light_all_datatypes/check_db_test.go | 140 - .../light_all_datatypes/dump/type_check.sql | 78 - tests/e2e/mysql2mysql/medium/check_db_test.go | 52 - .../no_auto_value_on_zero/check_db_test.go | 131 - .../dump/no_auto_value_on_zero.sql | 4 - .../partitioned_table/check_db_test.go | 73 - .../partitioned_table/dump/dump.sql | 9 - .../mysql2mysql/pkeychanges/check_db_test.go | 135 - .../pkeychanges/dump/type_check.sql | 15 - .../mysql2mysql/replace_fkey/common/test.go | 159 - .../mysql2mysql/replace_fkey/dump/fkey.sql | 13 - .../test_per_table/check_db_test.go | 24 - .../test_per_transaction/check_db_test.go | 25 - tests/e2e/mysql2mysql/scheme/check_db_test.go | 59 - tests/e2e/mysql2mysql/scheme/dump/scheme.sql | 711 - .../snapshot_without_pk/check_db_test.go | 52 - .../snapshot_without_pk/dump/dump.sql | 15 - .../tx_boundaries/check_db_test.go | 171 - .../mysql2mysql/tx_boundaries/dump/update.sql | 34 - tests/e2e/mysql2mysql/update/check_db_test.go | 130 - tests/e2e/mysql2mysql/update/dump/update.sql | 29 - .../update_cp1251/check_db_test.go | 130 - .../mysql2mysql/update_cp1251/dump/update.sql | 29 - .../update_minimal/check_db_test.go | 136 - .../update_minimal/dump/update_minimal.sql | 31 - .../update_unicode/check_db_test.go | 129 - .../update_unicode/dump/update.sql | 29 - tests/e2e/mysql2mysql/view/check_db_test.go | 57 - tests/e2e/mysql2mysql/view/dump/update.sql | 30 - tests/e2e/pg2ch/alters/alters_test.go | 149 - tests/e2e/pg2ch/alters/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/alters/dump/pg/dump.sql | 13 - .../pg2ch/alters_with_defaults/alters_test.go | 120 - .../alters_with_defaults/dump/ch/dump.sql | 1 - .../alters_with_defaults/dump/pg/dump.sql | 13 - .../chmapper.chmapper.TestSnapshot/extracted | 325 - .../e2e/pg2ch/chmapper/canondata/result.json | 5 - tests/e2e/pg2ch/chmapper/check_db_test.go | 111 - tests/e2e/pg2ch/chmapper/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/chmapper/dump/pg/dump.sql | 91 - tests/e2e/pg2ch/comparator.go | 83 - .../e2e/pg2ch/date_overflow/check_db_test.go | 54 - .../e2e/pg2ch/date_overflow/dump/ch/dump.sql | 1 - .../e2e/pg2ch/date_overflow/dump/pg/dump.sql | 15 - tests/e2e/pg2ch/dbt/check_db_test.go | 84 - tests/e2e/pg2ch/dbt/init_ch.sql | 1 - tests/e2e/pg2ch/dbt/init_pg.sql | 11 - tests/e2e/pg2ch/empty_keys/check_db_test.go | 56 - tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql | 17 - tests/e2e/pg2ch/replication/check_db_test.go | 96 - tests/e2e/pg2ch/replication/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/replication/dump/pg/dump.sql | 13 - .../e2e/pg2ch/replication_mv/check_db_test.go | 104 - .../e2e/pg2ch/replication_mv/dump/ch/dump.sql | 30 - .../e2e/pg2ch/replication_mv/dump/pg/dump.sql | 14 - tests/e2e/pg2ch/snapshot/check_db_test.go | 55 - tests/e2e/pg2ch/snapshot/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/snapshot/dump/pg/dump.sql | 160 - .../check_db_test.go | 76 - .../dump/ch/dump.sql | 1 - .../check_db_test.go | 75 - .../dump/ch/dump.sql | 1 - .../dump/pg/dump.sql | 26 - .../check_db_test.go | 55 - .../dump/ch/dump.sql | 1 - .../dump/pg/dump.sql | 9 - .../check_db_test.go | 59 - .../dump/ch/dump.sql | 1 - .../dump/pg/dump.sql | 17 - tests/e2e/pg2ch/snapshottsv1/check_db_test.go | 56 - tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql | 160 - .../check_tables_inclusion_test.go | 49 - .../pg2ch/tables_inclusion/dump/ch/dump.sql | 1 - .../pg2ch/tables_inclusion/dump/pg/dump.sql | 37 - tests/e2e/pg2ch/timestamp/check_db_test.go | 43 - tests/e2e/pg2ch/timestamp/dump/ch/dump.sql | 1 - tests/e2e/pg2ch/timestamp/dump/pg/dump.sql | 8 - .../debezium_replication/check_db_test.go | 366 - .../debezium_replication/init_source/dump.sql | 106 - tests/e2e/pg2mysql/snapshot/check_db_test.go | 42 - .../e2e/pg2mysql/snapshot/dump/type_check.sql | 90 - tests/e2e/pg2pg/access/check_db_test.go | 124 - tests/e2e/pg2pg/access/dump/dump.sql | 16 - tests/e2e/pg2pg/all_types/check_db_test.go | 88 - tests/e2e/pg2pg/bytea_key/check_db_test.go | 68 - .../e2e/pg2pg/bytea_key/init_source/dump.sql | 4 - .../e2e/pg2pg/bytea_key/init_target/dump.sql | 4 - .../debezium/all_datatypes/check_db_test.go | 178 - .../all_datatypes/init_source/dump.sql | 213 - .../all_datatypes/init_target/init.sql | 3 - .../all_datatypes_arr/check_db_test.go | 124 - .../all_datatypes_arr/init_source/dump.sql | 170 - .../all_datatypes_arr/init_target/init.sql | 3 - .../all_datatypes_nohomo/check_db_test.go | 163 - .../all_datatypes_nohomo/init_source/dump.sql | 213 - .../all_datatypes_nohomo/init_target/init.sql | 3 - .../all_datatypes_nohomo_arr/check_db_test.go | 124 - .../init_source/dump.sql | 170 - .../init_target/init.sql | 3 - .../all_datatypes_serde/check_db_test.go | 202 - .../all_datatypes_serde/init_source/dump.sql | 213 - .../all_datatypes_serde/init_target/init.sql | 3 - .../all_datatypes_serde_arr/check_db_test.go | 162 - .../init_source/dump.sql | 170 - .../init_target/init.sql | 3 - .../check_db_test.go | 142 - .../init_source/dump.sql | 170 - .../init_target/init.sql | 3 - .../check_db_test.go | 184 - .../init_source/dump.sql | 170 - .../init_target/init.sql | 3 - .../check_db_test.go | 181 - .../init_source/dump.sql | 213 - .../init_target/init.sql | 3 - .../check_db_test.go | 80 - .../init_source/dump.sql | 209 - .../init_target/init.sql | 3 - .../check_db_test.go | 246 - .../init_source/dump.sql | 213 - .../init_target/init.sql | 3 - .../check_db_test.go | 244 - .../init_source/dump.sql | 213 - .../init_target/init.sql | 3 - .../check_db_test.go | 82 - .../init_source/dump.sql | 19 - .../check_db_test.go | 93 - .../init_source/dump.sql | 32 - tests/e2e/pg2pg/drop_tables/drop_test.go | 290 - tests/e2e/pg2pg/drop_tables/dump/snapshot.sql | 41 - .../e2e/pg2pg/drop_tables/dump_1/snapshot.sql | 18 - .../enum_with_fallbacks/check_db_test.go | 53 - .../enum_with_fallbacks/init_dst/init.sql | 6 - .../enum_with_fallbacks/init_src/init.sql | 8 - .../insufficient_privileges/check_db_test.go | 105 - .../init_source/init.sql | 18 - .../e2e/pg2pg/insufficient_privileges/util.go | 82 - tests/e2e/pg2pg/jsonb/check_db_test.go | 75 - tests/e2e/pg2pg/jsonb/init_source/init.sql | 14 - tests/e2e/pg2pg/jsonb/init_target/init.sql | 6 - tests/e2e/pg2pg/multiindex/check_db_test.go | 156 - .../e2e/pg2pg/multiindex/init_source/dump.sql | 6 - .../e2e/pg2pg/multiindex/init_target/dump.sql | 6 - .../pg2pg/namesake_tables/check_db_test.go | 27 - .../pg2pg/namesake_tables/dump/type_check.sql | 15 - .../null_temporals_tsv_1/check_db_test.go | 37 - .../pg2pg/null_temporals_tsv_1/dump/dump.sql | 7 - .../all_parts/dump/initial.sql | 105 - .../all_parts/partitioned_tables_test.go | 227 - .../dump/initial.sql | 107 - .../partitioned_tables_test.go | 227 - .../some_parts/dump/initial.sql | 81 - .../some_parts/partitioned_tables_test.go | 205 - tests/e2e/pg2pg/pg_dump/check_db_test.go | 117 - tests/e2e/pg2pg/pg_dump/dump/type_check.sql | 68 - tests/e2e/pg2pg/pkey_update/check_db_test.go | 48 - .../pg2pg/pkey_update/init_source/dump.sql | 12 - .../pg2pg/pkey_update/init_target/dump.sql | 6 - tests/e2e/pg2pg/replication/check_db_test.go | 110 - .../e2e/pg2pg/replication/dump/type_check.sql | 421 - .../check_db_test.go | 86 - .../replication_replica_identity/helpers.go | 83 - .../init_source/dump.sql | 63 - .../init_target/dump.sql | 20 - .../check_db_test.go | 43 - .../init_source/dump.sql | 9 - .../pg2pg/replication_toast/check_db_test.go | 166 - .../replication_toast/init_source/dump.sql | 22 - .../replication_toast/init_target/dump.sql | 13 - .../pg2pg/replication_view/check_db_test.go | 42 - .../replication_view/init_source/dump.sql | 19 - .../replication_view/init_target/dump.sql | 7 - tests/e2e/pg2pg/snapshot/check_db_test.go | 84 - tests/e2e/pg2pg/snapshot/dump/type_check.sql | 172 - .../snapshot_missing_public/check_db_test.go | 79 - .../snapshot_missing_public/dump/dump.sql | 11 - .../table_capital_letter/check_db_test.go | 36 - .../table_capital_letter/dump/type_check.sql | 5 - .../pg2pg/time_with_fallback/check_db_test.go | 53 - .../time_with_fallback/init_source/init.sql | 5 - .../time_with_fallback/init_target/init.sql | 4 - .../e2e/pg2pg/tx_boundaries/check_db_test.go | 95 - .../pg2pg/tx_boundaries/dump/type_check.sql | 5 - .../e2e/pg2pg/unusual_dates/check_db_test.go | 76 - tests/e2e/pg2pg/unusual_dates/dump/dump.sql | 18 - tests/e2e/pg2pg/user_types/check_db_test.go | 79 - .../e2e/pg2pg/user_types/init_source/init.sql | 34 - .../sample2ch/replication/check_db_test.go | 55 - tests/e2e/sample2ch/replication/dump/dst.sql | 1 - .../testdata/change_item_key.txt | 17 - .../testdata/change_item_val.txt | 456 - 521 files changed, 93437 deletions(-) delete mode 100644 pkg/dataplane/provideradapter/airbyte/adapter.go delete mode 100644 pkg/dataplane/provideradapter/clickhouse/adapter.go delete mode 100644 pkg/dataplane/provideradapter/elastic/adapter.go delete mode 100644 pkg/dataplane/provideradapter/eventhub/adapter.go delete mode 100644 pkg/dataplane/provideradapter/kafka/adapter.go delete mode 100644 pkg/dataplane/provideradapter/logbroker/adapter.go delete mode 100644 pkg/dataplane/provideradapter/mongo/adapter.go delete mode 100644 pkg/dataplane/provideradapter/mysql/adapter.go delete mode 100644 pkg/dataplane/provideradapter/postgres/adapter.go delete mode 100644 pkg/dataplane/provideradapter/s3/adapter.go delete mode 100644 pkg/dataplane/provideradapter/ydb/adapter.go delete mode 100644 pkg/dataplane/provideradapter/yds/adapter.go delete mode 100644 pkg/dataplane/provideradapter/yt/adapter.go delete mode 100644 recipe/mongo/README.md delete mode 100644 recipe/mongo/cmd/binurl/README.md delete mode 100644 recipe/mongo/cmd/binurl/binary_fetcher.go delete mode 100644 recipe/mongo/example/configs/auth.yaml delete mode 100644 recipe/mongo/example/launch_cluster/README.md delete mode 100644 recipe/mongo/example/launch_cluster/main.go delete mode 100644 recipe/mongo/example/recipe_usage/README.md delete mode 100644 recipe/mongo/example/recipe_usage/sample_test.go delete mode 100644 recipe/mongo/pkg/binurl/binary_links.go delete mode 100644 recipe/mongo/pkg/cluster/cluster.go delete mode 100644 recipe/mongo/pkg/cluster/config_replica_set.go delete mode 100644 recipe/mongo/pkg/cluster/environment_info.go delete mode 100644 recipe/mongo/pkg/cluster/mongod.go delete mode 100644 recipe/mongo/pkg/cluster/mongos.go delete mode 100644 recipe/mongo/pkg/cluster/shard_replica_set.go delete mode 100644 recipe/mongo/pkg/config/config.go delete mode 100644 recipe/mongo/pkg/tar/tar.go delete mode 100644 recipe/mongo/pkg/util/test_common.go delete mode 100644 recipe/mongo/pkg/util/yatest.go delete mode 100644 recipe/mongo/recipe.go delete mode 100644 recipe/mongo/test/4.4/cluster_test.go delete mode 100644 recipe/mongo/test/4.4/mongocluster.yaml delete mode 100644 recipe/mongo/test/5.0/cluster_test.go delete mode 100644 recipe/mongo/test/5.0/mongocluster.yaml delete mode 100644 recipe/mongo/test/6.0/cluster_test.go delete mode 100644 recipe/mongo/test/6.0/mongocluster.yaml delete mode 100644 tests/canon/all_db_test.go delete mode 100644 tests/canon/clickhouse/README.md delete mode 100644 tests/canon/clickhouse/canon_test.go delete mode 100644 tests/canon/clickhouse/snapshot/data.sql delete mode 100644 tests/canon/gotest/canondata/result.json delete mode 100644 tests/canon/mongo/README.md delete mode 100644 tests/canon/mongo/canon_docs.go delete mode 100644 tests/canon/mongo/canon_test.go delete mode 100644 tests/canon/mongo/gotest/canondata/result.json delete mode 100644 tests/canon/mysql/canon_sql.go delete mode 100644 tests/canon/mysql/canon_test.go delete mode 100644 tests/canon/mysql/canondata/result.json delete mode 100644 tests/canon/mysql/dump/date_types.sql delete mode 100644 tests/canon/mysql/dump/initial_data.sql delete mode 100644 tests/canon/mysql/dump/json_types.sql delete mode 100644 tests/canon/mysql/dump/numeric_types.sql delete mode 100644 tests/canon/mysql/dump/numeric_types_bit.sql delete mode 100644 tests/canon/mysql/dump/numeric_types_boolean.sql delete mode 100644 tests/canon/mysql/dump/numeric_types_decimal.sql delete mode 100644 tests/canon/mysql/dump/numeric_types_float.sql delete mode 100644 tests/canon/mysql/dump/numeric_types_int.sql delete mode 100644 tests/canon/mysql/dump/spatial_types.sql delete mode 100644 tests/canon/mysql/dump/string_types.sql delete mode 100644 tests/canon/mysql/dump/string_types_emoji.sql delete mode 100644 tests/canon/parser/README.md delete mode 100644 tests/canon/parser/canon_static_generic_test.go delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_json_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_mdb_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_taxi_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tskv_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted delete mode 100644 tests/canon/parser/gotest/canondata/result.json delete mode 100644 tests/canon/parser/samples/dynamic/sample_proto/sample_proto/README.MD delete mode 100644 tests/canon/parser/samples/dynamic/sample_proto/sample_proto/sample_proto.proto delete mode 100644 tests/canon/parser/samples/dynamic/sample_proto/test_case.go delete mode 100644 tests/canon/parser/samples/static/generic/json.config.json delete mode 100644 tests/canon/parser/samples/static/generic/json.sample delete mode 100644 tests/canon/parser/samples/static/generic/mdb.config.json delete mode 100644 tests/canon/parser/samples/static/generic/mdb.sample delete mode 100644 tests/canon/parser/samples/static/generic/metrika.config.json delete mode 100644 tests/canon/parser/samples/static/generic/metrika.sample delete mode 100644 tests/canon/parser/samples/static/generic/metrika_complex.config.json delete mode 100644 tests/canon/parser/samples/static/generic/metrika_complex.sample delete mode 100644 tests/canon/parser/samples/static/generic/taxi.config.json delete mode 100644 tests/canon/parser/samples/static/generic/taxi.sample delete mode 100644 tests/canon/parser/samples/static/generic/tm-5249.config.json delete mode 100644 tests/canon/parser/samples/static/generic/tm-5249.sample delete mode 100644 tests/canon/parser/samples/static/generic/tskv.config.json delete mode 100644 tests/canon/parser/samples/static/generic/tskv.sample delete mode 100644 tests/canon/parser/samples/static/logfeller/_type_check_rules.yaml delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr-log-2.yaml delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr-log.yaml delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr-new-log.yaml delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr.config.json delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr.sample delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr_new.config.json delete mode 100644 tests/canon/parser/samples/static/logfeller/kikimr_new.sample delete mode 100644 tests/canon/parser/samples/static/logfeller/sensitive.config.json delete mode 100644 tests/canon/parser/samples/static/logfeller/sensitive.sample delete mode 100644 tests/canon/parser/testcase/test_case.go delete mode 100644 tests/canon/postgres/canon_sql.go delete mode 100644 tests/canon/postgres/canon_test.go delete mode 100644 tests/canon/postgres/dump/array_types.sql delete mode 100644 tests/canon/postgres/dump/date_types.sql delete mode 100644 tests/canon/postgres/dump/geom_types.sql delete mode 100644 tests/canon/postgres/dump/numeric_types.sql delete mode 100644 tests/canon/postgres/dump/text_types.sql delete mode 100644 tests/canon/postgres/dump/wtf_types.sql delete mode 100644 tests/canon/postgres/gotest/canondata/result.json delete mode 100644 tests/canon/s3/csv/canon_test.go delete mode 100644 tests/canon/s3/csv/canondata/csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted delete mode 100644 tests/canon/s3/csv/canondata/csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted delete mode 100644 tests/canon/s3/csv/canondata/result.json delete mode 100644 tests/canon/s3/jsonline/canon_test.go delete mode 100644 tests/canon/s3/jsonline/canondata/result.json delete mode 100644 tests/canon/s3/parquet/canon_test.go delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted delete mode 100644 tests/canon/s3/parquet/canondata/result.json delete mode 100644 tests/canon/sequences/README.md delete mode 100644 tests/canon/sequences/canondata/result.json delete mode 100644 tests/canon/sequences/dump/00_insert_update_delete.sql delete mode 100644 tests/canon/sequences/dump/01_updatepk.sql delete mode 100644 tests/canon/sequences/dump/02_insert_update_insert.sql delete mode 100644 tests/canon/sequences/dump/init.insert_update_delete.sql delete mode 100644 tests/canon/sequences/sequences_test.go delete mode 100644 tests/canon/ydb/canon_test.go delete mode 100644 tests/canon/ydb/canondata/result.json delete mode 100644 tests/canon/yt/canon_test.go delete mode 100644 tests/canon/yt/canondata/result.json delete mode 100644 tests/e2e/ch2ch/db_alt_name/check_db_test.go delete mode 100644 tests/e2e/ch2ch/db_alt_name/dump/dst.sql delete mode 100644 tests/e2e/ch2ch/db_alt_name/dump/src.sql delete mode 100644 tests/e2e/ch2ch/db_complex_name/check_db_test.go delete mode 100644 tests/e2e/ch2ch/db_complex_name/dump/dst.sql delete mode 100644 tests/e2e/ch2ch/db_complex_name/dump/src.sql delete mode 100644 tests/e2e/ch2ch/incremental_many_shards/check_db_test.go delete mode 100644 tests/e2e/ch2ch/incremental_many_shards/dump/dst.sql delete mode 100644 tests/e2e/ch2ch/incremental_many_shards/dump/src.sql delete mode 100644 tests/e2e/ch2ch/incremental_one_shard/check_db_test.go delete mode 100644 tests/e2e/ch2ch/incremental_one_shard/dump/dst.sql delete mode 100644 tests/e2e/ch2ch/incremental_one_shard/dump/src.sql delete mode 100644 tests/e2e/ch2ch/snapshot/check_db_test.go delete mode 100644 tests/e2e/ch2ch/snapshot/dump/dst.sql delete mode 100644 tests/e2e/ch2ch/snapshot/dump/src.sql delete mode 100644 tests/e2e/ch2ch/snapshot_test_csv_different_values/check_db_test.go delete mode 100644 tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/dst.sql delete mode 100644 tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/src.sql delete mode 100644 tests/e2e/ch2yt/static_table/check_db_test.go delete mode 100644 tests/e2e/ch2yt/static_table/dump/src.sql delete mode 100644 tests/e2e/kafka2ch/blank_parser/ch_init.sql delete mode 100644 tests/e2e/kafka2ch/blank_parser/check_db_test.go delete mode 100644 tests/e2e/kafka2ch/replication/canondata/replication.replication.TestReplication/extracted delete mode 100644 tests/e2e/kafka2ch/replication/canondata/result.json delete mode 100644 tests/e2e/kafka2ch/replication/check_db_test.go delete mode 100644 tests/e2e/kafka2ch/replication/dump/ch/dump.sql delete mode 100644 tests/e2e/kafka2ch/replication_mv/canondata/replication.replication.TestReplication/extracted delete mode 100644 tests/e2e/kafka2ch/replication_mv/canondata/result.json delete mode 100644 tests/e2e/kafka2ch/replication_mv/check_db_test.go delete mode 100644 tests/e2e/kafka2ch/replication_mv/dump/ch/dump.sql delete mode 100644 tests/e2e/kafka2kafka/metering/canondata/result.json delete mode 100644 tests/e2e/kafka2kafka/metering/metering_test.go delete mode 100644 tests/e2e/kafka2kafka/mirror/mirror_test.go delete mode 100644 tests/e2e/kafka2kafka/multi_topic/canondata/result.json delete mode 100644 tests/e2e/kafka2kafka/multi_topic/mirror_test.go delete mode 100644 tests/e2e/kafka2mongo/replication/check_db_test.go delete mode 100644 tests/e2e/kafka2mongo/replication/dump/date_time.sql delete mode 100644 tests/e2e/kafka2mysql/filter_rows/check_db_test.go delete mode 100644 tests/e2e/kafka2mysql/filter_rows/dump/date_time.sql delete mode 100644 tests/e2e/kafka2mysql/replication/check_db_test.go delete mode 100644 tests/e2e/kafka2mysql/replication/dump/date_time.sql delete mode 100644 tests/e2e/mongo2ch/snapshot/check_db_test.go delete mode 100644 tests/e2e/mongo2ch/snapshot/dump.sql delete mode 100644 tests/e2e/mongo2ch/snapshot_flatten/canondata/result.json delete mode 100644 tests/e2e/mongo2ch/snapshot_flatten/canondata/snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted delete mode 100644 tests/e2e/mongo2ch/snapshot_flatten/check_db_test.go delete mode 100644 tests/e2e/mongo2ch/snapshot_flatten/dump.sql delete mode 100644 tests/e2e/mongo2mock/slots/slot_test.go delete mode 100644 tests/e2e/mongo2mock/tech_db_permission/permission_test.go delete mode 100644 tests/e2e/mongo2mongo/add_db_on_snapshot/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/bson_order/reorder_test.go delete mode 100644 tests/e2e/mongo2mongo/db_rename/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/db_rename_rep/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/replication/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/replication_update_model/check_db_test.go delete mode 100644 tests/e2e/mongo2mongo/rps/replication_source/rps_test.go delete mode 100644 tests/e2e/mongo2mongo/rps/rps.go delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db1.yaml delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db2.yaml delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db1.yaml delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db2.yaml delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go delete mode 100644 tests/e2e/mongo2mongo/snapshot/check_db_test.go delete mode 100644 tests/e2e/mysql2ch/comparators.go delete mode 100644 tests/e2e/mysql2ch/replication/check_db_test.go delete mode 100644 tests/e2e/mysql2ch/replication/dump/ch/dump.sql delete mode 100644 tests/e2e/mysql2ch/replication/dump/mysql/dump.sql delete mode 100644 tests/e2e/mysql2ch/replication_minimal/check_db_test.go delete mode 100644 tests/e2e/mysql2ch/replication_minimal/dump/ch/dump.sql delete mode 100644 tests/e2e/mysql2ch/replication_minimal/dump/mysql/dump.sql delete mode 100644 tests/e2e/mysql2ch/snapshot/check_db_test.go delete mode 100644 tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql delete mode 100644 tests/e2e/mysql2ch/snapshot/dump/mysql/dump.sql delete mode 100644 tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go delete mode 100644 tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql delete mode 100644 tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0 delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1 delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.2 delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.3 delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.4 delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/canondata/result.json delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/check_db_test.go delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/init_source/dump.sql delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql delete mode 100644 tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql delete mode 100644 tests/e2e/mysql2kafka/debezium/snapshot/canondata/result.json delete mode 100644 tests/e2e/mysql2kafka/debezium/snapshot/canondata/snapshot.snapshot.TestSnapshot/extracted delete mode 100644 tests/e2e/mysql2kafka/debezium/snapshot/check_db_test.go delete mode 100644 tests/e2e/mysql2kafka/debezium/snapshot/init_source/dump.sql delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/dump/dump.sql delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_snapshot/check_db_test.go delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_snapshot/dump/dump.sql delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt delete mode 100644 tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt delete mode 100644 tests/e2e/mysql2mock/non_utf8_charset/check_db_test.go delete mode 100644 tests/e2e/mysql2mock/non_utf8_charset/dump/dump.sql delete mode 100644 tests/e2e/mysql2mock/timezone/canondata/result.json delete mode 100644 tests/e2e/mysql2mock/timezone/check_db_test.go delete mode 100644 tests/e2e/mysql2mock/timezone/dump/dump.sql delete mode 100644 tests/e2e/mysql2mock/views/check_db_test.go delete mode 100644 tests/e2e/mysql2mock/views/dump/dump.sql delete mode 100644 tests/e2e/mysql2mysql/alters/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/alters/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/binary/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/binary/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/cascade_deletes/common/test.go delete mode 100644 tests/e2e/mysql2mysql/cascade_deletes/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/cascade_deletes/test_per_table/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/cascade_deletes/test_per_transaction/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/cleanup_tables/cleanup_test.go delete mode 100644 tests/e2e/mysql2mysql/cleanup_tables/source/dump.sql delete mode 100644 tests/e2e/mysql2mysql/cleanup_tables/target/dump.sql delete mode 100644 tests/e2e/mysql2mysql/comment/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/comment/dump/comment.sql delete mode 100644 tests/e2e/mysql2mysql/consistent_snapshot/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/consistent_snapshot/dump/consistent_snapshot.sql delete mode 100644 tests/e2e/mysql2mysql/date_time/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/date_time/dump/date_time.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted delete mode 100644 tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted.0 delete mode 100644 tests/e2e/mysql2mysql/float/canondata/result.json delete mode 100644 tests/e2e/mysql2mysql/float/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/float/dump/dump.sql delete mode 100644 tests/e2e/mysql2mysql/float/increment.sql delete mode 100644 tests/e2e/mysql2mysql/geometry/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/geometry/dump/geometry.sql delete mode 100644 tests/e2e/mysql2mysql/light/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/light/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/light_all_datatypes/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/light_all_datatypes/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/medium/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/no_auto_value_on_zero/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/no_auto_value_on_zero/dump/no_auto_value_on_zero.sql delete mode 100644 tests/e2e/mysql2mysql/partitioned_table/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/partitioned_table/dump/dump.sql delete mode 100644 tests/e2e/mysql2mysql/pkeychanges/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/pkeychanges/dump/type_check.sql delete mode 100644 tests/e2e/mysql2mysql/replace_fkey/common/test.go delete mode 100644 tests/e2e/mysql2mysql/replace_fkey/dump/fkey.sql delete mode 100644 tests/e2e/mysql2mysql/replace_fkey/test_per_table/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/replace_fkey/test_per_transaction/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/scheme/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/scheme/dump/scheme.sql delete mode 100644 tests/e2e/mysql2mysql/snapshot_without_pk/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/snapshot_without_pk/dump/dump.sql delete mode 100644 tests/e2e/mysql2mysql/tx_boundaries/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/tx_boundaries/dump/update.sql delete mode 100644 tests/e2e/mysql2mysql/update/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/update/dump/update.sql delete mode 100644 tests/e2e/mysql2mysql/update_cp1251/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/update_cp1251/dump/update.sql delete mode 100644 tests/e2e/mysql2mysql/update_minimal/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/update_minimal/dump/update_minimal.sql delete mode 100644 tests/e2e/mysql2mysql/update_unicode/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/update_unicode/dump/update.sql delete mode 100644 tests/e2e/mysql2mysql/view/check_db_test.go delete mode 100644 tests/e2e/mysql2mysql/view/dump/update.sql delete mode 100644 tests/e2e/pg2ch/alters/alters_test.go delete mode 100644 tests/e2e/pg2ch/alters/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/alters/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/alters_with_defaults/alters_test.go delete mode 100644 tests/e2e/pg2ch/alters_with_defaults/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/alters_with_defaults/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/chmapper/canondata/chmapper.chmapper.TestSnapshot/extracted delete mode 100644 tests/e2e/pg2ch/chmapper/canondata/result.json delete mode 100644 tests/e2e/pg2ch/chmapper/check_db_test.go delete mode 100644 tests/e2e/pg2ch/chmapper/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/chmapper/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/comparator.go delete mode 100644 tests/e2e/pg2ch/date_overflow/check_db_test.go delete mode 100644 tests/e2e/pg2ch/date_overflow/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/dbt/check_db_test.go delete mode 100644 tests/e2e/pg2ch/dbt/init_ch.sql delete mode 100644 tests/e2e/pg2ch/dbt/init_pg.sql delete mode 100644 tests/e2e/pg2ch/empty_keys/check_db_test.go delete mode 100644 tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/replication/check_db_test.go delete mode 100644 tests/e2e/pg2ch/replication/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/replication/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/replication_mv/check_db_test.go delete mode 100644 tests/e2e/pg2ch/replication_mv/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/replication_mv/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot/check_db_test.go delete mode 100644 tests/e2e/pg2ch/snapshot/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_canon_types/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/check_db_test.go delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_special_values/check_db_test.go delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/check_db_test.go delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshottsv1/check_db_test.go delete mode 100644 tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/tables_inclusion/check_tables_inclusion_test.go delete mode 100644 tests/e2e/pg2ch/tables_inclusion/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/tables_inclusion/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2ch/timestamp/check_db_test.go delete mode 100644 tests/e2e/pg2ch/timestamp/dump/ch/dump.sql delete mode 100644 tests/e2e/pg2ch/timestamp/dump/pg/dump.sql delete mode 100644 tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go delete mode 100644 tests/e2e/pg2kafkamock/debezium_replication/init_source/dump.sql delete mode 100644 tests/e2e/pg2mysql/snapshot/check_db_test.go delete mode 100644 tests/e2e/pg2mysql/snapshot/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/access/check_db_test.go delete mode 100644 tests/e2e/pg2pg/access/dump/dump.sql delete mode 100644 tests/e2e/pg2pg/all_types/check_db_test.go delete mode 100644 tests/e2e/pg2pg/bytea_key/check_db_test.go delete mode 100644 tests/e2e/pg2pg/bytea_key/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/bytea_key/init_target/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_arr/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_arr/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_arr/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_nohomo/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go delete mode 100644 tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/drop_tables/drop_test.go delete mode 100644 tests/e2e/pg2pg/drop_tables/dump/snapshot.sql delete mode 100644 tests/e2e/pg2pg/drop_tables/dump_1/snapshot.sql delete mode 100644 tests/e2e/pg2pg/enum_with_fallbacks/check_db_test.go delete mode 100644 tests/e2e/pg2pg/enum_with_fallbacks/init_dst/init.sql delete mode 100644 tests/e2e/pg2pg/enum_with_fallbacks/init_src/init.sql delete mode 100644 tests/e2e/pg2pg/insufficient_privileges/check_db_test.go delete mode 100644 tests/e2e/pg2pg/insufficient_privileges/init_source/init.sql delete mode 100644 tests/e2e/pg2pg/insufficient_privileges/util.go delete mode 100644 tests/e2e/pg2pg/jsonb/check_db_test.go delete mode 100644 tests/e2e/pg2pg/jsonb/init_source/init.sql delete mode 100644 tests/e2e/pg2pg/jsonb/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/multiindex/check_db_test.go delete mode 100644 tests/e2e/pg2pg/multiindex/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/multiindex/init_target/dump.sql delete mode 100644 tests/e2e/pg2pg/namesake_tables/check_db_test.go delete mode 100644 tests/e2e/pg2pg/namesake_tables/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/null_temporals_tsv_1/check_db_test.go delete mode 100644 tests/e2e/pg2pg/null_temporals_tsv_1/dump/dump.sql delete mode 100644 tests/e2e/pg2pg/partitioned_tables/all_parts/dump/initial.sql delete mode 100644 tests/e2e/pg2pg/partitioned_tables/all_parts/partitioned_tables_test.go delete mode 100644 tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/dump/initial.sql delete mode 100644 tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/partitioned_tables_test.go delete mode 100644 tests/e2e/pg2pg/partitioned_tables/some_parts/dump/initial.sql delete mode 100644 tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go delete mode 100644 tests/e2e/pg2pg/pg_dump/check_db_test.go delete mode 100644 tests/e2e/pg2pg/pg_dump/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/pkey_update/check_db_test.go delete mode 100644 tests/e2e/pg2pg/pkey_update/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/pkey_update/init_target/dump.sql delete mode 100644 tests/e2e/pg2pg/replication/check_db_test.go delete mode 100644 tests/e2e/pg2pg/replication/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/replication_replica_identity/check_db_test.go delete mode 100644 tests/e2e/pg2pg/replication_replica_identity/helpers.go delete mode 100644 tests/e2e/pg2pg/replication_replica_identity/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/replication_replica_identity/init_target/dump.sql delete mode 100644 tests/e2e/pg2pg/replication_special_values/check_db_test.go delete mode 100644 tests/e2e/pg2pg/replication_special_values/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/replication_toast/check_db_test.go delete mode 100644 tests/e2e/pg2pg/replication_toast/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/replication_toast/init_target/dump.sql delete mode 100644 tests/e2e/pg2pg/replication_view/check_db_test.go delete mode 100644 tests/e2e/pg2pg/replication_view/init_source/dump.sql delete mode 100644 tests/e2e/pg2pg/replication_view/init_target/dump.sql delete mode 100644 tests/e2e/pg2pg/snapshot/check_db_test.go delete mode 100644 tests/e2e/pg2pg/snapshot/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go delete mode 100644 tests/e2e/pg2pg/snapshot_missing_public/dump/dump.sql delete mode 100644 tests/e2e/pg2pg/table_capital_letter/check_db_test.go delete mode 100644 tests/e2e/pg2pg/table_capital_letter/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/time_with_fallback/check_db_test.go delete mode 100644 tests/e2e/pg2pg/time_with_fallback/init_source/init.sql delete mode 100644 tests/e2e/pg2pg/time_with_fallback/init_target/init.sql delete mode 100644 tests/e2e/pg2pg/tx_boundaries/check_db_test.go delete mode 100644 tests/e2e/pg2pg/tx_boundaries/dump/type_check.sql delete mode 100644 tests/e2e/pg2pg/unusual_dates/check_db_test.go delete mode 100644 tests/e2e/pg2pg/unusual_dates/dump/dump.sql delete mode 100644 tests/e2e/pg2pg/user_types/check_db_test.go delete mode 100644 tests/e2e/pg2pg/user_types/init_source/init.sql delete mode 100644 tests/e2e/sample2ch/replication/check_db_test.go delete mode 100644 tests/e2e/sample2ch/replication/dump/dst.sql delete mode 100644 tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_key.txt delete mode 100644 tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_val.txt diff --git a/.mapping.json b/.mapping.json index 8b4b21d3..e4349dba 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3276,20 +3276,7 @@ "pkg/csv/splitter.go":"transfer_manager/go/pkg/csv/splitter.go", "pkg/csv/splitter_test.go":"transfer_manager/go/pkg/csv/splitter_test.go", "pkg/data/common.go":"transfer_manager/go/pkg/data/common.go", - "pkg/dataplane/provideradapter/airbyte/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/airbyte/adapter.go", - "pkg/dataplane/provideradapter/clickhouse/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/clickhouse/adapter.go", - "pkg/dataplane/provideradapter/elastic/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/elastic/adapter.go", - "pkg/dataplane/provideradapter/eventhub/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/eventhub/adapter.go", "pkg/dataplane/provideradapter/glue.go":"transfer_manager/go/pkg/dataplane/provideradapter/glue.go", - "pkg/dataplane/provideradapter/kafka/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/kafka/adapter.go", - "pkg/dataplane/provideradapter/logbroker/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/logbroker/adapter.go", - "pkg/dataplane/provideradapter/mongo/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/mongo/adapter.go", - "pkg/dataplane/provideradapter/mysql/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/mysql/adapter.go", - "pkg/dataplane/provideradapter/postgres/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/postgres/adapter.go", - "pkg/dataplane/provideradapter/s3/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/s3/adapter.go", - "pkg/dataplane/provideradapter/ydb/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/ydb/adapter.go", - "pkg/dataplane/provideradapter/yds/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/yds/adapter.go", - "pkg/dataplane/provideradapter/yt/adapter.go":"transfer_manager/go/pkg/dataplane/provideradapter/yt/adapter.go", "pkg/dataplane/providers.go":"transfer_manager/go/pkg/dataplane/providers.go", "pkg/dataplane/transformer.go":"transfer_manager/go/pkg/dataplane/transformer.go", "pkg/dbaas/abstract.go":"transfer_manager/go/pkg/dbaas/abstract.go", @@ -4695,47 +4682,10 @@ "pkg/yc/user_account.go":"transfer_manager/go/pkg/yc/user_account.go", "pkg/yc/vpc.go":"transfer_manager/go/pkg/yc/vpc.go", "pkg/yc/ydb.go":"transfer_manager/go/pkg/yc/ydb.go", - "recipe/mongo/README.md":"transfer_manager/go/recipe/mongo/README.md", - "recipe/mongo/cmd/binurl/README.md":"transfer_manager/go/recipe/mongo/cmd/binurl/README.md", - "recipe/mongo/cmd/binurl/binary_fetcher.go":"transfer_manager/go/recipe/mongo/cmd/binurl/binary_fetcher.go", - "recipe/mongo/example/configs/auth.yaml":"transfer_manager/go/recipe/mongo/example/configs/auth.yaml", - "recipe/mongo/example/launch_cluster/README.md":"transfer_manager/go/recipe/mongo/example/launch_cluster/README.md", - "recipe/mongo/example/launch_cluster/main.go":"transfer_manager/go/recipe/mongo/example/launch_cluster/main.go", - "recipe/mongo/example/recipe_usage/README.md":"transfer_manager/go/recipe/mongo/example/recipe_usage/README.md", - "recipe/mongo/example/recipe_usage/sample_test.go":"transfer_manager/go/recipe/mongo/example/recipe_usage/sample_test.go", - "recipe/mongo/pkg/binurl/binary_links.go":"transfer_manager/go/recipe/mongo/pkg/binurl/binary_links.go", - "recipe/mongo/pkg/cluster/cluster.go":"transfer_manager/go/recipe/mongo/pkg/cluster/cluster.go", - "recipe/mongo/pkg/cluster/config_replica_set.go":"transfer_manager/go/recipe/mongo/pkg/cluster/config_replica_set.go", - "recipe/mongo/pkg/cluster/environment_info.go":"transfer_manager/go/recipe/mongo/pkg/cluster/environment_info.go", - "recipe/mongo/pkg/cluster/mongod.go":"transfer_manager/go/recipe/mongo/pkg/cluster/mongod.go", - "recipe/mongo/pkg/cluster/mongos.go":"transfer_manager/go/recipe/mongo/pkg/cluster/mongos.go", - "recipe/mongo/pkg/cluster/shard_replica_set.go":"transfer_manager/go/recipe/mongo/pkg/cluster/shard_replica_set.go", - "recipe/mongo/pkg/config/config.go":"transfer_manager/go/recipe/mongo/pkg/config/config.go", - "recipe/mongo/pkg/tar/tar.go":"transfer_manager/go/recipe/mongo/pkg/tar/tar.go", - "recipe/mongo/pkg/util/test_common.go":"transfer_manager/go/recipe/mongo/pkg/util/test_common.go", - "recipe/mongo/pkg/util/yatest.go":"transfer_manager/go/recipe/mongo/pkg/util/yatest.go", - "recipe/mongo/recipe.go":"transfer_manager/go/recipe/mongo/recipe.go", - "recipe/mongo/test/4.4/cluster_test.go":"transfer_manager/go/recipe/mongo/test/4.4/cluster_test.go", - "recipe/mongo/test/4.4/mongocluster.yaml":"transfer_manager/go/recipe/mongo/test/4.4/mongocluster.yaml", - "recipe/mongo/test/5.0/cluster_test.go":"transfer_manager/go/recipe/mongo/test/5.0/cluster_test.go", - "recipe/mongo/test/5.0/mongocluster.yaml":"transfer_manager/go/recipe/mongo/test/5.0/mongocluster.yaml", - "recipe/mongo/test/6.0/cluster_test.go":"transfer_manager/go/recipe/mongo/test/6.0/cluster_test.go", - "recipe/mongo/test/6.0/mongocluster.yaml":"transfer_manager/go/recipe/mongo/test/6.0/mongocluster.yaml", "tests/canon/all_databases.go":"transfer_manager/go/tests/canon/all_databases.go", - "tests/canon/all_db_test.go":"transfer_manager/go/tests/canon/all_db_test.go", "tests/canon/all_replication_sequences.go":"transfer_manager/go/tests/canon/all_replication_sequences.go", - "tests/canon/clickhouse/README.md":"transfer_manager/go/tests/canon/clickhouse/README.md", - "tests/canon/clickhouse/canon_test.go":"transfer_manager/go/tests/canon/clickhouse/canon_test.go", "tests/canon/clickhouse/canondata/clickhouse.clickhouse.TestCanonSource_canon_0#01/extracted":"transfer_manager/go/tests/canon/clickhouse/canondata/clickhouse.clickhouse.TestCanonSource_canon_0#01/extracted", "tests/canon/clickhouse/canondata/result.json":"transfer_manager/go/tests/canon/clickhouse/canondata/result.json", - "tests/canon/clickhouse/snapshot/data.sql":"transfer_manager/go/tests/canon/clickhouse/snapshot/data.sql", - "tests/canon/gotest/canondata/result.json":"transfer_manager/go/tests/canon/gotest/canondata/result.json", - "tests/canon/mongo/README.md":"transfer_manager/go/tests/canon/mongo/README.md", - "tests/canon/mongo/canon_docs.go":"transfer_manager/go/tests/canon/mongo/canon_docs.go", - "tests/canon/mongo/canon_test.go":"transfer_manager/go/tests/canon/mongo/canon_test.go", - "tests/canon/mongo/gotest/canondata/result.json":"transfer_manager/go/tests/canon/mongo/gotest/canondata/result.json", - "tests/canon/mysql/canon_sql.go":"transfer_manager/go/tests/canon/mysql/canon_sql.go", - "tests/canon/mysql/canon_test.go":"transfer_manager/go/tests/canon/mysql/canon_test.go", "tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_initial_canon_0#01/extracted":"transfer_manager/go/tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_initial_canon_0#01/extracted", "tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_initial_canon_0#03/extracted":"transfer_manager/go/tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_initial_canon_0#03/extracted", "tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_json_types_canon_0#01/extracted":"transfer_manager/go/tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_json_types_canon_0#01/extracted", @@ -4752,69 +4702,6 @@ "tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_numeric_types_int_canon_0#03/extracted":"transfer_manager/go/tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_numeric_types_int_canon_0#03/extracted", "tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_string_types_emoji_canon_0#01/extracted":"transfer_manager/go/tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_string_types_emoji_canon_0#01/extracted", "tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_string_types_emoji_canon_0#03/extracted":"transfer_manager/go/tests/canon/mysql/canondata/mysql.mysql.TestCanonSource_string_types_emoji_canon_0#03/extracted", - "tests/canon/mysql/canondata/result.json":"transfer_manager/go/tests/canon/mysql/canondata/result.json", - "tests/canon/mysql/dump/date_types.sql":"transfer_manager/go/tests/canon/mysql/dump/date_types.sql", - "tests/canon/mysql/dump/initial_data.sql":"transfer_manager/go/tests/canon/mysql/dump/initial_data.sql", - "tests/canon/mysql/dump/json_types.sql":"transfer_manager/go/tests/canon/mysql/dump/json_types.sql", - "tests/canon/mysql/dump/numeric_types.sql":"transfer_manager/go/tests/canon/mysql/dump/numeric_types.sql", - "tests/canon/mysql/dump/numeric_types_bit.sql":"transfer_manager/go/tests/canon/mysql/dump/numeric_types_bit.sql", - "tests/canon/mysql/dump/numeric_types_boolean.sql":"transfer_manager/go/tests/canon/mysql/dump/numeric_types_boolean.sql", - "tests/canon/mysql/dump/numeric_types_decimal.sql":"transfer_manager/go/tests/canon/mysql/dump/numeric_types_decimal.sql", - "tests/canon/mysql/dump/numeric_types_float.sql":"transfer_manager/go/tests/canon/mysql/dump/numeric_types_float.sql", - "tests/canon/mysql/dump/numeric_types_int.sql":"transfer_manager/go/tests/canon/mysql/dump/numeric_types_int.sql", - "tests/canon/mysql/dump/spatial_types.sql":"transfer_manager/go/tests/canon/mysql/dump/spatial_types.sql", - "tests/canon/mysql/dump/string_types.sql":"transfer_manager/go/tests/canon/mysql/dump/string_types.sql", - "tests/canon/mysql/dump/string_types_emoji.sql":"transfer_manager/go/tests/canon/mysql/dump/string_types_emoji.sql", - "tests/canon/parser/README.md":"transfer_manager/go/tests/canon/parser/README.md", - "tests/canon/parser/canon_static_generic_test.go":"transfer_manager/go/tests/canon/parser/canon_static_generic_test.go", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_json_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_json_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_mdb_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_mdb_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_taxi_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_taxi_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tskv_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tskv_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted", - "tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted":"transfer_manager/go/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted", - "tests/canon/parser/gotest/canondata/result.json":"transfer_manager/go/tests/canon/parser/gotest/canondata/result.json", - "tests/canon/parser/samples/dynamic/sample_proto/sample_proto/README.MD":"transfer_manager/go/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/README.MD", - "tests/canon/parser/samples/dynamic/sample_proto/sample_proto/sample_proto.proto":"transfer_manager/go/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/sample_proto.proto", - "tests/canon/parser/samples/dynamic/sample_proto/test_case.go":"transfer_manager/go/tests/canon/parser/samples/dynamic/sample_proto/test_case.go", - "tests/canon/parser/samples/static/generic/json.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/json.config.json", - "tests/canon/parser/samples/static/generic/json.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/json.sample", - "tests/canon/parser/samples/static/generic/mdb.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/mdb.config.json", - "tests/canon/parser/samples/static/generic/mdb.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/mdb.sample", - "tests/canon/parser/samples/static/generic/metrika.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/metrika.config.json", - "tests/canon/parser/samples/static/generic/metrika.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/metrika.sample", - "tests/canon/parser/samples/static/generic/metrika_complex.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/metrika_complex.config.json", - "tests/canon/parser/samples/static/generic/metrika_complex.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/metrika_complex.sample", - "tests/canon/parser/samples/static/generic/taxi.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/taxi.config.json", - "tests/canon/parser/samples/static/generic/taxi.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/taxi.sample", - "tests/canon/parser/samples/static/generic/tm-5249.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/tm-5249.config.json", - "tests/canon/parser/samples/static/generic/tm-5249.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/tm-5249.sample", - "tests/canon/parser/samples/static/generic/tskv.config.json":"transfer_manager/go/tests/canon/parser/samples/static/generic/tskv.config.json", - "tests/canon/parser/samples/static/generic/tskv.sample":"transfer_manager/go/tests/canon/parser/samples/static/generic/tskv.sample", - "tests/canon/parser/samples/static/logfeller/_type_check_rules.yaml":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/_type_check_rules.yaml", - "tests/canon/parser/samples/static/logfeller/kikimr-log-2.yaml":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr-log-2.yaml", - "tests/canon/parser/samples/static/logfeller/kikimr-log.yaml":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr-log.yaml", - "tests/canon/parser/samples/static/logfeller/kikimr-new-log.yaml":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr-new-log.yaml", - "tests/canon/parser/samples/static/logfeller/kikimr.config.json":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr.config.json", - "tests/canon/parser/samples/static/logfeller/kikimr.sample":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr.sample", - "tests/canon/parser/samples/static/logfeller/kikimr_new.config.json":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr_new.config.json", - "tests/canon/parser/samples/static/logfeller/kikimr_new.sample":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/kikimr_new.sample", - "tests/canon/parser/samples/static/logfeller/sensitive.config.json":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/sensitive.config.json", - "tests/canon/parser/samples/static/logfeller/sensitive.sample":"transfer_manager/go/tests/canon/parser/samples/static/logfeller/sensitive.sample", - "tests/canon/parser/testcase/test_case.go":"transfer_manager/go/tests/canon/parser/testcase/test_case.go", - "tests/canon/postgres/canon_sql.go":"transfer_manager/go/tests/canon/postgres/canon_sql.go", - "tests/canon/postgres/canon_test.go":"transfer_manager/go/tests/canon/postgres/canon_test.go", - "tests/canon/postgres/dump/array_types.sql":"transfer_manager/go/tests/canon/postgres/dump/array_types.sql", - "tests/canon/postgres/dump/date_types.sql":"transfer_manager/go/tests/canon/postgres/dump/date_types.sql", - "tests/canon/postgres/dump/geom_types.sql":"transfer_manager/go/tests/canon/postgres/dump/geom_types.sql", - "tests/canon/postgres/dump/numeric_types.sql":"transfer_manager/go/tests/canon/postgres/dump/numeric_types.sql", - "tests/canon/postgres/dump/text_types.sql":"transfer_manager/go/tests/canon/postgres/dump/text_types.sql", - "tests/canon/postgres/dump/wtf_types.sql":"transfer_manager/go/tests/canon/postgres/dump/wtf_types.sql", "tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_array_types_canon_0#01/extracted":"transfer_manager/go/tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_array_types_canon_0#01/extracted", "tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_array_types_canon_0#03/extracted":"transfer_manager/go/tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_array_types_canon_0#03/extracted", "tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_date_types_canon_0#01/extracted":"transfer_manager/go/tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_date_types_canon_0#01/extracted", @@ -4827,58 +4714,12 @@ "tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_text_types_canon_0#03/extracted":"transfer_manager/go/tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_text_types_canon_0#03/extracted", "tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_wtf_types_canon_0#01/extracted":"transfer_manager/go/tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_wtf_types_canon_0#01/extracted", "tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_wtf_types_canon_0#03/extracted":"transfer_manager/go/tests/canon/postgres/gotest/canondata/gotest.gotest.TestCanonSource_wtf_types_canon_0#03/extracted", - "tests/canon/postgres/gotest/canondata/result.json":"transfer_manager/go/tests/canon/postgres/gotest/canondata/result.json", "tests/canon/reference/dump.go":"transfer_manager/go/tests/canon/reference/dump.go", "tests/canon/reference/reference.go":"transfer_manager/go/tests/canon/reference/reference.go", "tests/canon/reference/table.go":"transfer_manager/go/tests/canon/reference/table.go", - "tests/canon/s3/csv/canon_test.go":"transfer_manager/go/tests/canon/s3/csv/canon_test.go", - "tests/canon/s3/csv/canondata/csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted":"transfer_manager/go/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted", - "tests/canon/s3/csv/canondata/csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted":"transfer_manager/go/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted", - "tests/canon/s3/csv/canondata/result.json":"transfer_manager/go/tests/canon/s3/csv/canondata/result.json", - "tests/canon/s3/jsonline/canon_test.go":"transfer_manager/go/tests/canon/s3/jsonline/canon_test.go", - "tests/canon/s3/jsonline/canondata/result.json":"transfer_manager/go/tests/canon/s3/jsonline/canondata/result.json", - "tests/canon/s3/parquet/canon_test.go":"transfer_manager/go/tests/canon/s3/parquet/canon_test.go", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted":"transfer_manager/go/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted", - "tests/canon/s3/parquet/canondata/result.json":"transfer_manager/go/tests/canon/s3/parquet/canondata/result.json", - "tests/canon/sequences/README.md":"transfer_manager/go/tests/canon/sequences/README.md", - "tests/canon/sequences/canondata/result.json":"transfer_manager/go/tests/canon/sequences/canondata/result.json", "tests/canon/sequences/canondata/sequences.sequences.TestCanonizeSequences_insert_update_delete_canon_0/extracted":"transfer_manager/go/tests/canon/sequences/canondata/sequences.sequences.TestCanonizeSequences_insert_update_delete_canon_0/extracted", "tests/canon/sequences/canondata/sequences.sequences.TestCanonizeSequences_insert_update_insert_canon_0/extracted":"transfer_manager/go/tests/canon/sequences/canondata/sequences.sequences.TestCanonizeSequences_insert_update_insert_canon_0/extracted", "tests/canon/sequences/canondata/sequences.sequences.TestCanonizeSequences_updatepk_canon_0/extracted":"transfer_manager/go/tests/canon/sequences/canondata/sequences.sequences.TestCanonizeSequences_updatepk_canon_0/extracted", - "tests/canon/sequences/dump/00_insert_update_delete.sql":"transfer_manager/go/tests/canon/sequences/dump/00_insert_update_delete.sql", - "tests/canon/sequences/dump/01_updatepk.sql":"transfer_manager/go/tests/canon/sequences/dump/01_updatepk.sql", - "tests/canon/sequences/dump/02_insert_update_insert.sql":"transfer_manager/go/tests/canon/sequences/dump/02_insert_update_insert.sql", - "tests/canon/sequences/dump/init.insert_update_delete.sql":"transfer_manager/go/tests/canon/sequences/dump/init.insert_update_delete.sql", - "tests/canon/sequences/sequences_test.go":"transfer_manager/go/tests/canon/sequences/sequences_test.go", "tests/canon/validator/aggregator.go":"transfer_manager/go/tests/canon/validator/aggregator.go", "tests/canon/validator/canonizator.go":"transfer_manager/go/tests/canon/validator/canonizator.go", "tests/canon/validator/counter.go":"transfer_manager/go/tests/canon/validator/counter.go", @@ -4887,370 +4728,10 @@ "tests/canon/validator/sequencer.go":"transfer_manager/go/tests/canon/validator/sequencer.go", "tests/canon/validator/typesystem.go":"transfer_manager/go/tests/canon/validator/typesystem.go", "tests/canon/validator/values_type_checker.go":"transfer_manager/go/tests/canon/validator/values_type_checker.go", - "tests/canon/ydb/canon_test.go":"transfer_manager/go/tests/canon/ydb/canon_test.go", - "tests/canon/ydb/canondata/result.json":"transfer_manager/go/tests/canon/ydb/canondata/result.json", "tests/canon/ydb/canondata/ydb.ydb.TestCanonSource_canon_0#01/extracted":"transfer_manager/go/tests/canon/ydb/canondata/ydb.ydb.TestCanonSource_canon_0#01/extracted", - "tests/canon/yt/canon_test.go":"transfer_manager/go/tests/canon/yt/canon_test.go", - "tests/canon/yt/canondata/result.json":"transfer_manager/go/tests/canon/yt/canondata/result.json", "tests/canon/yt/canondata/yt.yt.TestCanonSourceWithDataObjects_canon_0/extracted":"transfer_manager/go/tests/canon/yt/canondata/yt.yt.TestCanonSourceWithDataObjects_canon_0/extracted", "tests/canon/yt/canondata/yt.yt.TestCanonSourceWithDirInDataObjects_canon_0/extracted":"transfer_manager/go/tests/canon/yt/canondata/yt.yt.TestCanonSourceWithDirInDataObjects_canon_0/extracted", "tests/canon/yt/canondata/yt.yt.TestCanonSource_canon_0/extracted":"transfer_manager/go/tests/canon/yt/canondata/yt.yt.TestCanonSource_canon_0/extracted", - "tests/e2e/ch2ch/db_alt_name/check_db_test.go":"transfer_manager/go/tests/e2e/ch2ch/db_alt_name/check_db_test.go", - "tests/e2e/ch2ch/db_alt_name/dump/dst.sql":"transfer_manager/go/tests/e2e/ch2ch/db_alt_name/dump/dst.sql", - "tests/e2e/ch2ch/db_alt_name/dump/src.sql":"transfer_manager/go/tests/e2e/ch2ch/db_alt_name/dump/src.sql", - "tests/e2e/ch2ch/db_complex_name/check_db_test.go":"transfer_manager/go/tests/e2e/ch2ch/db_complex_name/check_db_test.go", - "tests/e2e/ch2ch/db_complex_name/dump/dst.sql":"transfer_manager/go/tests/e2e/ch2ch/db_complex_name/dump/dst.sql", - "tests/e2e/ch2ch/db_complex_name/dump/src.sql":"transfer_manager/go/tests/e2e/ch2ch/db_complex_name/dump/src.sql", - "tests/e2e/ch2ch/incremental_many_shards/check_db_test.go":"transfer_manager/go/tests/e2e/ch2ch/incremental_many_shards/check_db_test.go", - "tests/e2e/ch2ch/incremental_many_shards/dump/dst.sql":"transfer_manager/go/tests/e2e/ch2ch/incremental_many_shards/dump/dst.sql", - "tests/e2e/ch2ch/incremental_many_shards/dump/src.sql":"transfer_manager/go/tests/e2e/ch2ch/incremental_many_shards/dump/src.sql", - "tests/e2e/ch2ch/incremental_one_shard/check_db_test.go":"transfer_manager/go/tests/e2e/ch2ch/incremental_one_shard/check_db_test.go", - "tests/e2e/ch2ch/incremental_one_shard/dump/dst.sql":"transfer_manager/go/tests/e2e/ch2ch/incremental_one_shard/dump/dst.sql", - "tests/e2e/ch2ch/incremental_one_shard/dump/src.sql":"transfer_manager/go/tests/e2e/ch2ch/incremental_one_shard/dump/src.sql", - "tests/e2e/ch2ch/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/ch2ch/snapshot/check_db_test.go", - "tests/e2e/ch2ch/snapshot/dump/dst.sql":"transfer_manager/go/tests/e2e/ch2ch/snapshot/dump/dst.sql", - "tests/e2e/ch2ch/snapshot/dump/src.sql":"transfer_manager/go/tests/e2e/ch2ch/snapshot/dump/src.sql", - "tests/e2e/ch2ch/snapshot_test_csv_different_values/check_db_test.go":"transfer_manager/go/tests/e2e/ch2ch/snapshot_test_csv_different_values/check_db_test.go", - "tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/dst.sql":"transfer_manager/go/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/dst.sql", - "tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/src.sql":"transfer_manager/go/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/src.sql", - "tests/e2e/ch2yt/static_table/check_db_test.go":"transfer_manager/go/tests/e2e/ch2yt/static_table/check_db_test.go", - "tests/e2e/ch2yt/static_table/dump/src.sql":"transfer_manager/go/tests/e2e/ch2yt/static_table/dump/src.sql", - "tests/e2e/kafka2ch/blank_parser/ch_init.sql":"transfer_manager/go/tests/e2e/kafka2ch/blank_parser/ch_init.sql", - "tests/e2e/kafka2ch/blank_parser/check_db_test.go":"transfer_manager/go/tests/e2e/kafka2ch/blank_parser/check_db_test.go", - "tests/e2e/kafka2ch/replication/canondata/replication.replication.TestReplication/extracted":"transfer_manager/go/tests/e2e/kafka2ch/replication/canondata/replication.replication.TestReplication/extracted", - "tests/e2e/kafka2ch/replication/canondata/result.json":"transfer_manager/go/tests/e2e/kafka2ch/replication/canondata/result.json", - "tests/e2e/kafka2ch/replication/check_db_test.go":"transfer_manager/go/tests/e2e/kafka2ch/replication/check_db_test.go", - "tests/e2e/kafka2ch/replication/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/kafka2ch/replication/dump/ch/dump.sql", - "tests/e2e/kafka2ch/replication_mv/canondata/replication.replication.TestReplication/extracted":"transfer_manager/go/tests/e2e/kafka2ch/replication_mv/canondata/replication.replication.TestReplication/extracted", - "tests/e2e/kafka2ch/replication_mv/canondata/result.json":"transfer_manager/go/tests/e2e/kafka2ch/replication_mv/canondata/result.json", - "tests/e2e/kafka2ch/replication_mv/check_db_test.go":"transfer_manager/go/tests/e2e/kafka2ch/replication_mv/check_db_test.go", - "tests/e2e/kafka2ch/replication_mv/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/kafka2ch/replication_mv/dump/ch/dump.sql", - "tests/e2e/kafka2kafka/metering/canondata/result.json":"transfer_manager/go/tests/e2e/kafka2kafka/metering/canondata/result.json", - "tests/e2e/kafka2kafka/metering/metering_test.go":"transfer_manager/go/tests/e2e/kafka2kafka/metering/metering_test.go", - "tests/e2e/kafka2kafka/mirror/mirror_test.go":"transfer_manager/go/tests/e2e/kafka2kafka/mirror/mirror_test.go", - "tests/e2e/kafka2kafka/multi_topic/canondata/result.json":"transfer_manager/go/tests/e2e/kafka2kafka/multi_topic/canondata/result.json", - "tests/e2e/kafka2kafka/multi_topic/mirror_test.go":"transfer_manager/go/tests/e2e/kafka2kafka/multi_topic/mirror_test.go", - "tests/e2e/kafka2mongo/replication/check_db_test.go":"transfer_manager/go/tests/e2e/kafka2mongo/replication/check_db_test.go", - "tests/e2e/kafka2mongo/replication/dump/date_time.sql":"transfer_manager/go/tests/e2e/kafka2mongo/replication/dump/date_time.sql", - "tests/e2e/kafka2mysql/filter_rows/check_db_test.go":"transfer_manager/go/tests/e2e/kafka2mysql/filter_rows/check_db_test.go", - "tests/e2e/kafka2mysql/filter_rows/dump/date_time.sql":"transfer_manager/go/tests/e2e/kafka2mysql/filter_rows/dump/date_time.sql", - "tests/e2e/kafka2mysql/replication/check_db_test.go":"transfer_manager/go/tests/e2e/kafka2mysql/replication/check_db_test.go", - "tests/e2e/kafka2mysql/replication/dump/date_time.sql":"transfer_manager/go/tests/e2e/kafka2mysql/replication/dump/date_time.sql", - "tests/e2e/mongo2ch/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2ch/snapshot/check_db_test.go", - "tests/e2e/mongo2ch/snapshot/dump.sql":"transfer_manager/go/tests/e2e/mongo2ch/snapshot/dump.sql", - "tests/e2e/mongo2ch/snapshot_flatten/canondata/result.json":"transfer_manager/go/tests/e2e/mongo2ch/snapshot_flatten/canondata/result.json", - "tests/e2e/mongo2ch/snapshot_flatten/canondata/snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted":"transfer_manager/go/tests/e2e/mongo2ch/snapshot_flatten/canondata/snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted", - "tests/e2e/mongo2ch/snapshot_flatten/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2ch/snapshot_flatten/check_db_test.go", - "tests/e2e/mongo2ch/snapshot_flatten/dump.sql":"transfer_manager/go/tests/e2e/mongo2ch/snapshot_flatten/dump.sql", - "tests/e2e/mongo2mock/slots/slot_test.go":"transfer_manager/go/tests/e2e/mongo2mock/slots/slot_test.go", - "tests/e2e/mongo2mock/tech_db_permission/permission_test.go":"transfer_manager/go/tests/e2e/mongo2mock/tech_db_permission/permission_test.go", - "tests/e2e/mongo2mongo/add_db_on_snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/add_db_on_snapshot/check_db_test.go", - "tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go", - "tests/e2e/mongo2mongo/bson_order/reorder_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/bson_order/reorder_test.go", - "tests/e2e/mongo2mongo/db_rename/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/db_rename/check_db_test.go", - "tests/e2e/mongo2mongo/db_rename_rep/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/db_rename_rep/check_db_test.go", - "tests/e2e/mongo2mongo/replication/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/replication/check_db_test.go", - "tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go", - "tests/e2e/mongo2mongo/replication_update_model/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/replication_update_model/check_db_test.go", - "tests/e2e/mongo2mongo/rps/replication_source/rps_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go", - "tests/e2e/mongo2mongo/rps/rps.go":"transfer_manager/go/tests/e2e/mongo2mongo/rps/rps.go", - "tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db1.yaml":"transfer_manager/go/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db1.yaml", - "tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db2.yaml":"transfer_manager/go/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db2.yaml", - "tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go", - "tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db1.yaml":"transfer_manager/go/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db1.yaml", - "tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db2.yaml":"transfer_manager/go/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db2.yaml", - "tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go", - "tests/e2e/mongo2mongo/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mongo2mongo/snapshot/check_db_test.go", - "tests/e2e/mysql2ch/comparators.go":"transfer_manager/go/tests/e2e/mysql2ch/comparators.go", - "tests/e2e/mysql2ch/replication/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/replication/check_db_test.go", - "tests/e2e/mysql2ch/replication/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/replication/dump/ch/dump.sql", - "tests/e2e/mysql2ch/replication/dump/mysql/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/replication/dump/mysql/dump.sql", - "tests/e2e/mysql2ch/replication_minimal/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/replication_minimal/check_db_test.go", - "tests/e2e/mysql2ch/replication_minimal/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/replication_minimal/dump/ch/dump.sql", - "tests/e2e/mysql2ch/replication_minimal/dump/mysql/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/replication_minimal/dump/mysql/dump.sql", - "tests/e2e/mysql2ch/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/snapshot/check_db_test.go", - "tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql", - "tests/e2e/mysql2ch/snapshot/dump/mysql/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot/dump/mysql/dump.sql", - "tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go", - "tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql", - "tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql", - "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted", - "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0", - "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1", - "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.2":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.2", - "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.3":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.3", - "tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.4":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.4", - "tests/e2e/mysql2kafka/debezium/replication/canondata/result.json":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/result.json", - "tests/e2e/mysql2kafka/debezium/replication/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/check_db_test.go", - "tests/e2e/mysql2kafka/debezium/replication/init_source/dump.sql":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/init_source/dump.sql", - "tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql", - "tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql", - "tests/e2e/mysql2kafka/debezium/snapshot/canondata/result.json":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/snapshot/canondata/result.json", - "tests/e2e/mysql2kafka/debezium/snapshot/canondata/snapshot.snapshot.TestSnapshot/extracted":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/snapshot/canondata/snapshot.snapshot.TestSnapshot/extracted", - "tests/e2e/mysql2kafka/debezium/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/snapshot/check_db_test.go", - "tests/e2e/mysql2kafka/debezium/snapshot/init_source/dump.sql":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/snapshot/init_source/dump.sql", - "tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go", - "tests/e2e/mysql2mock/debezium/debezium_replication/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/dump/dump.sql", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt", - "tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_snapshot/check_db_test.go", - "tests/e2e/mysql2mock/debezium/debezium_snapshot/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_snapshot/dump/dump.sql", - "tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt", - "tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt":"transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt", - "tests/e2e/mysql2mock/non_utf8_charset/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mock/non_utf8_charset/check_db_test.go", - "tests/e2e/mysql2mock/non_utf8_charset/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mock/non_utf8_charset/dump/dump.sql", - "tests/e2e/mysql2mock/timezone/canondata/result.json":"transfer_manager/go/tests/e2e/mysql2mock/timezone/canondata/result.json", - "tests/e2e/mysql2mock/timezone/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mock/timezone/check_db_test.go", - "tests/e2e/mysql2mock/timezone/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mock/timezone/dump/dump.sql", - "tests/e2e/mysql2mock/views/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mock/views/check_db_test.go", - "tests/e2e/mysql2mock/views/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mock/views/dump/dump.sql", - "tests/e2e/mysql2mysql/alters/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/alters/check_db_test.go", - "tests/e2e/mysql2mysql/alters/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/alters/dump/type_check.sql", - "tests/e2e/mysql2mysql/binary/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/binary/check_db_test.go", - "tests/e2e/mysql2mysql/binary/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/binary/dump/type_check.sql", - "tests/e2e/mysql2mysql/cascade_deletes/common/test.go":"transfer_manager/go/tests/e2e/mysql2mysql/cascade_deletes/common/test.go", - "tests/e2e/mysql2mysql/cascade_deletes/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/cascade_deletes/dump/type_check.sql", - "tests/e2e/mysql2mysql/cascade_deletes/test_per_table/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/cascade_deletes/test_per_table/check_db_test.go", - "tests/e2e/mysql2mysql/cascade_deletes/test_per_transaction/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/cascade_deletes/test_per_transaction/check_db_test.go", - "tests/e2e/mysql2mysql/cleanup_tables/cleanup_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/cleanup_tables/cleanup_test.go", - "tests/e2e/mysql2mysql/cleanup_tables/source/dump.sql":"transfer_manager/go/tests/e2e/mysql2mysql/cleanup_tables/source/dump.sql", - "tests/e2e/mysql2mysql/cleanup_tables/target/dump.sql":"transfer_manager/go/tests/e2e/mysql2mysql/cleanup_tables/target/dump.sql", - "tests/e2e/mysql2mysql/comment/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/comment/check_db_test.go", - "tests/e2e/mysql2mysql/comment/dump/comment.sql":"transfer_manager/go/tests/e2e/mysql2mysql/comment/dump/comment.sql", - "tests/e2e/mysql2mysql/consistent_snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/consistent_snapshot/check_db_test.go", - "tests/e2e/mysql2mysql/consistent_snapshot/dump/consistent_snapshot.sql":"transfer_manager/go/tests/e2e/mysql2mysql/consistent_snapshot/dump/consistent_snapshot.sql", - "tests/e2e/mysql2mysql/date_time/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/date_time/check_db_test.go", - "tests/e2e/mysql2mysql/date_time/dump/date_time.sql":"transfer_manager/go/tests/e2e/mysql2mysql/date_time/dump/date_time.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/dump/type_check.sql", - "tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go", - "tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/dump/type_check.sql", - "tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted":"transfer_manager/go/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted", - "tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted.0":"transfer_manager/go/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted.0", - "tests/e2e/mysql2mysql/float/canondata/result.json":"transfer_manager/go/tests/e2e/mysql2mysql/float/canondata/result.json", - "tests/e2e/mysql2mysql/float/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/float/check_db_test.go", - "tests/e2e/mysql2mysql/float/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mysql/float/dump/dump.sql", - "tests/e2e/mysql2mysql/float/increment.sql":"transfer_manager/go/tests/e2e/mysql2mysql/float/increment.sql", - "tests/e2e/mysql2mysql/geometry/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/geometry/check_db_test.go", - "tests/e2e/mysql2mysql/geometry/dump/geometry.sql":"transfer_manager/go/tests/e2e/mysql2mysql/geometry/dump/geometry.sql", - "tests/e2e/mysql2mysql/light/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/light/check_db_test.go", - "tests/e2e/mysql2mysql/light/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/light/dump/type_check.sql", - "tests/e2e/mysql2mysql/light_all_datatypes/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/light_all_datatypes/check_db_test.go", - "tests/e2e/mysql2mysql/light_all_datatypes/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/light_all_datatypes/dump/type_check.sql", - "tests/e2e/mysql2mysql/medium/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/medium/check_db_test.go", - "tests/e2e/mysql2mysql/no_auto_value_on_zero/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/no_auto_value_on_zero/check_db_test.go", - "tests/e2e/mysql2mysql/no_auto_value_on_zero/dump/no_auto_value_on_zero.sql":"transfer_manager/go/tests/e2e/mysql2mysql/no_auto_value_on_zero/dump/no_auto_value_on_zero.sql", - "tests/e2e/mysql2mysql/partitioned_table/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/partitioned_table/check_db_test.go", - "tests/e2e/mysql2mysql/partitioned_table/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mysql/partitioned_table/dump/dump.sql", - "tests/e2e/mysql2mysql/pkeychanges/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/pkeychanges/check_db_test.go", - "tests/e2e/mysql2mysql/pkeychanges/dump/type_check.sql":"transfer_manager/go/tests/e2e/mysql2mysql/pkeychanges/dump/type_check.sql", - "tests/e2e/mysql2mysql/replace_fkey/common/test.go":"transfer_manager/go/tests/e2e/mysql2mysql/replace_fkey/common/test.go", - "tests/e2e/mysql2mysql/replace_fkey/dump/fkey.sql":"transfer_manager/go/tests/e2e/mysql2mysql/replace_fkey/dump/fkey.sql", - "tests/e2e/mysql2mysql/replace_fkey/test_per_table/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/replace_fkey/test_per_table/check_db_test.go", - "tests/e2e/mysql2mysql/replace_fkey/test_per_transaction/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/replace_fkey/test_per_transaction/check_db_test.go", - "tests/e2e/mysql2mysql/scheme/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/scheme/check_db_test.go", - "tests/e2e/mysql2mysql/scheme/dump/scheme.sql":"transfer_manager/go/tests/e2e/mysql2mysql/scheme/dump/scheme.sql", - "tests/e2e/mysql2mysql/snapshot_without_pk/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/snapshot_without_pk/check_db_test.go", - "tests/e2e/mysql2mysql/snapshot_without_pk/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2mysql/snapshot_without_pk/dump/dump.sql", - "tests/e2e/mysql2mysql/tx_boundaries/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/tx_boundaries/check_db_test.go", - "tests/e2e/mysql2mysql/tx_boundaries/dump/update.sql":"transfer_manager/go/tests/e2e/mysql2mysql/tx_boundaries/dump/update.sql", - "tests/e2e/mysql2mysql/update/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/update/check_db_test.go", - "tests/e2e/mysql2mysql/update/dump/update.sql":"transfer_manager/go/tests/e2e/mysql2mysql/update/dump/update.sql", - "tests/e2e/mysql2mysql/update_cp1251/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/update_cp1251/check_db_test.go", - "tests/e2e/mysql2mysql/update_cp1251/dump/update.sql":"transfer_manager/go/tests/e2e/mysql2mysql/update_cp1251/dump/update.sql", - "tests/e2e/mysql2mysql/update_minimal/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/update_minimal/check_db_test.go", - "tests/e2e/mysql2mysql/update_minimal/dump/update_minimal.sql":"transfer_manager/go/tests/e2e/mysql2mysql/update_minimal/dump/update_minimal.sql", - "tests/e2e/mysql2mysql/update_unicode/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/update_unicode/check_db_test.go", - "tests/e2e/mysql2mysql/update_unicode/dump/update.sql":"transfer_manager/go/tests/e2e/mysql2mysql/update_unicode/dump/update.sql", - "tests/e2e/mysql2mysql/view/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2mysql/view/check_db_test.go", - "tests/e2e/mysql2mysql/view/dump/update.sql":"transfer_manager/go/tests/e2e/mysql2mysql/view/dump/update.sql", - "tests/e2e/pg2ch/alters/alters_test.go":"transfer_manager/go/tests/e2e/pg2ch/alters/alters_test.go", - "tests/e2e/pg2ch/alters/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/alters/dump/ch/dump.sql", - "tests/e2e/pg2ch/alters/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/alters/dump/pg/dump.sql", - "tests/e2e/pg2ch/alters_with_defaults/alters_test.go":"transfer_manager/go/tests/e2e/pg2ch/alters_with_defaults/alters_test.go", - "tests/e2e/pg2ch/alters_with_defaults/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/alters_with_defaults/dump/ch/dump.sql", - "tests/e2e/pg2ch/alters_with_defaults/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/alters_with_defaults/dump/pg/dump.sql", - "tests/e2e/pg2ch/chmapper/canondata/chmapper.chmapper.TestSnapshot/extracted":"transfer_manager/go/tests/e2e/pg2ch/chmapper/canondata/chmapper.chmapper.TestSnapshot/extracted", - "tests/e2e/pg2ch/chmapper/canondata/result.json":"transfer_manager/go/tests/e2e/pg2ch/chmapper/canondata/result.json", - "tests/e2e/pg2ch/chmapper/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/chmapper/check_db_test.go", - "tests/e2e/pg2ch/chmapper/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/chmapper/dump/ch/dump.sql", - "tests/e2e/pg2ch/chmapper/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/chmapper/dump/pg/dump.sql", - "tests/e2e/pg2ch/comparator.go":"transfer_manager/go/tests/e2e/pg2ch/comparator.go", - "tests/e2e/pg2ch/date_overflow/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/date_overflow/check_db_test.go", - "tests/e2e/pg2ch/date_overflow/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/date_overflow/dump/ch/dump.sql", - "tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql", - "tests/e2e/pg2ch/dbt/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/dbt/check_db_test.go", - "tests/e2e/pg2ch/dbt/init_ch.sql":"transfer_manager/go/tests/e2e/pg2ch/dbt/init_ch.sql", - "tests/e2e/pg2ch/dbt/init_pg.sql":"transfer_manager/go/tests/e2e/pg2ch/dbt/init_pg.sql", - "tests/e2e/pg2ch/empty_keys/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/empty_keys/check_db_test.go", - "tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql", - "tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql", - "tests/e2e/pg2ch/replication/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/replication/check_db_test.go", - "tests/e2e/pg2ch/replication/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/replication/dump/ch/dump.sql", - "tests/e2e/pg2ch/replication/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/replication/dump/pg/dump.sql", - "tests/e2e/pg2ch/replication_mv/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/replication_mv/check_db_test.go", - "tests/e2e/pg2ch/replication_mv/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/replication_mv/dump/ch/dump.sql", - "tests/e2e/pg2ch/replication_mv/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/replication_mv/dump/pg/dump.sql", - "tests/e2e/pg2ch/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/snapshot/check_db_test.go", - "tests/e2e/pg2ch/snapshot/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot/dump/ch/dump.sql", - "tests/e2e/pg2ch/snapshot/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot/dump/pg/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go", - "tests/e2e/pg2ch/snapshot_and_replication_canon_types/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_canon_types/dump/ch/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/check_db_test.go", - "tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/ch/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/pg/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_special_values/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_special_values/check_db_test.go", - "tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/ch/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/pg/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/check_db_test.go", - "tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/ch/dump.sql", - "tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/pg/dump.sql", - "tests/e2e/pg2ch/snapshottsv1/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/snapshottsv1/check_db_test.go", - "tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql", - "tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql", - "tests/e2e/pg2ch/tables_inclusion/check_tables_inclusion_test.go":"transfer_manager/go/tests/e2e/pg2ch/tables_inclusion/check_tables_inclusion_test.go", - "tests/e2e/pg2ch/tables_inclusion/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/tables_inclusion/dump/ch/dump.sql", - "tests/e2e/pg2ch/tables_inclusion/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/tables_inclusion/dump/pg/dump.sql", - "tests/e2e/pg2ch/timestamp/check_db_test.go":"transfer_manager/go/tests/e2e/pg2ch/timestamp/check_db_test.go", - "tests/e2e/pg2ch/timestamp/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/timestamp/dump/ch/dump.sql", - "tests/e2e/pg2ch/timestamp/dump/pg/dump.sql":"transfer_manager/go/tests/e2e/pg2ch/timestamp/dump/pg/dump.sql", - "tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go":"transfer_manager/go/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go", - "tests/e2e/pg2kafkamock/debezium_replication/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2kafkamock/debezium_replication/init_source/dump.sql", - "tests/e2e/pg2mysql/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/pg2mysql/snapshot/check_db_test.go", - "tests/e2e/pg2mysql/snapshot/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2mysql/snapshot/dump/type_check.sql", - "tests/e2e/pg2pg/access/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/access/check_db_test.go", - "tests/e2e/pg2pg/access/dump/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/access/dump/dump.sql", - "tests/e2e/pg2pg/all_types/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/all_types/check_db_test.go", - "tests/e2e/pg2pg/bytea_key/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/bytea_key/check_db_test.go", - "tests/e2e/pg2pg/bytea_key/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/bytea_key/init_source/dump.sql", - "tests/e2e/pg2pg/bytea_key/init_target/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/bytea_key/init_target/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_arr/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_arr/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_arr/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_arr/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_nohomo/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_target/init.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_target/init.sql", - "tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/check_db_test.go", - "tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/init_source/dump.sql", - "tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go", - "tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/init_source/dump.sql", - "tests/e2e/pg2pg/drop_tables/drop_test.go":"transfer_manager/go/tests/e2e/pg2pg/drop_tables/drop_test.go", - "tests/e2e/pg2pg/drop_tables/dump/snapshot.sql":"transfer_manager/go/tests/e2e/pg2pg/drop_tables/dump/snapshot.sql", - "tests/e2e/pg2pg/drop_tables/dump_1/snapshot.sql":"transfer_manager/go/tests/e2e/pg2pg/drop_tables/dump_1/snapshot.sql", - "tests/e2e/pg2pg/enum_with_fallbacks/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/enum_with_fallbacks/check_db_test.go", - "tests/e2e/pg2pg/enum_with_fallbacks/init_dst/init.sql":"transfer_manager/go/tests/e2e/pg2pg/enum_with_fallbacks/init_dst/init.sql", - "tests/e2e/pg2pg/enum_with_fallbacks/init_src/init.sql":"transfer_manager/go/tests/e2e/pg2pg/enum_with_fallbacks/init_src/init.sql", - "tests/e2e/pg2pg/insufficient_privileges/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/insufficient_privileges/check_db_test.go", - "tests/e2e/pg2pg/insufficient_privileges/init_source/init.sql":"transfer_manager/go/tests/e2e/pg2pg/insufficient_privileges/init_source/init.sql", - "tests/e2e/pg2pg/insufficient_privileges/util.go":"transfer_manager/go/tests/e2e/pg2pg/insufficient_privileges/util.go", - "tests/e2e/pg2pg/jsonb/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/jsonb/check_db_test.go", - "tests/e2e/pg2pg/jsonb/init_source/init.sql":"transfer_manager/go/tests/e2e/pg2pg/jsonb/init_source/init.sql", - "tests/e2e/pg2pg/jsonb/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/jsonb/init_target/init.sql", - "tests/e2e/pg2pg/multiindex/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/multiindex/check_db_test.go", - "tests/e2e/pg2pg/multiindex/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/multiindex/init_source/dump.sql", - "tests/e2e/pg2pg/multiindex/init_target/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/multiindex/init_target/dump.sql", - "tests/e2e/pg2pg/namesake_tables/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/namesake_tables/check_db_test.go", - "tests/e2e/pg2pg/namesake_tables/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2pg/namesake_tables/dump/type_check.sql", - "tests/e2e/pg2pg/null_temporals_tsv_1/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/null_temporals_tsv_1/check_db_test.go", - "tests/e2e/pg2pg/null_temporals_tsv_1/dump/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/null_temporals_tsv_1/dump/dump.sql", - "tests/e2e/pg2pg/partitioned_tables/all_parts/dump/initial.sql":"transfer_manager/go/tests/e2e/pg2pg/partitioned_tables/all_parts/dump/initial.sql", - "tests/e2e/pg2pg/partitioned_tables/all_parts/partitioned_tables_test.go":"transfer_manager/go/tests/e2e/pg2pg/partitioned_tables/all_parts/partitioned_tables_test.go", - "tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/dump/initial.sql":"transfer_manager/go/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/dump/initial.sql", - "tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/partitioned_tables_test.go":"transfer_manager/go/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/partitioned_tables_test.go", - "tests/e2e/pg2pg/partitioned_tables/some_parts/dump/initial.sql":"transfer_manager/go/tests/e2e/pg2pg/partitioned_tables/some_parts/dump/initial.sql", - "tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go":"transfer_manager/go/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go", - "tests/e2e/pg2pg/pg_dump/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/pg_dump/check_db_test.go", - "tests/e2e/pg2pg/pg_dump/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2pg/pg_dump/dump/type_check.sql", - "tests/e2e/pg2pg/pkey_update/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/pkey_update/check_db_test.go", - "tests/e2e/pg2pg/pkey_update/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/pkey_update/init_source/dump.sql", - "tests/e2e/pg2pg/pkey_update/init_target/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/pkey_update/init_target/dump.sql", - "tests/e2e/pg2pg/replication/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/replication/check_db_test.go", - "tests/e2e/pg2pg/replication/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2pg/replication/dump/type_check.sql", - "tests/e2e/pg2pg/replication_replica_identity/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/replication_replica_identity/check_db_test.go", - "tests/e2e/pg2pg/replication_replica_identity/helpers.go":"transfer_manager/go/tests/e2e/pg2pg/replication_replica_identity/helpers.go", - "tests/e2e/pg2pg/replication_replica_identity/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_replica_identity/init_source/dump.sql", - "tests/e2e/pg2pg/replication_replica_identity/init_target/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_replica_identity/init_target/dump.sql", - "tests/e2e/pg2pg/replication_special_values/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/replication_special_values/check_db_test.go", - "tests/e2e/pg2pg/replication_special_values/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_special_values/init_source/dump.sql", - "tests/e2e/pg2pg/replication_toast/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/replication_toast/check_db_test.go", - "tests/e2e/pg2pg/replication_toast/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_toast/init_source/dump.sql", - "tests/e2e/pg2pg/replication_toast/init_target/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_toast/init_target/dump.sql", - "tests/e2e/pg2pg/replication_view/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/replication_view/check_db_test.go", - "tests/e2e/pg2pg/replication_view/init_source/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_view/init_source/dump.sql", - "tests/e2e/pg2pg/replication_view/init_target/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/replication_view/init_target/dump.sql", - "tests/e2e/pg2pg/snapshot/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/snapshot/check_db_test.go", - "tests/e2e/pg2pg/snapshot/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2pg/snapshot/dump/type_check.sql", - "tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go", - "tests/e2e/pg2pg/snapshot_missing_public/dump/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/snapshot_missing_public/dump/dump.sql", - "tests/e2e/pg2pg/table_capital_letter/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/table_capital_letter/check_db_test.go", - "tests/e2e/pg2pg/table_capital_letter/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2pg/table_capital_letter/dump/type_check.sql", - "tests/e2e/pg2pg/time_with_fallback/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/time_with_fallback/check_db_test.go", - "tests/e2e/pg2pg/time_with_fallback/init_source/init.sql":"transfer_manager/go/tests/e2e/pg2pg/time_with_fallback/init_source/init.sql", - "tests/e2e/pg2pg/time_with_fallback/init_target/init.sql":"transfer_manager/go/tests/e2e/pg2pg/time_with_fallback/init_target/init.sql", - "tests/e2e/pg2pg/tx_boundaries/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/tx_boundaries/check_db_test.go", - "tests/e2e/pg2pg/tx_boundaries/dump/type_check.sql":"transfer_manager/go/tests/e2e/pg2pg/tx_boundaries/dump/type_check.sql", - "tests/e2e/pg2pg/unusual_dates/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/unusual_dates/check_db_test.go", - "tests/e2e/pg2pg/unusual_dates/dump/dump.sql":"transfer_manager/go/tests/e2e/pg2pg/unusual_dates/dump/dump.sql", - "tests/e2e/pg2pg/user_types/check_db_test.go":"transfer_manager/go/tests/e2e/pg2pg/user_types/check_db_test.go", - "tests/e2e/pg2pg/user_types/init_source/init.sql":"transfer_manager/go/tests/e2e/pg2pg/user_types/init_source/init.sql", - "tests/e2e/sample2ch/replication/check_db_test.go":"transfer_manager/go/tests/e2e/sample2ch/replication/check_db_test.go", - "tests/e2e/sample2ch/replication/dump/dst.sql":"transfer_manager/go/tests/e2e/sample2ch/replication/dump/dst.sql", - "tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_key.txt":"transfer_manager/go/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_key.txt", - "tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_val.txt":"transfer_manager/go/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_val.txt", "tests/helpers/README.md":"transfer_manager/go/tests/helpers/README.md", "tests/helpers/abstract.go":"transfer_manager/go/tests/helpers/abstract.go", "tests/helpers/activate_delivery_wrapper.go":"transfer_manager/go/tests/helpers/activate_delivery_wrapper.go", diff --git a/pkg/dataplane/provideradapter/airbyte/adapter.go b/pkg/dataplane/provideradapter/airbyte/adapter.go deleted file mode 100644 index 5bf3d976..00000000 --- a/pkg/dataplane/provideradapter/airbyte/adapter.go +++ /dev/null @@ -1,35 +0,0 @@ -package airbyte - -import ( - "fmt" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/airbyte" -) - -// ycAirbyteRegistry is a mirror for airbyte connector images. -// since it is public and doesn't require auth, this is not config value, just a const -const ycAirbyteRegistry = "cr.yandex/crppchvoaop8l0jmgvvj" - -func init() { - provideradapter.Register[*airbyte.AirbyteSource](func(val *airbyte.AirbyteSource) provideradapter.Adapter { - return &AdapterSource{AirbyteSource: val} - }) -} - -type AdapterSource struct { - *airbyte.AirbyteSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Image != "" { - return nil - } - defaultImage, ok := airbyte.DefaultImages[s.EndpointType] - if !ok { - return xerrors.Errorf("unknown default airbyte image for endpoint type %d", s.EndpointType) - } - s.Image = fmt.Sprintf("%s/%s", ycAirbyteRegistry, defaultImage) - return nil -} diff --git a/pkg/dataplane/provideradapter/clickhouse/adapter.go b/pkg/dataplane/provideradapter/clickhouse/adapter.go deleted file mode 100644 index d1bb295d..00000000 --- a/pkg/dataplane/provideradapter/clickhouse/adapter.go +++ /dev/null @@ -1,53 +0,0 @@ -package model - -import ( - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" -) - -func init() { - provideradapter.Register[*model.ChSource](func(val *model.ChSource) provideradapter.Adapter { - return &AdapterSource{ChSource: val} - }) - provideradapter.Register[*model.ChDestination](func(val *model.ChDestination) provideradapter.Adapter { - return &AdapterDestination{ChDestination: val} - }) -} - -type AdapterSource struct { - *model.ChSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Token == "" { - if cpconfig.IsInternalCloud() { - // Backward compatibility - s.Token = string(cpconfig.InternalCloud.MDBToken) - } - } - if dpconfig.IsDataPlane() { - s.RootCACertPaths = dpconfig.RootCA() - } - - return nil -} - -type AdapterDestination struct { - *model.ChDestination -} - -func (d *AdapterDestination) WithConfig() error { - if d.Token == "" { - if cpconfig.IsInternalCloud() { - // Backward compatibility - d.Token = string(cpconfig.InternalCloud.MDBToken) - } - } - if dpconfig.IsDataPlane() { - d.RootCACertPaths = dpconfig.RootCA() - } - - return nil -} diff --git a/pkg/dataplane/provideradapter/elastic/adapter.go b/pkg/dataplane/provideradapter/elastic/adapter.go deleted file mode 100644 index 48d2be16..00000000 --- a/pkg/dataplane/provideradapter/elastic/adapter.go +++ /dev/null @@ -1,55 +0,0 @@ -package elastic - -import ( - "os" - - "github.com/doublecloud/tross/internal/core/xerrors" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/elastic" -) - -func init() { - provideradapter.Register[*elastic.ElasticSearchSource](func(val *elastic.ElasticSearchSource) provideradapter.Adapter { - return &AdapterSource{ElasticSearchSource: val} - }) - provideradapter.Register[*elastic.ElasticSearchDestination](func(val *elastic.ElasticSearchDestination) provideradapter.Adapter { - return &AdapterDestination{ElasticSearchDestination: val} - }) -} - -type AdapterSource struct { - *elastic.ElasticSearchSource -} - -func (s *AdapterSource) WithConfig() error { - if s.ClusterID != "" { - if len(dpconfig.RootCA()) == 0 { - return xerrors.NewSentinel("Can't read default cert: no files given") - } - fileCaCert, err := os.ReadFile(dpconfig.RootCA()[0]) - if err != nil { - return xerrors.Errorf("Can't read default cert PEM file: %w", err) - } - s.TLSFile = string(fileCaCert) - } - return nil -} - -type AdapterDestination struct { - *elastic.ElasticSearchDestination -} - -func (d *AdapterDestination) WithConfig() error { - if d.ClusterID != "" { - if len(dpconfig.RootCA()) == 0 { - return xerrors.NewSentinel("Can't read default cert: no files given") - } - fileCaCert, err := os.ReadFile(dpconfig.RootCA()[0]) - if err != nil { - return xerrors.Errorf("Can't read default cert PEM file: %w", err) - } - d.TLSFile = string(fileCaCert) - } - return nil -} diff --git a/pkg/dataplane/provideradapter/eventhub/adapter.go b/pkg/dataplane/provideradapter/eventhub/adapter.go deleted file mode 100644 index 63f5a2a1..00000000 --- a/pkg/dataplane/provideradapter/eventhub/adapter.go +++ /dev/null @@ -1,24 +0,0 @@ -package eventhub - -import ( - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/eventhub" -) - -func init() { - provideradapter.Register[*eventhub.EventHubSource](func(val *eventhub.EventHubSource) provideradapter.Adapter { - return &AdapterSource{EventHubSource: val} - }) -} - -type AdapterSource struct { - *eventhub.EventHubSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Transformer != nil { - s.Transformer.CloudFunctionsBaseURL = dpconfig.ExternalCloud.CloudFunctionsBaseURL - } - return nil -} diff --git a/pkg/dataplane/provideradapter/kafka/adapter.go b/pkg/dataplane/provideradapter/kafka/adapter.go deleted file mode 100644 index b44c228b..00000000 --- a/pkg/dataplane/provideradapter/kafka/adapter.go +++ /dev/null @@ -1,70 +0,0 @@ -package kafka - -import ( - "os" - - "github.com/doublecloud/tross/internal/core/xerrors" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/providers/logbroker" -) - -func init() { - provideradapter.Register[*kafka.KafkaSource](func(val *kafka.KafkaSource) provideradapter.Adapter { - return &AdapterSource{KafkaSource: val} - }) - provideradapter.Register[*kafka.KafkaDestination](func(val *kafka.KafkaDestination) provideradapter.Adapter { - return &AdapterDestination{KafkaDestination: val} - }) -} - -type AdapterSource struct { - *kafka.KafkaSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Connection.ClusterID != "" { - if len(dpconfig.RootCA()) > 0 { - fileCaCert, err := os.ReadFile(dpconfig.RootCA()[0]) - if err != nil { - return xerrors.Errorf("read default cert PEM file: %w", err) - } - s.Connection.TLSFile = string(fileCaCert) - s.Connection.TLS = logbroker.EnabledTLS - } else { - s.Connection.TLS = logbroker.DefaultTLS - } - } else if s.Connection.TLS == "" { - s.Connection.TLS = logbroker.DisabledTLS - } - if s.Transformer != nil { - s.Transformer.CloudFunctionsBaseURL = dpconfig.ExternalCloud.CloudFunctionsBaseURL - } - return nil -} - -type AdapterDestination struct { - *kafka.KafkaDestination -} - -func (d *AdapterDestination) WithConfig() error { - if d.Connection == nil { - return xerrors.Errorf("connection is require") - } - if d.Connection.ClusterID != "" { - if len(dpconfig.RootCA()) > 0 { - fileCaCert, err := os.ReadFile(dpconfig.RootCA()[0]) - if err != nil { - return xerrors.Errorf("read default cert PEM file: %w", err) - } - d.Connection.TLSFile = string(fileCaCert) - d.Connection.TLS = logbroker.EnabledTLS - } else { - d.Connection.TLS = logbroker.DefaultTLS - } - } else if d.Connection.TLS == "" { - d.Connection.TLS = logbroker.DisabledTLS - } - return nil -} diff --git a/pkg/dataplane/provideradapter/logbroker/adapter.go b/pkg/dataplane/provideradapter/logbroker/adapter.go deleted file mode 100644 index 3d449d38..00000000 --- a/pkg/dataplane/provideradapter/logbroker/adapter.go +++ /dev/null @@ -1,108 +0,0 @@ -package logbroker - -import ( - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/kikimr/public/sdk/go/ydb" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/config/env" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/logbroker" - yccreds "github.com/doublecloud/tross/pkg/yc/credentials" -) - -func init() { - provideradapter.Register[*logbroker.LbSource](func(val *logbroker.LbSource) provideradapter.Adapter { - return &AdapterLBSource{LbSource: val} - }) - provideradapter.Register[*logbroker.LfSource](func(val *logbroker.LfSource) provideradapter.Adapter { - return &AdapterLFSource{LfSource: val} - }) - provideradapter.Register[*logbroker.LbDestination](func(val *logbroker.LbDestination) provideradapter.Adapter { - return &AdapterDestination{LbDestination: val} - }) -} - -type AdapterLBSource struct { - *logbroker.LbSource -} - -func (s *AdapterLBSource) WithConfig() error { - var err error - s.Credentials, err = getCreds(s.Credentials, s.Token) - if err != nil { - return xerrors.Errorf("error getting lb creds: %w", err) - } - s.RootCAFiles = dpconfig.RootCA() - s.TLS = overrideTLS(s.TLS) - return nil -} - -type AdapterLFSource struct { - *logbroker.LfSource -} - -func (s *AdapterLFSource) WithConfig() error { - var err error - s.Credentials, err = getCreds(s.Credentials, s.Token) - if err != nil { - return xerrors.Errorf("error getting lb creds: %w", err) - } - s.RootCAFiles = dpconfig.RootCA() - s.TLS = overrideTLS(s.TLS) - return nil -} - -type AdapterDestination struct { - *logbroker.LbDestination -} - -func (d *AdapterDestination) WithConfig() error { - var err error - d.Credentials, err = getCreds(d.Credentials, d.Token) - if err != nil { - return xerrors.Errorf("error getting lb creds: %w", err) - } - d.RootCAFiles = dpconfig.RootCA() - d.TLS = overrideTLS(d.TLS) - return nil -} - -func getCreds(creds ydb.Credentials, currentToken string) (ydb.Credentials, error) { - if creds != nil { - return creds, nil - } - switch e := env.Get(); e { - case env.EnvironmentInternal: - token := currentToken - if token == "" { - // fallback to robot token for legacy endpoints - if cpconfig.IsInternalCloud() { - token = string(cpconfig.InternalCloud.LogbrokerToken) - } else if dpconfig.IsInternalCloud() { - token = string(dpconfig.InternalCloud.LogbrokerToken) - } - } - // Current tests require nil creds - if token == "" { - return nil, nil - } - return ydb.AuthTokenCredentials{AuthToken: token}, nil - case env.EnvironmentYC: - cc, err := yccreds.NewIamCreds(logger.Log) - if err != nil { - return nil, xerrors.Errorf("error getting cloud creds for logbroker: %w", err) - } - return cc, nil - default: - return nil, xerrors.Errorf("unsupported environment %s for logbroker", e.String()) - } -} - -func overrideTLS(currentTLS logbroker.TLSMode) logbroker.TLSMode { - if dpconfig.IsExternalCloud() { - return logbroker.EnabledTLS - } - return currentTLS -} diff --git a/pkg/dataplane/provideradapter/mongo/adapter.go b/pkg/dataplane/provideradapter/mongo/adapter.go deleted file mode 100644 index 1c612f89..00000000 --- a/pkg/dataplane/provideradapter/mongo/adapter.go +++ /dev/null @@ -1,34 +0,0 @@ -package mongo - -import ( - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/mongo" -) - -func init() { - provideradapter.Register[*mongo.MongoSource](func(val *mongo.MongoSource) provideradapter.Adapter { - return &AdapterSource{MongoSource: val} - }) - provideradapter.Register[*mongo.MongoDestination](func(val *mongo.MongoDestination) provideradapter.Adapter { - return &AdapterDestination{MongoDestination: val} - }) -} - -type AdapterSource struct { - *mongo.MongoSource -} - -func (s *AdapterSource) WithConfig() error { - s.RootCAFiles = dpconfig.RootCA() - return nil -} - -type AdapterDestination struct { - *mongo.MongoDestination -} - -func (d *AdapterDestination) WithConfig() error { - d.RootCAFiles = dpconfig.RootCA() - return nil -} diff --git a/pkg/dataplane/provideradapter/mysql/adapter.go b/pkg/dataplane/provideradapter/mysql/adapter.go deleted file mode 100644 index 65cfb313..00000000 --- a/pkg/dataplane/provideradapter/mysql/adapter.go +++ /dev/null @@ -1,57 +0,0 @@ -package mysql - -import ( - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/config/env" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/mysql" -) - -func init() { - provideradapter.Register[*mysql.MysqlSource](func(val *mysql.MysqlSource) provideradapter.Adapter { - return &AdapterSource{MysqlSource: val} - }) - provideradapter.Register[*mysql.MysqlDestination](func(val *mysql.MysqlDestination) provideradapter.Adapter { - return &AdapterDestination{MysqlDestination: val} - }) -} - -type AdapterSource struct { - *mysql.MysqlSource -} - -func (s *AdapterSource) WithConfig() error { - s.RootCAFiles = dpconfig.RootCA() - if s.Token == "" { - if cpconfig.IsInternalCloud() { - s.Token = string(cpconfig.InternalCloud.MDBToken) - } else if dpconfig.IsInternalCloud() { - s.Token = string(dpconfig.InternalCloud.MDBToken) - } - } - - if !s.YdbTracking && !env.IsTest() { - s.YdbTracking = cpconfig.IsInternalCloud() || dpconfig.IsInternalCloud() - } - s.IsPublic = cpconfig.IsExternalCloud() || dpconfig.IsExternalCloud() - return nil -} - -type AdapterDestination struct { - *mysql.MysqlDestination -} - -func (d *AdapterDestination) WithConfig() error { - d.RootCAFiles = dpconfig.RootCA() - if d.Token == "" { - if cpconfig.IsInternalCloud() { - d.Token = string(cpconfig.InternalCloud.MDBToken) - } else if dpconfig.IsInternalCloud() { - d.Token = string(dpconfig.InternalCloud.MDBToken) - } - } - d.IsPublic = cpconfig.IsExternalCloud() || dpconfig.IsExternalCloud() - - return nil -} diff --git a/pkg/dataplane/provideradapter/postgres/adapter.go b/pkg/dataplane/provideradapter/postgres/adapter.go deleted file mode 100644 index ea6cd477..00000000 --- a/pkg/dataplane/provideradapter/postgres/adapter.go +++ /dev/null @@ -1,50 +0,0 @@ -package postgres - -import ( - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - pg "github.com/doublecloud/tross/pkg/providers/postgres" -) - -func init() { - provideradapter.Register[*pg.PgSource](func(val *pg.PgSource) provideradapter.Adapter { - return &AdapterSource{PgSource: val} - }) - provideradapter.Register[*pg.PgDestination](func(val *pg.PgDestination) provideradapter.Adapter { - return &AdapterDestination{PgDestination: val} - }) -} - -type AdapterSource struct { - *pg.PgSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Token == "" { - if cpconfig.IsInternalCloud() { - s.Token = string(cpconfig.InternalCloud.MDBToken) - } else if dpconfig.IsInternalCloud() { - s.Token = string(dpconfig.InternalCloud.MDBToken) - } - } - if dpconfig.Common != nil && len(s.PgDumpCommand) == 0 { - s.PgDumpCommand = dpconfig.Common.PgDumpCommand - } - return nil -} - -type AdapterDestination struct { - *pg.PgDestination -} - -func (d *AdapterDestination) WithConfig() error { - if d.Token == "" { - if cpconfig.IsInternalCloud() { - d.Token = string(cpconfig.InternalCloud.MDBToken) - } else if dpconfig.IsInternalCloud() { - d.Token = string(dpconfig.InternalCloud.MDBToken) - } - } - return nil -} diff --git a/pkg/dataplane/provideradapter/s3/adapter.go b/pkg/dataplane/provideradapter/s3/adapter.go deleted file mode 100644 index 8db6728c..00000000 --- a/pkg/dataplane/provideradapter/s3/adapter.go +++ /dev/null @@ -1,32 +0,0 @@ -package s3 - -import ( - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/s3" -) - -func init() { - provideradapter.Register[*s3.S3Destination](func(val *s3.S3Destination) provideradapter.Adapter { - return &AdapterDestination{S3Destination: val} - }) -} - -type AdapterDestination struct { - *s3.S3Destination -} - -func (d *AdapterDestination) WithConfig() error { - if d.Endpoint == "" && d.ServiceAccountID != "" { - if cpconfig.IsExternalCloud() { - d.Endpoint = cpconfig.ExternalCloud.S3Endpoint - } else if dpconfig.IsExternalCloud() { - d.Endpoint = dpconfig.ExternalCloud.S3Endpoint - } - } - if d.Region == "" && (dpconfig.IsInternalCloud() || cpconfig.IsInternalCloud()) { - d.Region = "ru-central1" - } - return nil -} diff --git a/pkg/dataplane/provideradapter/ydb/adapter.go b/pkg/dataplane/provideradapter/ydb/adapter.go deleted file mode 100644 index ee69e878..00000000 --- a/pkg/dataplane/provideradapter/ydb/adapter.go +++ /dev/null @@ -1,142 +0,0 @@ -package ydb - -import ( - "context" - "os" - "path" - "strings" - "time" - - ydbpb "github.com/doublecloud/tross/cloud/bitbucket/public-api/yandex/cloud/ydb/v1" - "github.com/doublecloud/tross/internal/core/xerrors" - server "github.com/doublecloud/tross/pkg/abstract/model" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/config/env" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/ydb" - ycsdk "github.com/doublecloud/tross/pkg/yc/sdk" -) - -func init() { - provideradapter.Register[*ydb.YdbSource](func(val *ydb.YdbSource) provideradapter.Adapter { - return &AdapterSource{YdbSource: val} - }) - provideradapter.Register[*ydb.YdbDestination](func(val *ydb.YdbDestination) provideradapter.Adapter { - return &AdapterDestination{YdbDestination: val} - }) -} - -type AdapterSource struct { - *ydb.YdbSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Token == "" { - if cpconfig.IsInternalCloud() { - s.Token = server.SecretString(cpconfig.InternalCloud.YdbToken) - } else if dpconfig.IsInternalCloud() { - s.Token = server.SecretString(dpconfig.InternalCloud.YDBToken) - } - } - if dpconfig.IsExternalCloud() { - s.TLSEnabled = true - s.UserdataAuth = dpconfig.IsExternalCloud() && !env.IsTest() - s.RootCAFiles = dpconfig.RootCA() - } - - if dpconfig.IsExternalCloud() || cpconfig.IsExternalCloud() { - if s.Instance == "" { - dbID := path.Base(s.Database) - if len(dbID) == 0 { - dbID = s.Database - } - // resolve database from YDB - sdk, err := ycsdk.Instance() - if err != nil { - return xerrors.Errorf("Unable to resolve YC SDK instance: %w", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - db, err := sdk.YDB().GetDatabase(ctx, &ydbpb.GetDatabaseRequest{DatabaseId: dbID}) - if err != nil { - return xerrors.Errorf("Unable to get database from YC: %w", err) - } - parts := strings.Split(db.Endpoint, "/?") - if len(parts) != 2 { - return xerrors.Errorf("unable to parse YDB endpoint from: %v", db.Endpoint) - } - s.Instance = strings.ReplaceAll(parts[0], "grpcs://", "") - } - } - return nil -} - -type AdapterDestination struct { - *ydb.YdbDestination -} - -func (d *AdapterDestination) WithConfig() error { - if d.Token == "" { - if cpconfig.IsInternalCloud() { - d.Token = server.SecretString(cpconfig.InternalCloud.YdbToken) - } else if dpconfig.IsInternalCloud() { - d.Token = server.SecretString(dpconfig.InternalCloud.YDBToken) - } - } - if dpconfig.IsExternalCloud() { - d.UserdataAuth = !env.IsTest() - } - if dpconfig.IsExternalCloud() { - d.TLSEnabled = true - d.RootCAFiles = dpconfig.RootCA() - } - if !dpconfig.IsInternalCloud() { - host, err := os.Hostname() - if err != nil { - return xerrors.Errorf("Unable to resolve hostname: %v", err) - } - // We treat first host in group as primary host this host must do rotation. - d.Primary = strings.HasSuffix(host, "-1") - } else { - logicalIDX, _ := os.LookupEnv("YT_JOB_COOKIE") - d.Primary = logicalIDX == "0" - } - - var err error - d.Instance, err = GetInstance(d.Database, d.Instance) - if err != nil { - return xerrors.Errorf("error resolving YDB instance: %w", err) - } - return nil -} - -func GetInstance(database, instance string) (string, error) { - if dpconfig.IsExternalCloud() || cpconfig.IsExternalCloud() { - if instance != "" { - return instance, nil - } else { - dbID := path.Base(database) - if len(dbID) == 0 { - dbID = database - } - // resolve database from YDB - sdk, err := ycsdk.Instance() - if err != nil { - return "", xerrors.Errorf("Unable to resolve YC SDK instance: %w", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - db, err := sdk.YDB().GetDatabase(ctx, &ydbpb.GetDatabaseRequest{DatabaseId: dbID}) - if err != nil { - return "", xerrors.Errorf("Unable to get database from YC: %w", err) - } - parts := strings.Split(db.Endpoint, "/?") - if len(parts) != 2 { - return "", xerrors.Errorf("unable to parse YDB endpoint from: %v", db.Endpoint) - } - return strings.ReplaceAll(parts[0], "grpcs://", ""), nil - } - } - return instance, nil -} diff --git a/pkg/dataplane/provideradapter/yds/adapter.go b/pkg/dataplane/provideradapter/yds/adapter.go deleted file mode 100644 index a22a0a78..00000000 --- a/pkg/dataplane/provideradapter/yds/adapter.go +++ /dev/null @@ -1,72 +0,0 @@ -package yds - -import ( - server "github.com/doublecloud/tross/pkg/abstract/model" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/config/env" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/yds" -) - -func init() { - provideradapter.Register[*yds.YDSSource](func(val *yds.YDSSource) provideradapter.Adapter { - return &AdapterSource{YDSSource: val} - }) - provideradapter.Register[*yds.YDSDestination](func(val *yds.YDSDestination) provideradapter.Adapter { - return &AdapterDestination{YDSDestination: val} - }) -} - -type AdapterSource struct { - *yds.YDSSource -} - -func (s *AdapterSource) WithConfig() error { - if s.Endpoint == "" { - // if endpoint not specified - then its serverless YDS - if cpconfig.IsExternalCloud() { - s.Endpoint = cpconfig.ExternalCloud.YDSServerlessEndpoint - } else if dpconfig.IsExternalCloud() { - s.Endpoint = dpconfig.ExternalCloud.YDSServerlessEndpoint - } - } - if s.Transformer != nil { - s.Transformer.CloudFunctionsBaseURL = dpconfig.ExternalCloud.CloudFunctionsBaseURL - } - if dpconfig.IsInternalCloud() || env.IsTest() { - s.Token = server.SecretString(dpconfig.InternalCloud.LogbrokerToken) - } - if dpconfig.IsExternalCloud() { - s.UserdataAuth = !env.IsTest() - } - s.RootCAFiles = dpconfig.RootCA() - if !env.IsTest() { - s.TLSEnalbed = dpconfig.IsExternalCloud() - } - s.RootCAFiles = dpconfig.RootCA() - - return nil -} - -type AdapterDestination struct { - *yds.YDSDestination -} - -func (d *AdapterDestination) WithConfig() error { - if d.LbDstConfig.Instance == "" { - // if endpoint not specified - then its serverless YDS - if cpconfig.IsExternalCloud() { - d.LbDstConfig.Instance = cpconfig.ExternalCloud.YDSServerlessEndpoint - } else if dpconfig.IsExternalCloud() { - d.LbDstConfig.Instance = dpconfig.ExternalCloud.YDSServerlessEndpoint - } - } - if dpconfig.IsExternalCloud() { - d.UserdataAuth = !env.IsTest() - } - d.RootCAFiles = dpconfig.RootCA() - d.TLSEnalbed = dpconfig.IsExternalCloud() - d.RootCAFiles = dpconfig.RootCA() - return nil -} diff --git a/pkg/dataplane/provideradapter/yt/adapter.go b/pkg/dataplane/provideradapter/yt/adapter.go deleted file mode 100644 index dfd4f6a8..00000000 --- a/pkg/dataplane/provideradapter/yt/adapter.go +++ /dev/null @@ -1,48 +0,0 @@ -package yt - -import ( - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/dataplane/provideradapter" - "github.com/doublecloud/tross/pkg/providers/yt" -) - -func init() { - provideradapter.Register[*yt.LfStagingDestination](func(val *yt.LfStagingDestination) provideradapter.Adapter { - return &AdapterStaging{LfStagingDestination: val} - }) - provideradapter.Register[*yt.YtDestinationWrapper](func(val *yt.YtDestinationWrapper) provideradapter.Adapter { - return &AdapterDestination{YtDestinationWrapper: val} - }) -} - -type AdapterStaging struct { - *yt.LfStagingDestination -} - -func (d *AdapterStaging) WithConfig() error { - if d.YtToken == "" { - if cpconfig.IsInternalCloud() { - d.YtToken = string(cpconfig.InternalCloud.YtToken) - } else if dpconfig.IsInternalCloud() { - d.YtToken = string(dpconfig.InternalCloud.YtToken) - } - } - - return nil -} - -type AdapterDestination struct { - *yt.YtDestinationWrapper -} - -func (d *AdapterDestination) WithConfig() error { - if d.Model.Token == "" { - if cpconfig.IsInternalCloud() { - d.Model.Token = string(cpconfig.InternalCloud.YtToken) - } else if dpconfig.IsInternalCloud() { - d.Model.Token = string(dpconfig.InternalCloud.YtToken) - } - } - return nil -} diff --git a/pkg/dataplane/providers.go b/pkg/dataplane/providers.go index 995a7fbe..801734d5 100644 --- a/pkg/dataplane/providers.go +++ b/pkg/dataplane/providers.go @@ -1,22 +1,8 @@ package dataplane import ( - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/airbyte" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/clickhouse" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/elastic" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/eventhub" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/kafka" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/logbroker" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/mongo" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/mysql" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/postgres" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/s3" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/ydb" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/yds" - _ "github.com/doublecloud/tross/pkg/dataplane/provideradapter/yt" _ "github.com/doublecloud/tross/pkg/metering/writer/kafka" _ "github.com/doublecloud/tross/pkg/metering/writer/kinesis" - _ "github.com/doublecloud/tross/pkg/metering/writer/logbroker" _ "github.com/doublecloud/tross/pkg/metering/writer/stdout" _ "github.com/doublecloud/tross/pkg/providers/airbyte" _ "github.com/doublecloud/tross/pkg/providers/clickhouse" diff --git a/recipe/mongo/README.md b/recipe/mongo/README.md deleted file mode 100644 index 0b7138a1..00000000 --- a/recipe/mongo/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Рецепт для шардированной MongoDB 4.x - -В основе данного рецепта лежит туториал по развёртке -шардированного кластера: https://www.mongodb.com/docs/manual/tutorial/deploy-shard-cluster/ -По сути, рецепт автоматизирует этот процесс и в зависимости -от описания шардов предоставляет вам готовый кластер, к которому -можно подключиться. - -Пока что поддерживаются тольо версии 4.0, 4.2, 4.4. - -Проблема с поддержкой версии 5.0+: DEVTOOLSSUPPORT-21944 - -## Структура проекта -* `example` -- примеры - * `recipe_usage` -- пример использования рецепта - * `configs` -- примеры конфигураций шардированных кластеров - * `launch_cluster` -- пример запуск кластера напрямую из Go кода, а - не из рецепта -* `pkg` -- рецепт написан на языке Go, поэтому здесь содержится - код запуска рецепта - * `cluster` -- инфраструктурный код запуска шардированного кластера - * `config` -- код для обработки конфигурации кластера - * `binurl` -- вспомогательный код парсинга странички с - бинарями MongoDB - * `tar` -- код для работы с `tar` архивами в рецепте и в упаковщике - бинарей MongoDB в единый архив - * `test` -- проверка работоспособности рецепта. На текущий момент - поддерживаются только версии 4.0, 4.2, 4.4. На данный момент - поддержка 5.0+ невозможно, смотри тикет - [DEVTOOLSSUPPORT-21944](https://st.yandex-team.ru/DEVTOOLSSUPPORT-21944) -* `cmd` -- additional utility - * `binurl` -- utility that helps pack official MongoDB distribution - with a variety of version for specific platform and OS into - single archive, that can be uploaded into Sandbox. - * `launch_cluster` -- just example how you can deploy your cluster - from code diff --git a/recipe/mongo/cmd/binurl/README.md b/recipe/mongo/cmd/binurl/README.md deleted file mode 100644 index f8f31c18..00000000 --- a/recipe/mongo/cmd/binurl/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# `binurl` CLI utility -## Purpose -It's single purpose: crawling of [long page of MongoDB distributives]( -https://www.mongodb.com/download-center/community/releases/archive) -in order to make list of interesting ones - -## Build -Fast variant: ```go build .``` - -Canonical variant: ```ya make -DDATA_TRANSFER_DISABLE_CGO .``` - -## Usage -### Find MacOS binaries with major patch per version -```./binurl -os macos -arch x86_64 -patch-collapse``` -### Find linux binaries (distribution ubuntu) with major patch per version -```./binurl -os linux -arch x86_64 -tag-contains "ubuntu" -patch-collapse -max-distr-collapse``` - -## Auxiliary -`ubuntu_links.txt` contains mostly `ubuntu1604` version that -can be used in Arcadia Distbuild which uses `ubuntu1604`. - diff --git a/recipe/mongo/cmd/binurl/binary_fetcher.go b/recipe/mongo/cmd/binurl/binary_fetcher.go deleted file mode 100644 index a321f27c..00000000 --- a/recipe/mongo/cmd/binurl/binary_fetcher.go +++ /dev/null @@ -1,215 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io" - "net/http" - "os" - "strings" - "time" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/pkg/util/cli" - "github.com/doublecloud/tross/recipe/mongo/pkg/binurl" -) - -var ( - osFlag string - archFlag string - distributiveSubstr string - minDistroFilter string - maxDistroFilter string - patchCollapseFlag bool - maxDistrCollapseFlag bool -) - -type CmdOS string - -var ( - CmdOSUnspecified CmdOS = "" - CmdOSMacos CmdOS = "macos" - CmdOSLinux CmdOS = "linux" -) - -type CmdArch string - -var ( - CmdArchUnspecified CmdArch = "" - CmdArchAMD64 CmdArch = "x86_64" - CmdArchARM64 CmdArch = "arm64" -) - -func main() { - flag.StringVar(&osFlag, "os", "", "specify OS, available values: macos | linux") - flag.StringVar(&distributiveSubstr, "tag-contains", "", "check that tag has contain specified string (useful when finding certain distributives)") - flag.StringVar(&archFlag, "arch", "", "specify arch, available values: x86_64 | arm64") - flag.BoolVar(&patchCollapseFlag, "patch-collapse", false, "specify this flag to leave only major patches per each version") - flag.StringVar(&minDistroFilter, "min-distro", "", "filters distributions to be greater or equal than specified value, e.g. 'ubuntu1804' specifies version greater or equal than 18.04 version of Ubuntu") - flag.StringVar(&maxDistroFilter, "max-distro", "", "filters distributions to be less or equal than specified value, e.g. 'ubuntu1804' specifies version less or equal than 18.04 version of Ubuntu") - flag.BoolVar(&maxDistrCollapseFlag, "max-distr-collapse", false, "when distribution of OS differs like 'ubuntu1804...' and 'ubuntu2004', you may want to take freshest one") - flag.BoolVar(&maxDistrCollapseFlag, "no-sleep", false, "do not sleep before merging files into single archive") - flag.Parse() - - cmdOS := CmdOS(osFlag) - switch cmdOS { - case CmdOSUnspecified: - case CmdOSMacos: - case CmdOSLinux: - default: - fmt.Println("invalid value for flag --os") - flag.PrintDefaults() - os.Exit(2) - } - - arch := CmdArch(archFlag) - switch arch { - case CmdArchUnspecified: - case CmdArchAMD64: - case CmdArchARM64: - default: - fmt.Println("invalid value for flag --arch") - flag.PrintDefaults() - os.Exit(3) - } - - var links []binurl.BinaryLinks - // make fancy CLI while waiting for heavy operation - sp := cli.NewSpinner(7, 70*time.Millisecond) - sp.Start() - - //links, err := FetchAllBinaries() - var err error - links, err = getLinksByCriteria( - cmdOS, - arch, - distributiveSubstr, - minDistroFilter, - maxDistroFilter, - patchCollapseFlag, - maxDistrCollapseFlag, - ) - if err != nil { - fmt.Println("Cannot get links by criteria:", err) - os.Exit(4) - } - - sp.Stop() - - fmt.Printf("Fetched %d links\n", len(links)) - - printOutList(links) -} - -func printOutList(links []binurl.BinaryLinks) { - for _, link := range links { - fmt.Printf("%7s(%7s), %11s -> %s\n", link.OS, link.Arch, link.Version, link.URL) - } -} - -func getLinksByCriteria( - operationSystem CmdOS, - arch CmdArch, - distributiveSubstr string, - minDistributive string, - maxDistributive string, - patchCollapse bool, - maxDistrCollapse bool, -) ([]binurl.BinaryLinks, error) { - links, err := FetchAllBinaries() - if err != nil { - return nil, xerrors.Errorf("cannot fetch all binaries: %w", err) - } - - switch operationSystem { - case CmdOSUnspecified: - case CmdOSMacos: - links = slices.Filter(links, func(link binurl.BinaryLinks) bool { - switch link.OS { - case binurl.OperationSystemMacos: - return true - case binurl.OperationSystemOsx: - return true - case binurl.OperationSystemOsxSsl: - return true - } - return false - }) - case CmdOSLinux: - links = slices.Filter(links, func(link binurl.BinaryLinks) bool { - switch link.OS { - case binurl.OperationSystemLinux: - return true - } - return false - }) - } - - switch arch { - case CmdArchUnspecified: - case CmdArchAMD64: - links = slices.Filter(links, func(link binurl.BinaryLinks) bool { - return link.Arch == binurl.ArchAMD64 - }) - case CmdArchARM64: - links = slices.Filter(links, func(link binurl.BinaryLinks) bool { - return link.Arch == binurl.ArchARM64 || link.Arch == binurl.ArchAArch64 - }) - - } - - links = slices.Filter(links, func(link binurl.BinaryLinks) bool { - tag := strings.TrimSpace(link.Tag) - distro := strings.TrimSpace(distributiveSubstr) - return strings.Contains(tag, distro) && - (maxDistributive == "" || link.Distributive <= maxDistributive) && - (minDistributive == "" || link.Distributive >= minDistributive) - }) - - if patchCollapse { - links, err = binurl.TakeMaxPatchVersion(links) - if err != nil { - return nil, xerrors.Errorf("cannot aggregate out patch versions: %w", err) - } - } - if maxDistrCollapse { - links, err = binurl.TakeMaxDistrVersion(links) - if err != nil { - return nil, xerrors.Errorf("cannot aggregate out max distributions versions: %w", err) - } - } - return links, nil -} - -func fetchMongoDBArchivePage() ([]byte, error) { - pageURL := binurl.MongodbArchiveURL - rsp, err := http.Get(pageURL) - if err != nil { - return nil, xerrors.Errorf("unable to get page '%s': %w", pageURL, err) - } - if rsp.StatusCode/100 != 2 { - return nil, xerrors.Errorf("status code is not 2xx, actually it is %d", rsp.StatusCode) - } - defer rsp.Body.Close() - - return io.ReadAll(rsp.Body) -} - -func FetchAllBinaries() ([]binurl.BinaryLinks, error) { - bytes, err := fetchMongoDBArchivePage() - if err != nil { - return nil, xerrors.Errorf("cannot fetch mongodb archive page: %w", err) - } - htmlATags := binurl.HTMLARegexp.FindAllString(string(bytes), -1) - - links := []binurl.BinaryLinks{} - for _, htmlATag := range htmlATags { - link := binurl.MakeBinaryLinkByHTMLA(htmlATag) - if link == nil { - continue - } - links = append(links, *link) - } - return links, nil -} diff --git a/recipe/mongo/example/configs/auth.yaml b/recipe/mongo/example/configs/auth.yaml deleted file mode 100644 index 310af68c..00000000 --- a/recipe/mongo/example/configs/auth.yaml +++ /dev/null @@ -1,16 +0,0 @@ -configReplicaSet: - amount: 3 - config: -shards: - amount: 2 - shardReplicaSet: - amount: 4 - config: -mongos: - security: - authorization: enabled -postSteps: - createAdminUser: - user: user - password: P@ssw0rd - authSource: db diff --git a/recipe/mongo/example/launch_cluster/README.md b/recipe/mongo/example/launch_cluster/README.md deleted file mode 100644 index 1a9e9f44..00000000 --- a/recipe/mongo/example/launch_cluster/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Launching MongoDB sharded cluster from code -This is just simple example how to launch your -custom cluster locally. - -Note, that this code only launches MongoDB cluster -on succes, but does not stop it anyway. You should -stop it manually right now with this tricky hacks: - -```ps aux | grep mongo | awk '{print $2}' | xargs -L1 kill``` - -And don't forget to cleanup folders that holds shard data and logs: -**IMPORTANT NOTE** be careful where you invoke this command - -```rm -rf logs workspace``` diff --git a/recipe/mongo/example/launch_cluster/main.go b/recipe/mongo/example/launch_cluster/main.go deleted file mode 100644 index 484d9bee..00000000 --- a/recipe/mongo/example/launch_cluster/main.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "os" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/log/zap" - mongoshardedcluster "github.com/doublecloud/tross/recipe/mongo/pkg/cluster" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" -) - -func main() { - launchCluster() -} - -func launchCluster() { - lgr, err := zap.New(zap.CLIConfig(log.InfoLevel)) - if err != nil { - panic(err) - } - - // Specify path to the .yaml config file that describes your cluster - yamlConfig := "/Users/kry127/arcadia/transfer_manager/go/recipe/mongo/config/sample/auth.yaml" - - // Then parse config into our internal representation - clusterConfig, err := mongoshardedconfig.GetConfigFromYaml(yamlConfig) - if err != nil { - lgr.Error("unable to parse .yaml config file", log.Error(err), - log.String("path", yamlConfig)) - os.Exit(1) - } - - // Provide the code with environment information: - envInfo := mongoshardedcluster.EnvironmentInfo{ - // Download binary from official MongoDB site: - // https://www.mongodb.com/download-center/community/releases/archive - // Select 'Archive' option to download, unpack archieve to the folder and - // here you should specify a path to that folder - BinaryPath: "/Users/kry127/Desktop/mongodb-binaries/mongodb-macos-x86_64-5.0.12", - // Define path where to persist database data and other additional info - WorkspacePath: "/Users/kry127/Desktop/mongodb-binaries/workspace", - // Define path where to put logs and cluster configuration files - LogsPath: "/Users/kry127/Desktop/mongodb-binaries/logs", - // Advanced setting: put here dynamic libraries to use LD_PRELOAD hack - // e.g. may be useful to inject libcurl_3 dependency without installation - // of the library to the target OS - LdPreload: []string{}, - } - - // Then, you are ready to start your MongoDB sharded cluster from Go code: - cluster, err := mongoshardedcluster.StartCluster(lgr, envInfo, clusterConfig) - if err != nil { - // if error occured, all allocated resources in the intermediate stages will be freed - // until executing following line - lgr.Error("unable to start sharded cluster", log.Error(err), - log.Any("env_info", envInfo)) - os.Exit(2) - } - // don't forget to close your cluster - defer func() { - err = mongoshardedcluster.StopCluster(lgr, envInfo) - if err != nil { - lgr.Error("unable to stop sharded cluster", log.Error(err), - log.Any("env_info", envInfo)) - } - }() - - lgr.Infof("Cluster is ready! Connect to %s with credentials username=%s, auth_source=%s", - cluster.MongoSList[0].Fqdn(), - clusterConfig.PostSteps.CreateAdminUser.User, - clusterConfig.PostSteps.CreateAdminUser.AuthSource, - ) - - // here you can perform certain work with cluster - time.Sleep(10 * time.Second) -} diff --git a/recipe/mongo/example/recipe_usage/README.md b/recipe/mongo/example/recipe_usage/README.md deleted file mode 100644 index 04d335f1..00000000 --- a/recipe/mongo/example/recipe_usage/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Пример использования рецепта шардированной MongoDB в тесте - -Для описания шардированного кластера используется файл в формате `.yaml`, -который лежит в репозитори Arcadia. Чтобы добавить файл конфигурации -в тест, в файле `ya.make` данного рецепта необходимо: -1. Добавить файл конфигурации с помощью директивы `DATA` -2. Прописать файл конфигурации в переменную окружения `MONGO_SHARDED_CLUSTER_CONFIG` -3. Включить с помощью директивы `INCLUDE` рецепт шардированной MongoDB - -Получаются следующие строчки (они есть в `ya.make`): -``` -DATA(arcadia/transfer_manager/go/recipe/mongo/example/configs/auth.yaml) -ENV(MONGO_SHARDED_CLUSTER_CONFIG="transfer_manager/go/recipe/mongo/example/configs/auth.yaml") -INCLUDE(${ARCADIA_ROOT}/transfer_manager/go/recipe/mongo/recipe.inc) -``` - -При запуске теста будут опубликованы следующие переменные -для подключения к инстансу `mongos`, к которому уже -будут подключены все шарды данных: -* `MONGO_SHARDED_CLUSTER_HOST` -- mongos хост (всегда localhost) -* `MONGO_SHARDED_CLUSTER_PORT` -- mongos порт -* `MONGO_SHARDED_CLUSTER_USERNAME` -- имя пользователя -* `MONGO_SHARDED_CLUSTER_PASSWORD` -- пароль -* `MONGO_SHARDED_CLUSTER_AUTH_SOURCE` -- источник аутентификации - -**Примечание:** если вы не указали в конфигурации `postSteps->createAdminUser`, то -вам бессмысленно использовать имя пользователя, пароль и источник аутентификации -- вы можете -подключиться к кластеру только в режиме "напрямую". Пример такого -подключения на языке Go может быть найден, например, -[здесь](https://a.yandex-team.ru/arc_vcs/transfer_manager/go/recipe/mongo/pkg/cluster/mongod.go?rev=aebb2179ea#L265): -в драйвере `MongoDB` для Go можно указать Direct подключение для работы с хостами. - -Если есть необходимость развернуть два и более шардированных кластера, то -можно передать несколько конфигураций кластеров в переменную окружения `MONGO_SHARDED_CLUSTER_CONFIG`, -разделённых символом двоеточия. Например, запуск двух кластеров с описанием -`db1.yaml` и `db2.yaml`: -``` -DEPENDS(transfer_manager/go/recipe/mongo) -DATA(arcadia/transfer_manager/go/tests/e2e/mongo2mongo/rps/replication_source_sharded/db1.yaml) -DATA(arcadia/transfer_manager/go/tests/e2e/mongo2mongo/rps/replication_source_sharded/db2.yaml) -ENV(MONGO_SHARDED_CLUSTER_CONFIG="transfer_manager/go/tests/e2e/mongo2mongo/rps/replication_source_sharded/db1.yaml:transfer_manager/go/tests/e2e/mongo2mongo/rps/replication_source_sharded/db2.yaml") -INCLUDE(${ARCADIA_ROOT}/transfer_manager/go/recipe/mongo/recipe.inc) -``` - diff --git a/recipe/mongo/example/recipe_usage/sample_test.go b/recipe/mongo/example/recipe_usage/sample_test.go deleted file mode 100644 index 894fe80c..00000000 --- a/recipe/mongo/example/recipe_usage/sample_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package example - -import ( - "testing" - - "github.com/doublecloud/tross/recipe/mongo/pkg/util" -) - -func TestSample(t *testing.T) { - util.TestMongoShardedClusterRecipe(t) -} diff --git a/recipe/mongo/pkg/binurl/binary_links.go b/recipe/mongo/pkg/binurl/binary_links.go deleted file mode 100644 index ccec11c7..00000000 --- a/recipe/mongo/pkg/binurl/binary_links.go +++ /dev/null @@ -1,256 +0,0 @@ -package binurl - -import ( - "fmt" - "regexp" - "strings" - - "github.com/blang/semver/v4" - "github.com/doublecloud/tross/internal/slices" -) - -// BinaryLinks represent MongoDB binary distribution. -type BinaryLinks struct { - URL string - Tag string - Arch Arch - OS OperationSystem - Distributive string - Version semver.Version -} - -const ( - MongodbArchiveURL = "https://www.mongodb.com/download-center/community/releases/archive" - NotSpaceNorHyphenRegex = `[^ \t\r\n\f-]*` - // TagProtoRegexp capture group comments: - // #0: Tag - // #1: OS - // #2: Arch - // #3: reserved - // #4: Distributive (if any) - // #5: Version - TagProtoRegexp = `mongodb-(%s)-(%s)(-(%s))?-(%s)` - // HTMLAProtoRegexp capture group comments: - // #0: HTML tag - // #1: URL - // #2: Tag - // #3: OS - // #4: Arch - // #5: reserved - // #6: Distributive (if any) - // #7: Version - HTMLAProtoRegexp = `(` + TagProtoRegexp + `\.tgz)<\/a>` - // URLProtoRegexp capture group comments: - // #0: URL - // #1: Tag - // #2: OS - // #3: Arch - // #4: reserved - // #5: Distributive (if any) - // #6: Version - URLProtoRegexp = `https:\/\/fastdl\.mongodb\.org\/\S+\/(` + TagProtoRegexp + `)\.tgz` -) - -var ( - TagRegexp = buildRegexp(TagProtoRegexp) - HTMLARegexp = buildRegexp(HTMLAProtoRegexp) - URLRegexp = buildRegexp(URLProtoRegexp) -) - -func buildRegexp(tmpl string) *regexp.Regexp { - opSystems := slices.Map(OperationSystems, func(op OperationSystem) string { return string(op) }) - archs := slices.Map(Archs, func(op Arch) string { return string(op) }) - - re := fmt.Sprintf(tmpl, - strings.Join(opSystems, "|"), // capture group #3 - strings.Join(archs, "|"), // capture group #4 - NotSpaceNorHyphenRegex, // capture group #6 - NotSpaceNorHyphenRegex, // capture group #7 - ) - return regexp.MustCompile(re) -} - -func MakeBinaryLinkByHTMLA(HTMLA string) *BinaryLinks { - submatch := HTMLARegexp.FindStringSubmatch(HTMLA) - if submatch == nil { - return nil - } - version, err := semver.ParseTolerant(submatch[7]) - if err != nil { - version = semver.Version{} - } - return &BinaryLinks{ - URL: submatch[1], - Tag: submatch[2], - OS: ToOs(submatch[3]), - Arch: ToArch(submatch[4]), - Distributive: submatch[6], - Version: version, - } -} - -func MakeBinaryLinkByURL(URL string) *BinaryLinks { - submatch := URLRegexp.FindStringSubmatch(URL) - if submatch == nil { - return nil - } - version, err := semver.ParseTolerant(submatch[6]) - if err != nil { - version = semver.Version{} - } - return &BinaryLinks{ - URL: URL, - Tag: submatch[1], - OS: ToOs(submatch[2]), - Arch: ToArch(submatch[3]), - Distributive: submatch[5], - Version: version, - } -} - -func MakeBinaryLinkByTag(Tag string) *BinaryLinks { - submatch := TagRegexp.FindStringSubmatch(Tag) - if submatch == nil { - return nil - } - version, err := semver.ParseTolerant(submatch[5]) - if err != nil { - version = semver.Version{} - } - return &BinaryLinks{ - URL: "", - Tag: Tag, - OS: ToOs(submatch[1]), - Arch: ToArch(submatch[2]), - Distributive: submatch[4], - Version: version, - } -} - -type OperationSystem string - -var ( - OperationSystemUnspecified OperationSystem = "" - OperationSystemMacos OperationSystem = "macos" - OperationSystemOsx OperationSystem = "osx" - OperationSystemOsxSsl OperationSystem = "osx-ssl" - OperationSystemLinux OperationSystem = "linux" - - OperationSystems = []OperationSystem{ - OperationSystemMacos, - OperationSystemOsx, - OperationSystemOsxSsl, - OperationSystemLinux, - } -) - -func ToOs(os string) OperationSystem { - candidate := OperationSystem(os) - switch candidate { - case OperationSystemMacos: - fallthrough - case OperationSystemOsx: - fallthrough - case OperationSystemOsxSsl: - fallthrough - case OperationSystemLinux: - return candidate - } - return OperationSystemUnspecified -} - -type Arch string - -var ( - ArchUnspecified Arch = "" - ArchAMD64 Arch = "x86_64" - ArchARM64 Arch = "arm64" - ArchAArch64 Arch = "aarch64" - - Archs = []Arch{ - ArchAMD64, - ArchARM64, - ArchAArch64, - } -) - -func ToArch(arch string) Arch { - candidate := Arch(arch) - switch candidate { - case ArchAMD64: - fallthrough - case ArchARM64: - fallthrough - case ArchAArch64: - return candidate - } - return ArchUnspecified -} - -// TakeMaxPatchVersion method is useful when you wish to find -// the most modern patch -func TakeMaxPatchVersion(links []BinaryLinks) ([]BinaryLinks, error) { - type versionKey struct { - Arch Arch - Distributive string - Major uint64 - Minor uint64 - } - resultMap := map[versionKey]BinaryLinks{} - for _, link := range links { - key := versionKey{ - Arch: link.Arch, - Distributive: link.Distributive, - Major: link.Version.Major, - Minor: link.Version.Minor, - } - oldLink, ok := resultMap[key] - if !ok { - resultMap[key] = link - continue - } - - if oldLink.Version.LT(link.Version) { - resultMap[key] = link - continue - } - } - - result := []BinaryLinks{} - for _, link := range resultMap { - result = append(result, link) - } - return result, nil -} - -// TakeMaxDistrVersion method is useful when you wish to find -// the most modern distribution (e.g. ubuntu1804 vs ubuntu2004) -func TakeMaxDistrVersion(links []BinaryLinks) ([]BinaryLinks, error) { - type versionKey struct { - Arch Arch - Version string - } - resultMap := map[versionKey]BinaryLinks{} - for _, link := range links { - key := versionKey{ - Arch: link.Arch, - Version: link.Version.String(), - } - oldLink, ok := resultMap[key] - if !ok { - resultMap[key] = link - continue - } - - if oldLink.Distributive < link.Distributive { - resultMap[key] = link - continue - } - } - - result := []BinaryLinks{} - for _, link := range resultMap { - result = append(result, link) - } - return result, nil -} diff --git a/recipe/mongo/pkg/cluster/cluster.go b/recipe/mongo/pkg/cluster/cluster.go deleted file mode 100644 index 601a6042..00000000 --- a/recipe/mongo/pkg/cluster/cluster.go +++ /dev/null @@ -1,190 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/pkg/util" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" -) - -type Cluster struct { - MongoSList []MongoS - ShardList []ShardReplicaSet - ConfigRS ConfigReplicaSet -} - -func (c Cluster) GetAllPids() ([]int, error) { - var pids []int - for _, mongos := range c.MongoSList { - pid, err := extractPidFromPidFile(mongos.PidFilePath) - if err != nil { - return nil, xerrors.Errorf("error reading pid from file '%s': %w", mongos.PidFilePath, err) - } - pids = append(pids, pid) - } - for _, mongod := range c.ConfigRS.MongoDaemons { - pid, err := extractPidFromPidFile(mongod.PidFilePath) - if err != nil { - return nil, xerrors.Errorf("error reading pid from file '%s': %w", mongod.PidFilePath, err) - } - pids = append(pids, pid) - } - for _, shard := range c.ShardList { - for _, mongod := range shard.MongoDaemons { - pid, err := extractPidFromPidFile(mongod.PidFilePath) - if err != nil { - return nil, xerrors.Errorf("error reading pid from file '%s': %w", mongod.PidFilePath, err) - } - pids = append(pids, pid) - } - } - return pids, nil -} - -func StartCluster(logger log.Logger, envInfo EnvironmentInfo, config mongoshardedconfig.MongoShardedClusterConfig) (Cluster, error) { - logger.Infof("Starting sharded mongodb cluster with %d shards", config.ShardReplicaSet.ReplicaSetCount()) - - var rollbacks util.Rollbacks - defer rollbacks.Do() - - // launch config replica set - configReplicaSet, err := LaunchConfigReplicaSet(logger, envInfo, config.ConfigReplicaSet) - if err != nil { - return Cluster{}, xerrors.Errorf("cannot start config replica set: %w", err) - } - rollbacks.Add(func() { - err := configReplicaSet.Close() - if err != nil { - logger.Error("Unable to close config replica set in rollback", log.Error(err)) - } - }) - - // launch mongod instances for each shard - shardReplicaSets := []ShardReplicaSet{} - for i := uint(0); i < config.ShardReplicaSet.ReplicaSetCount(); i++ { - shardCfg, err := config.ShardReplicaSet.GetReplicaSetConfig(i) - if err != nil { - return Cluster{}, xerrors.Errorf("cannot get shard replica set config: %w", err) - } - defaultShardName := fmt.Sprintf("rs%02d", i+1) - shardReplicaSet, err := LaunchMongoShard(logger, envInfo, shardCfg, defaultShardName) - if err != nil { - return Cluster{}, xerrors.Errorf("cannot start shard #%d replica set: %w", i+1, err) - } - rollbacks.Add(func() { - err := shardReplicaSet.Close() - if err != nil { - logger.Error("Unable to close shard replica set in rollback", log.Error(err)) - } - }) - shardReplicaSets = append(shardReplicaSets, shardReplicaSet) - } - if len(shardReplicaSets) == 0 { - return Cluster{}, xerrors.Errorf("no data shards replica sets were launched") - } - // launch mongos instance - mongos, err := StartSingleMongos(logger, envInfo, configReplicaSet, shardReplicaSets, config.Mongos) - if err != nil { - return Cluster{}, xerrors.Errorf("error starting single mongos instance: %w", err) - } - rollbacks.Add(func() { - err := mongos.Close() - if err != nil { - logger.Error("Unable to close shard replica set in rollback", log.Error(err)) - } - }) - // perform post-actions from config - err = mongos.WithRootConnection(func(client *mongo.Client) error { - createAdminReq := config.PostSteps.CreateAdminUser - if createAdminReq.User == "" && createAdminReq.Password == "" && createAdminReq.AuthSource == "" { - // no need to create anything - return nil - } - if createAdminReq.User == "" { - return xerrors.Errorf("empty username in admin user description") - } - if createAdminReq.Password == "" { - return xerrors.Errorf("empty password in admin user description") - } - if createAdminReq.AuthSource == "" { - return xerrors.Errorf("empty auth source in admin user description") - } - r := client.Database(createAdminReq.AuthSource).RunCommand(context.TODO(), - bson.D{ - {Key: "createUser", Value: createAdminReq.User}, - {Key: "pwd", Value: createAdminReq.Password}, - {Key: "roles", Value: bson.A{ - bson.D{{Key: "db", Value: "admin"}, {Key: "role", Value: "clusterAdmin"}}, - }}, - }, - ) - if r.Err() != nil { - return xerrors.Errorf("unable to create admin user: %w", r.Err()) - } - return nil - }) - if err != nil { - return Cluster{}, xerrors.Errorf("unable to perform post-actions when configuring sharded cluster: %w", err) - } - - result := Cluster{ - MongoSList: []MongoS{mongos}, - ShardList: shardReplicaSets, - ConfigRS: configReplicaSet, - } - err = saveAllPids(result, envInfo) - if err != nil { - return Cluster{}, xerrors.Errorf("error saving all pids: %w", err) - } - - // return result - logger.Infof("Done starting sharded mongodb cluster! Connect to mongos instance 'localhost:%d'", mongos.Port) - rollbacks.Cancel() - return result, nil -} - -func saveAllPids(cluster Cluster, envInfo EnvironmentInfo) error { - pids, err := cluster.GetAllPids() - if err != nil { - return xerrors.Errorf("cannot get all pids for cluster") - } - - pidsFileContent := strings.Join( - slices.Map(pids, func(pid int) string { return fmt.Sprint(pid) }), - "\n") - return os.WriteFile(envInfo.PidFilePath(), []byte(pidsFileContent), 0755) -} - -func StopCluster(logger log.Logger, envInfo EnvironmentInfo) error { - data, err := os.ReadFile(envInfo.PidFilePath()) - if err != nil { - return xerrors.Errorf("unable to read file: %w", err) - } - - var multiErr util.Errors - pidsAsStrings := strings.Split(string(data), "\n") - for _, pidAsString := range pidsAsStrings { - pidInt, err := extractPidFromString(pidAsString) - if err != nil { - multiErr = util.AppendErr(multiErr, err) - continue - } - err = closeWithPidInt(pidInt) - if err != nil { - multiErr = util.AppendErr(multiErr, err) - continue - } - } - if !multiErr.Empty() { - return multiErr - } - return nil -} diff --git a/recipe/mongo/pkg/cluster/config_replica_set.go b/recipe/mongo/pkg/cluster/config_replica_set.go deleted file mode 100644 index e00513c8..00000000 --- a/recipe/mongo/pkg/cluster/config_replica_set.go +++ /dev/null @@ -1,96 +0,0 @@ -package shmongo - -import ( - "context" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/util" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" -) - -const ( - ConfigDefaultReplicaSetName = "rscfg" -) - -type ConfigReplicaSet struct { - ReplicaSet string - MongoDaemons []MongoD -} - -func (c *ConfigReplicaSet) Close() error { - var errs util.Errors - for _, mongod := range c.MongoDaemons { - err := mongod.Close() - errs = util.AppendErr(errs, err) - } - if len(errs) == 0 { - return nil - } - return errs -} - -func LaunchConfigReplicaSet( - logger log.Logger, - binInfo EnvironmentInfo, - config mongoshardedconfig.ReplicaSetConfig, -) (ConfigReplicaSet, error) { - if config.MongodConfigCount() == 0 { - return ConfigReplicaSet{}, xerrors.Errorf("try to launch config replica set with zero mongod instances specified") - } - ReplicaSet, err := InferReplicaSet(config, ConfigDefaultReplicaSetName) - if err != nil { - return ConfigReplicaSet{}, xerrors.Errorf("cannot infer replica set name from config: %w", err) - } - mongods, err := StartReplicaSet(logger, binInfo, config, ReplicaSet, MongoDConfigSvr) - if err != nil { - return ConfigReplicaSet{}, xerrors.Errorf("cannot start replica set: %w", err) - } - - result := ConfigReplicaSet{ReplicaSet: ReplicaSet, MongoDaemons: mongods} - - var rollbacks util.Rollbacks - defer rollbacks.Do() - rollbacks.Add(func() { - err := result.Close() - if err != nil { - logger.Error("Unable to close config replica set in rollback", log.Error(err)) - } - }) - - if len(result.MongoDaemons) == 0 { - return ConfigReplicaSet{}, xerrors.Errorf("no mongod hosts to connect to") - } - mongod := result.MongoDaemons[0] - err = mongod.WithRootConnection(func(client *mongo.Client) error { - err := result.initConfigReplicaSet(client) - if err != nil { - return xerrors.Errorf("cannot init config replica set: %w", err) - } - return nil - }) - if err != nil { - return ConfigReplicaSet{}, xerrors.Errorf("root context failed: %w", err) - } - - rollbacks.Cancel() - return result, nil -} - -func (c *ConfigReplicaSet) initConfigReplicaSet(client *mongo.Client) error { - // https://www.mongodb.com/docs/manual/reference/command/replSetInitiate/#mongodb-dbcommand-dbcmd.replSetInitiate - var members bson.A - for i, md := range c.MongoDaemons { - members = append(members, bson.D{{Key: "_id", Value: i}, {Key: "host", Value: md.Fqdn()}}) - } - client.Database("admin").RunCommand(context.TODO(), bson.D{ - {Key: "replSetInitiate", Value: bson.D{ - {Key: "_id", Value: c.ReplicaSet}, - {Key: "configsvr", Value: true}, - {Key: "members", Value: members}, - }}, - }) - return nil -} diff --git a/recipe/mongo/pkg/cluster/environment_info.go b/recipe/mongo/pkg/cluster/environment_info.go deleted file mode 100644 index c0a52ea7..00000000 --- a/recipe/mongo/pkg/cluster/environment_info.go +++ /dev/null @@ -1,30 +0,0 @@ -package shmongo - -import "path" - -const ( - EnvMongoShardedClusterHost = "MONGO_SHARDED_CLUSTER_HOST" - EnvMongoShardedClusterPort = "MONGO_SHARDED_CLUSTER_PORT" - EnvMongoShardedClusterUsername = "MONGO_SHARDED_CLUSTER_USERNAME" - EnvMongoShardedClusterPassword = "MONGO_SHARDED_CLUSTER_PASSWORD" - EnvMongoShardedClusterAuthSource = "MONGO_SHARDED_CLUSTER_AUTH_SOURCE" -) - -type EnvironmentInfo struct { - BinaryPath string // path to mongo server - WorkspacePath string // path with write access where to store cluster info and logs - LogsPath string // separate path for logs - LdPreload []string // aux dynamic libraries to preload before launch -} - -func (b EnvironmentInfo) MongoDPath() string { - return path.Join(b.BinaryPath, "bin/mongod") -} - -func (b EnvironmentInfo) MongoSPath() string { - return path.Join(b.BinaryPath, "bin/mongos") -} - -func (b EnvironmentInfo) PidFilePath() string { - return path.Join(b.LogsPath, "mongods.pid") -} diff --git a/recipe/mongo/pkg/cluster/mongod.go b/recipe/mongo/pkg/cluster/mongod.go deleted file mode 100644 index ccfa24d1..00000000 --- a/recipe/mongo/pkg/cluster/mongod.go +++ /dev/null @@ -1,290 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "os/exec" - "path" - "strconv" - "strings" - "syscall" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/util" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -type MongoD struct { - Host string - Port int - PidFilePath string -} - -func (d MongoD) Fqdn() string { - return fmt.Sprintf("%s:%d", d.Host, d.Port) -} - -type MongoDMode string - -var ( - MongoDConfigSvr MongoDMode = "configsvr" - MongoDShardSvr MongoDMode = "shardsvr" -) - -func finishProcess(process *os.Process) error { - err := process.Signal(syscall.SIGTERM) - if err != nil { - return xerrors.Errorf("error sending SIGTERM signal to process: %w", err) - } - errChan := make(chan error) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - go func() { - _, err := process.Wait() - errChan <- err - }() - select { - case err := <-errChan: - if err != nil { - return xerrors.Errorf("error while waiting for process pid=%d cleanup: %w", process.Pid, err) - } - return nil - case <-ctx.Done(): - err := process.Release() - if err != nil { - return xerrors.Errorf("unable to release resources: %w", err) - } - return process.Kill() - } -} - -func (d MongoD) Close() error { - pidInt, err := extractPidFromPidFile(d.PidFilePath) - if err != nil { - return xerrors.Errorf("cannot extract pid: %w", err) - } - return closeWithPidInt(pidInt) -} - -func extractPidFromString(pidAsString string) (int, error) { - trimmedPidAsString := strings.TrimSpace(pidAsString) - pidInt, err := strconv.Atoi(trimmedPidAsString) - if err != nil { - return 0, xerrors.Errorf("wrong pid format: %w", err) - } - return pidInt, nil -} - -func extractPidFromPidFile(pidFilePath string) (int, error) { - pidFileContent, err := os.ReadFile(pidFilePath) - if err != nil { - return 0, xerrors.Errorf("cannot read pid: %w", err) - } - return extractPidFromString(string(pidFileContent)) -} - -func closeWithPidInt(pidInt int) error { - proc, err := os.FindProcess(pidInt) - if err != nil { - return xerrors.Errorf("cannot find process: %w", err) - } - return finishProcess(proc) -} - -func InferReplicaSet(config mongoshardedconfig.ReplicaSetConfig, defaultValue string) (string, error) { - found := false - - var result string - propertyPath := []string{"replication", "replSetName"} - for i := uint(0); i < config.MongodConfigCount(); i++ { - cfg, err := config.GetMongodConfig(i) - if err != nil { - return "", xerrors.Errorf("cannot get config %d: %w", i, err) - } - rs, ok := mongoshardedconfig.GetKey(cfg, propertyPath) - if !ok { - continue - } - rss, ok := rs.(string) - if !ok { - return "", xerrors.Errorf("property '%s' should have type string", strings.Join(propertyPath, ".")) - } - if found && result != rss { - return "", xerrors.Errorf( - "config replica set conflict -- found two replica set names: %s and %s -- you should specify only one custom replica set", - result, rs) - } - result = rss - found = true - } - if !found { - // return standard name for config replica set - return defaultValue, nil - } - return result, nil -} - -func getPort(cfg interface{}) (int, error) { - var port int - userPort, hasUserPort := mongoshardedconfig.GetKey(cfg, mongoshardedconfig.PropNetPort) - userPortAsInt, isPint := userPort.(int) - - if hasUserPort && isPint && userPortAsInt != 0 { - port = userPortAsInt - } else { - freePort, err := util.GetFreePort() - if err != nil { - return 0, xerrors.Errorf("cannot get next free port: %w", err) - } - port = freePort - } - return port, nil -} - -func StartReplicaSet( - logger log.Logger, - binInfo EnvironmentInfo, - config mongoshardedconfig.ReplicaSetConfig, - replicaSet string, - mode MongoDMode, -) ([]MongoD, error) { - logger.Infof("Starting shard with replica set '%s' with %d mongod instances", replicaSet, config.MongodConfigCount()) - - if config.MongodConfigCount() == 0 { - return nil, xerrors.Errorf("try to launch config replica set with zero mongod instances specified") - } - result := []MongoD{} - var rollbacks util.Rollbacks - defer rollbacks.Do() - for i := uint(0); i < config.MongodConfigCount(); i++ { - // prepare essential configuration values and allocate machine resources - commonSubPath := fmt.Sprintf("%s_%s_mongod_%d", mode, replicaSet, i) - dbPath := path.Join(binInfo.WorkspacePath, commonSubPath, "storage") - logPath := path.Join(binInfo.LogsPath, commonSubPath, "mongod.log") - pidPath := path.Join(binInfo.WorkspacePath, commonSubPath, "mongod.pid") - - for _, pathToMake := range []string{dbPath, path.Dir(logPath), path.Dir(pidPath)} { - if err := os.MkdirAll(pathToMake, 0700); err != nil { - return nil, xerrors.Errorf("cannot make path '%s': %w", pathToMake, err) - } - } - - // get configuration - cfg, err := config.GetMongodConfig(i) - if err != nil { - return nil, xerrors.Errorf("cannot get config #%d: %w", i, err) - } - - port, err := getPort(cfg) - if err != nil { - return nil, xerrors.Errorf("cannot get port for config: %w", err) - } - - // save original user configuration for diagnostics - originalConfigPath := path.Join(binInfo.LogsPath, commonSubPath, "original.config.yaml") - err = mongoshardedconfig.ProduceMongodConfig(cfg, originalConfigPath) - if err != nil { - return nil, - xerrors.Errorf("cannot produce original config file for config replica set: %w", err) - } - - // override configuration with necessary parameters - if config.NoOverride { - logger.Warn("Skip override", log.String("replica_set", replicaSet)) - } else { - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropClusterRoleName, mode) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropReplicationSetName, replicaSet) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropStorageDBPath, dbPath) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropNetPort, port) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropNetBindIP, "localhost") - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropUnixDomainSocket, false) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropFork, true) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropPidFile, pidPath) - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropSystemLogDestination, "file") - cfg = mongoshardedconfig.OverridePathValue(cfg, mongoshardedconfig.PropSystemLogPath, logPath) - } - - // save working configuration before launching mongod instance - configPath := path.Join(binInfo.LogsPath, commonSubPath, "config.yaml") - err = mongoshardedconfig.ProduceMongodConfig(cfg, configPath) - if err != nil { - return nil, - xerrors.Errorf("cannot produce original config file for config replica set: %w", err) - } - - err = startMongoD(binInfo, configPath) - if err != nil { - return nil, xerrors.Errorf("cannot start mongod instance: %w", err) - } - mongod := MongoD{ - Host: "localhost", - Port: port, - PidFilePath: pidPath, - } - result = append(result, mongod) - rollbacks.Add(func() { - err := mongod.Close() - if err != nil { - logger.Error("Cannot close mongod instance", log.Any("params", cfg), - log.Any("mongod_description", mongod)) - } - }) - } - - rollbacks.Cancel() - return result, nil -} - -// startMongoD -- starts MongoD instance identified with subpath and configuration -// may override some settings that user set in config file -func startMongoD(binInfo EnvironmentInfo, configPath string) error { - mongoDExecPath := binInfo.MongoDPath() - argv := []string{ - "--config", configPath, - } - - cmd := exec.Command(mongoDExecPath, argv...) - ldPreloadEnv := fmt.Sprintf("LD_PRELOAD=%s", strings.Join(binInfo.LdPreload, " ")) - cmd.Env = append(cmd.Env, ldPreloadEnv) - co, err := cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("cannot start command: %w, combined output:\n%s", err, co) - } - return nil -} - -func (d MongoD) WithRootConnection(ctxFunc func(client *mongo.Client) error) (errResult error) { - hostSpec := d.Fqdn() - client, err := mongo.NewClient( - new(options.ClientOptions).SetHosts([]string{hostSpec}). - SetDirect(true), - ) - if err != nil { - return xerrors.Errorf("unable to create client for mongodb: %w", err) - } - err = client.Connect(context.TODO()) - if err != nil { - return xerrors.Errorf("unable to connect to mongodb: %w", err) - } - defer func() { - err := client.Disconnect(context.TODO()) - if err != nil { - errResult = util.NewErrs(err, errResult) - } - }() - - err = backoff.Retry(func() error { - return client.Ping(context.TODO(), nil) - }, backoff.NewExponentialBackOff()) - if err != nil { - return xerrors.Errorf("unable to ping mongod instance: %w", err) - } - - return ctxFunc(client) -} diff --git a/recipe/mongo/pkg/cluster/mongos.go b/recipe/mongo/pkg/cluster/mongos.go deleted file mode 100644 index c4fea0ee..00000000 --- a/recipe/mongo/pkg/cluster/mongos.go +++ /dev/null @@ -1,186 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "os/exec" - "path" - "strings" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/pkg/util" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -type MongoS struct { - Host string - Port int - PidFilePath string - Config ConfigReplicaSet - Shards []ShardReplicaSet -} - -func (s MongoS) Fqdn() string { - return fmt.Sprintf("%s:%d", s.Host, s.Port) -} - -func (s MongoS) Close() error { - pidInt, err := extractPidFromPidFile(s.PidFilePath) - if err != nil { - return xerrors.Errorf("cannot extract pid: %w", err) - } - return closeWithPidInt(pidInt) -} - -func StartSingleMongos( - logger log.Logger, - binInfo EnvironmentInfo, - configReplicaSet ConfigReplicaSet, - shardReplicaSets []ShardReplicaSet, - config interface{}) (MongoS, error) { - - logger.Infof("Starting mongos instance") - - var rollbacks util.Rollbacks - defer rollbacks.Do() - - // prepare essential configuration values and allocate machine resources - commonSubPath := fmt.Sprintf("mongos_%d", 0) - dbPath := path.Join(binInfo.WorkspacePath, commonSubPath, "storage") - logPath := path.Join(binInfo.LogsPath, commonSubPath, "mongod.log") - pidPath := path.Join(binInfo.WorkspacePath, commonSubPath, "mongos.pid") - - cfgHostList := strings.Join(slices.Map(configReplicaSet.MongoDaemons, func(d MongoD) string { - return d.Fqdn() - }), ",") - cfgHostsURL := fmt.Sprintf("%s/%s", configReplicaSet.ReplicaSet, cfgHostList) - - for _, pathToMake := range []string{dbPath, path.Dir(logPath), path.Dir(pidPath)} { - if err := os.MkdirAll(pathToMake, 0700); err != nil { - return MongoS{}, xerrors.Errorf("cannot make path '%s': %w", pathToMake, err) - } - } - - port, err := getPort(config) - if err != nil { - return MongoS{}, xerrors.Errorf("cannot get port for config: %w", err) - } - - // save original user configuration for diagnostics - originalConfigPath := path.Join(binInfo.LogsPath, commonSubPath, "original.config.yaml") - err = mongoshardedconfig.ProduceMongodConfig(config, originalConfigPath) - if err != nil { - return MongoS{}, - xerrors.Errorf("cannot produce original config file for config replica set: %w", err) - } - - // override configuration with necessary parameters - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropMongoSConfigDB, cfgHostsURL) - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropNetPort, port) - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropNetBindIP, "localhost") - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropUnixDomainSocket, false) - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropFork, true) - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropPidFile, pidPath) - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropSystemLogDestination, "file") - config = mongoshardedconfig.OverridePathValue(config, mongoshardedconfig.PropSystemLogPath, logPath) - - // save working configuration before launching mongod instance - configPath := path.Join(binInfo.LogsPath, commonSubPath, "config.yaml") - err = mongoshardedconfig.ProduceMongodConfig(config, configPath) - if err != nil { - return MongoS{}, - xerrors.Errorf("cannot produce original config file for config replica set: %w", err) - } - - err = launchMongos(binInfo, configPath) - if err != nil { - return MongoS{}, xerrors.Errorf("cannot start mongod instance: %w", err) - } - mongos := MongoS{ - Host: "localhost", - Port: port, - PidFilePath: pidPath, - Config: configReplicaSet, - Shards: shardReplicaSets, - } - rollbacks.Add(func() { - err := mongos.Close() - if err != nil { - logger.Error("cannot close mongos in rollback", log.Error(err)) - } - }) - - err = mongos.WithRootConnection(func(client *mongo.Client) error { - for _, shard := range shardReplicaSets { - err := mongos.addShardReplicaSet(client, shard) - if err != nil { - return xerrors.Errorf("cannot add shard to mongos: %w", err) - } - } - return nil - }) - if err != nil { - return MongoS{}, xerrors.Errorf("root context failed: %w", err) - } - - rollbacks.Cancel() - return mongos, nil -} - -func launchMongos(binInfo EnvironmentInfo, configPath string) error { - mongoSExecPath := binInfo.MongoSPath() - argv := []string{ - "--config", configPath, - } - - cmd := exec.Command(mongoSExecPath, argv...) - ldPreloadEnv := fmt.Sprintf("LD_PRELOAD=%s", strings.Join(binInfo.LdPreload, " ")) - cmd.Env = append(cmd.Env, ldPreloadEnv) - co, err := cmd.CombinedOutput() - if err != nil { - return xerrors.Errorf("cannot start command: %w, combined output:\n%s", err, co) - } - - return nil -} - -func (s MongoS) WithRootConnection(ctxFunc func(client *mongo.Client) error) (errResult error) { - hostSpec := s.Fqdn() - client, err := mongo.NewClient( - new(options.ClientOptions).SetHosts([]string{hostSpec}), - ) - if err != nil { - return xerrors.Errorf("unable to create client for mongodb: %w", err) - } - err = client.Connect(context.TODO()) - if err != nil { - return xerrors.Errorf("unable to connect to mongodb: %w", err) - } - defer func() { - err := client.Disconnect(context.TODO()) - if err != nil { - errResult = util.NewErrs(err, errResult) - } - }() - - return ctxFunc(client) -} - -func (s MongoS) addShardReplicaSet(client *mongo.Client, shard ShardReplicaSet) error { - shardHostList := strings.Join(slices.Map(shard.MongoDaemons, func(d MongoD) string { - return d.Fqdn() - }), ",") - shardHostsURL := fmt.Sprintf("%s/%s", shard.ReplicaSet, shardHostList) - sr := client.Database("admin").RunCommand(context.TODO(), - bson.D{{Key: "addShard", Value: shardHostsURL}}) - if sr.Err() != nil { - return sr.Err() - } - return nil -} diff --git a/recipe/mongo/pkg/cluster/shard_replica_set.go b/recipe/mongo/pkg/cluster/shard_replica_set.go deleted file mode 100644 index f9724103..00000000 --- a/recipe/mongo/pkg/cluster/shard_replica_set.go +++ /dev/null @@ -1,98 +0,0 @@ -package shmongo - -import ( - "context" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/util" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" -) - -type ShardReplicaSet struct { - ReplicaSet string - MongoDaemons []MongoD -} - -func (c *ShardReplicaSet) Close() error { - var errs util.Errors - for _, mongod := range c.MongoDaemons { - err := mongod.Close() - errs = util.AppendErr(errs, err) - } - if len(errs) == 0 { - return nil - } - return errs -} - -// LaunchMongoShardWithParams -// length of auxiliary `params` array equals mongod instances to be launched. -func LaunchMongoShard( - logger log.Logger, - binInfo EnvironmentInfo, - config mongoshardedconfig.ReplicaSetConfig, - defaultShardName string, -) (ShardReplicaSet, error) { - if config.MongodConfigCount() == 0 { - return ShardReplicaSet{}, xerrors.Errorf("try to launch config replica set with zero mongod instances specified") - } - ReplicaSet, err := InferReplicaSet(config, defaultShardName) - if err != nil { - return ShardReplicaSet{}, xerrors.Errorf("cannot infer replica set name from config: %w", err) - } - mongods, err := StartReplicaSet(logger, binInfo, config, ReplicaSet, MongoDShardSvr) - if err != nil { - return ShardReplicaSet{}, xerrors.Errorf("cannot start replica set: %w", err) - } - - result := ShardReplicaSet{ReplicaSet: ReplicaSet, MongoDaemons: mongods} - - var rollbacks util.Rollbacks - defer rollbacks.Do() - rollbacks.Add(func() { - err := result.Close() - if err != nil { - logger.Error("Unable to close config replica set in rollback", log.Error(err)) - } - }) - - if len(result.MongoDaemons) == 0 { - return ShardReplicaSet{}, xerrors.Errorf("no mongod hosts to connect to") - } - mongod := result.MongoDaemons[0] - err = mongod.WithRootConnection(func(client *mongo.Client) error { - err := result.initShardReplicaSet(client) - if err != nil { - return xerrors.Errorf("cannot init shard replica set: %w", err) - } - return nil - }) - if err != nil { - return ShardReplicaSet{}, xerrors.Errorf("root context failed: %w", err) - } - - rollbacks.Cancel() - return result, nil -} - -func (c *ShardReplicaSet) initShardReplicaSet(client *mongo.Client) error { - // https://www.mongodb.com/docs/manual/reference/command/replSetInitiate/#mongodb-dbcommand-dbcmd.replSetInitiate - // db.runCommand({replSetInitiate: {_id: "rs01", members: [{_id: 0, host: "localhost:56784"}, {_id: 1, host: "localhost:56785"}]}}) - var members bson.A - for i, md := range c.MongoDaemons { - members = append(members, bson.D{{Key: "_id", Value: i}, {Key: "host", Value: md.Fqdn()}}) - } - res := client.Database("admin").RunCommand(context.TODO(), bson.D{ - {Key: "replSetInitiate", Value: bson.D{ - {Key: "_id", Value: c.ReplicaSet}, - {Key: "members", Value: members}, - }}, - }) - if res.Err() != nil { - return res.Err() - } - return nil -} diff --git a/recipe/mongo/pkg/config/config.go b/recipe/mongo/pkg/config/config.go deleted file mode 100644 index 61cb254a..00000000 --- a/recipe/mongo/pkg/config/config.go +++ /dev/null @@ -1,193 +0,0 @@ -package shmongocfg - -import ( - _ "embed" - "os" - "regexp" - - "github.com/doublecloud/tross/internal/core/xerrors" - "gopkg.in/yaml.v2" -) - -const ( - MongoDefaultVersion = "4.4" -) - -var ( - MongoMajorVersionRegexp = regexp.MustCompile(`\d+.\d+`) - - PropClusterRoleName = []string{"sharding", "clusterRole"} - PropMongoSConfigDB = []string{"sharding", "configDB"} - PropReplicationSetName = []string{"replication", "replSetName"} - PropStorageDBPath = []string{"storage", "dbPath"} - PropNetPort = []string{"net", "port"} - PropNetBindIP = []string{"net", "bindIp"} - PropUnixDomainSocket = []string{"net", "unixDomainSocket", "enabled"} - PropFork = []string{"processManagement", "fork"} - PropPidFile = []string{"processManagement", "pidFilePath"} - PropSystemLogDestination = []string{"systemLog", "destination"} - PropSystemLogPath = []string{"systemLog", "path"} -) - -// MongoShardedClusterConfig repsresents config for whole sharded cluster -type MongoShardedClusterConfig struct { - MongoDBVersion string `yaml:"version"` - EnvPrefix string `yaml:"envPrefix"` - PostSteps PostStepsConfig `yaml:"postSteps"` - ConfigReplicaSet ReplicaSetConfig `yaml:"configReplicaSet"` - ShardReplicaSet ShardsConfig `yaml:"shards"` - // TODO(@kry127) support multiple mongos instances + add NoOverride support - Mongos interface{} `yaml:"mongos"` -} - -func (c *MongoShardedClusterConfig) WithDefaults() { - if c.MongoDBVersion == "" { - c.MongoDBVersion = MongoDefaultVersion - } -} - -func (c *MongoShardedClusterConfig) Validate() error { - if !MongoMajorVersionRegexp.MatchString(c.MongoDBVersion) { - return xerrors.Errorf("version '%s' does not comply strict format 'x.x', e.g. '4.4'", c.MongoDBVersion) - } - return nil -} - -type PostStepsConfig struct { - CreateAdminUser CreateAdminUserConfig `yaml:"createAdminUser"` -} - -type CreateAdminUserConfig struct { - User string `yaml:"user"` - Password string `yaml:"password"` - AuthSource string `yaml:"authSource"` -} - -// ReplicaSetConfig represents config for sharded cluster -// semantics: -// - Configs is configuration of each mongod instance in cluster. -// - If Amount and Config are specified, the configuration Config appended Amount times -// to the configuration Configs -// - if NoOverride has been explicitly set to true, then no modification -// of configuration will be performed. This is useful for legacy scenarios where some flags -// may be missing. -type ReplicaSetConfig struct { - NoOverride bool `yaml:"noOverride"` - Amount uint `yaml:"amount"` - Config interface{} `yaml:"config"` - Configs []interface{} `yaml:"configs"` -} - -func (r *ReplicaSetConfig) MongodConfigCount() uint { - return r.Amount + uint(len(r.Configs)) -} - -func (r *ReplicaSetConfig) GetMongodConfig(i uint) (interface{}, error) { - if i < r.Amount { - return r.Config, nil - } - if i-r.Amount < uint(len(r.Configs)) { - return r.Configs[i-r.Amount], nil - } - return nil, xerrors.Errorf("Invalid index of RS config: %d", i) -} - -type ShardsConfig struct { - Amount uint `yaml:"amount"` - Config ReplicaSetConfig `yaml:"shardReplicaSet"` - Configs []ReplicaSetConfig `yaml:"shardReplicaSets"` -} - -func (s *ShardsConfig) ReplicaSetCount() uint { - return s.Amount + uint(len(s.Configs)) -} - -func (s *ShardsConfig) GetReplicaSetConfig(i uint) (ReplicaSetConfig, error) { - if i < s.Amount { - return s.Config, nil - } - if i-s.Amount < uint(len(s.Configs)) { - return s.Configs[i-s.Amount], nil - } - return ReplicaSetConfig{}, xerrors.Errorf("Invalid index of RS config: %d", i) -} - -func GetConfigFromYaml(configPath string) (MongoShardedClusterConfig, error) { - configYamlFile, err := os.ReadFile(configPath) - if err != nil { - return MongoShardedClusterConfig{}, xerrors.Errorf("cannot read configuration file: %w", err) - } - - var config MongoShardedClusterConfig - - err = yaml.Unmarshal(configYamlFile, &config) - if err != nil { - return MongoShardedClusterConfig{}, xerrors.Errorf("cannot unmarshal configuration: %w", err) - } - - config.WithDefaults() - if err := config.Validate(); err != nil { - return MongoShardedClusterConfig{}, xerrors.Errorf("validation error: %w", err) - } - return config, nil -} - -func ProduceMongodConfig(object interface{}, path string) error { - bytes, err := yaml.Marshal(object) - if err != nil { - return xerrors.Errorf("cannot marshal configuration object: %w", err) - } - err = os.WriteFile(path, bytes, 0644) - if err != nil { - return xerrors.Errorf("cannot write configuration object as yaml: %w", err) - } - return nil -} - -func OverridePathValue(original interface{}, path []string, value interface{}) interface{} { - if len(path) == 0 { - return value - } - originalMap, isOriginalMap := original.(map[string]interface{}) - if !isOriginalMap { - return OverridePathValue(map[string]interface{}{}, path, value) - } - result := make(map[string]interface{}, len(originalMap)) - for k, v := range originalMap { - result[k] = v - } - - key := path[0] - var mergeWith interface{} = map[string]interface{}{} - oldValue, hasOldValue := result[key] - if hasOldValue { - mergeWith = oldValue - } - result[key] = OverridePathValue(mergeWith, path[1:], value) - - return result -} - -func GetKey(original interface{}, path []string) (interface{}, bool) { - if len(path) == 0 { - return original, true - } - originalMap, isOriginalMap := original.(map[string]interface{}) - if !isOriginalMap { - return nil, false - } - - key := path[0] - value, hasValue := originalMap[key] - if !hasValue { - return nil, false - } - return GetKey(value, path[1:]) -} - -func defaultizePathValue(original interface{}, path []string, value interface{}) interface{} { - if _, ok := GetKey(original, path); ok { - return OverridePathValue(original, path, value) - } - return original -} diff --git a/recipe/mongo/pkg/tar/tar.go b/recipe/mongo/pkg/tar/tar.go deleted file mode 100644 index b9132328..00000000 --- a/recipe/mongo/pkg/tar/tar.go +++ /dev/null @@ -1,90 +0,0 @@ -package tarutil - -import ( - "archive/tar" - "compress/gzip" - "fmt" - "io" - "os" - "path" - - "github.com/doublecloud/tross/internal/core/xerrors" -) - -func TarXzf(path string) (*tar.Reader, func() error, error) { - dummyFunc := func() error { return nil } - f, err := os.Open(path) - if err != nil { - return nil, dummyFunc, xerrors.Errorf("cannot open file '%s': %w", path, err) - } - gzw, err := gzip.NewReader(f) - if err != nil { - return nil, dummyFunc, xerrors.Errorf("cannot open gzip file '%s': %w", path, err) - } - return tar.NewReader(gzw), func() error { - err := f.Close() - if err != nil { - return err - } - return nil - }, err -} - -func UnpackArchive(archivePath, destinationPath string) error { - tarReader, streamClose, err := TarXzf(archivePath) - if err != nil { - return xerrors.Errorf("cannot open tar stream: %v", err) - } - defer func() { - err := streamClose() - if err != nil { - fmt.Printf("error occured on metaarchive stream close '%s': %v\n", archivePath, err) - } - }() - - return UnpackTar(tarReader, destinationPath, func(filename string) bool { return true }) -} - -func UnpackTar(tarReader *tar.Reader, destinationPath string, predicate func(string) bool) error { - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return xerrors.Errorf("next failed: %w", err) - } - - if predicate != nil && !predicate(header.Name) { - continue - } - - switch header.Typeflag { - case tar.TypeDir: - dirPath := path.Join(destinationPath, header.Name) - if err := os.MkdirAll(dirPath, 0755); err != nil { - return xerrors.Errorf("cannot create directory: %w", err) - } - case tar.TypeReg: - fullPath := path.Join(destinationPath, header.Name) - if err := os.MkdirAll(path.Dir(fullPath), 0755); err != nil { - return xerrors.Errorf("cannot create directory: %w", err) - } - err := func() error { - outFile, err := os.OpenFile(fullPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755) - if err != nil { - return xerrors.Errorf("cannot create file: %w", err) - } - defer outFile.Close() - if _, err := io.Copy(outFile, tarReader); err != nil { - return xerrors.Errorf("cannot copy contents to file: %w", err) - } - return nil - }() - if err != nil { - fmt.Printf("file '%s' extraction failure: %v\n", header.Name, err) - } - } - } - return nil -} diff --git a/recipe/mongo/pkg/util/test_common.go b/recipe/mongo/pkg/util/test_common.go deleted file mode 100644 index fead4989..00000000 --- a/recipe/mongo/pkg/util/test_common.go +++ /dev/null @@ -1,69 +0,0 @@ -package util - -import ( - "context" - "fmt" - "os" - "testing" - - mongoshardedcluster "github.com/doublecloud/tross/recipe/mongo/pkg/cluster" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func TestMongoShardedClusterRecipe(t *testing.T) { - require.True(t, true, "should be true") - for _, envVariable := range []string{ - mongoshardedcluster.EnvMongoShardedClusterHost, - mongoshardedcluster.EnvMongoShardedClusterPort, - mongoshardedcluster.EnvMongoShardedClusterUsername, - mongoshardedcluster.EnvMongoShardedClusterPassword, - mongoshardedcluster.EnvMongoShardedClusterAuthSource, - } { - _, ok := os.LookupEnv(envVariable) - require.True(t, ok, fmt.Sprintf("environment variable %s should be published "+ - "after successfully started sharded mongo recipe", envVariable)) - } - - hostSpec := fmt.Sprintf("%s:%s", - os.Getenv(mongoshardedcluster.EnvMongoShardedClusterHost), - os.Getenv(mongoshardedcluster.EnvMongoShardedClusterPort), - ) - client, err := mongo.NewClient( - new(options.ClientOptions). - SetHosts([]string{hostSpec}). - SetAuth(options.Credential{ - AuthMechanism: "", - AuthMechanismProperties: nil, - AuthSource: os.Getenv(mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Username: os.Getenv(mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: os.Getenv(mongoshardedcluster.EnvMongoShardedClusterPassword), - PasswordSet: false, - }), - ) - require.NoError(t, err) - err = client.Connect(context.Background()) - require.NoError(t, err) - defer func() { - err := client.Disconnect(context.TODO()) - require.NoError(t, err) - }() - - require.NoError(t, client.Ping(context.Background(), nil)) - - result, err := client.Database("test").Collection("test").InsertOne(context.Background(), - bson.D{{Key: "hello", Value: "world!"}}) - require.NoError(t, err) - require.NotNil(t, result) - require.NotNil(t, result.InsertedID) - - sr := client.Database("test").Collection("test").FindOne(context.Background(), - bson.D{{Key: "_id", Value: result.InsertedID}}) - require.NotNil(t, sr) - var dest bson.M - err = sr.Decode(&dest) - require.NoError(t, err) - require.Equal(t, dest["hello"], "world!") -} diff --git a/recipe/mongo/pkg/util/yatest.go b/recipe/mongo/pkg/util/yatest.go deleted file mode 100644 index 5d7b9af7..00000000 --- a/recipe/mongo/pkg/util/yatest.go +++ /dev/null @@ -1,21 +0,0 @@ -package util - -import ( - "github.com/doublecloud/tross/internal/test/yatest" -) - -func MakeOutputPath(relativePath string) string { - if yatest.HasRAMDrive() { - return yatest.OutputRAMDrivePath(relativePath) - } else { - return yatest.OutputPath(relativePath) - } -} - -func MakeWorkPath(relativePath string) string { - if yatest.HasRAMDrive() { - return yatest.RAMDrivePath(relativePath) - } else { - return yatest.WorkPath(relativePath) - } -} diff --git a/recipe/mongo/recipe.go b/recipe/mongo/recipe.go deleted file mode 100644 index 42078faa..00000000 --- a/recipe/mongo/recipe.go +++ /dev/null @@ -1,310 +0,0 @@ -// Basic instruction for Go recipes: -// https://docs.yandex-team.ru/devtools/test/environment#create-recipe -package main - -import ( - "encoding/json" - "fmt" - "os" - "path" - "strings" - - "github.com/blang/semver/v4" - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/log/zap" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/internal/test/recipe" - "github.com/doublecloud/tross/internal/test/yatest" - "github.com/doublecloud/tross/pkg/util" - "github.com/doublecloud/tross/recipe/mongo/pkg/binurl" - mongoshardedcluster "github.com/doublecloud/tross/recipe/mongo/pkg/cluster" - mongoshardedconfig "github.com/doublecloud/tross/recipe/mongo/pkg/config" - tarutil "github.com/doublecloud/tross/recipe/mongo/pkg/tar" - pathutil "github.com/doublecloud/tross/recipe/mongo/pkg/util" -) - -type shardedMongoRecipe struct{} - -func (y *shardedMongoRecipe) isLaunchedInDistBuild() bool { - val, ok := os.LookupEnv("YA_TEST_RUNNER") - return ok && val != "" -} - -func (y *shardedMongoRecipe) publishEnvVariables( - config mongoshardedconfig.MongoShardedClusterConfig, - cluster mongoshardedcluster.Cluster, -) error { - mongos := cluster.MongoSList[0] - // this is what you get when start recipe - for key, value := range map[string]string{ - config.EnvPrefix + mongoshardedcluster.EnvMongoShardedClusterHost: mongos.Host, - config.EnvPrefix + mongoshardedcluster.EnvMongoShardedClusterPort: fmt.Sprint(mongos.Port), - config.EnvPrefix + mongoshardedcluster.EnvMongoShardedClusterUsername: fmt.Sprintf(config.PostSteps.CreateAdminUser.User), - config.EnvPrefix + mongoshardedcluster.EnvMongoShardedClusterPassword: fmt.Sprintf(config.PostSteps.CreateAdminUser.Password), - config.EnvPrefix + mongoshardedcluster.EnvMongoShardedClusterAuthSource: fmt.Sprintf(config.PostSteps.CreateAdminUser.AuthSource), - } { - recipe.SetEnv(key, value) - } - return nil -} - -var ( - MongoVersion3 = semver.MustParse("3.0.0") - MongoVersion4 = semver.MustParse("4.0.0") - MongoVersion5 = semver.MustParse("5.0.0") - MongoVersion6 = semver.MustParse("6.0.0") - - MongoVersion4Dot4 = semver.MustParse("4.4.0") -) - -const ( - MongoBinaryRoot = "sharded-mongodb/mongodb-binaries" - SharedObjectsPath = "sharded-mongodb/so/" - LibCurlSO440 = "libcurl.so.4.4.0" - LibCurlSO450 = "libcurl.so.4.5.0" - LibIdnSO = "libidn.so.11" - LibSslSO100 = "libssl.so.1.0.0" - LibCryptoSO = "libcrypto.so.1.1" -) - -func (y *shardedMongoRecipe) archiveNamesToBinaryLinks() ([]binurl.BinaryLinks, error) { - entries, err := os.ReadDir(yatest.WorkPath(MongoBinaryRoot)) - if err != nil { - return nil, xerrors.Errorf("cannot list meta archive root: %w", err) - } - - result := []binurl.BinaryLinks{} - - for _, entry := range entries { - if strings.HasSuffix(entry.Name(), ".tgz") { - link := binurl.MakeBinaryLinkByTag(strings.TrimSuffix(entry.Name(), ".tgz")) - if link == nil { - continue - } - result = append(result, *link) - } - } - return result, nil -} - -// selectBinaryPrefix function selects the best packet (MongoDB binary) in packet with packets -func (y *shardedMongoRecipe) selectSuitableBinaryLink(binaryLinks []binurl.BinaryLinks, version string) (binurl.BinaryLinks, error) { - links := slices.Filter(binaryLinks, func(binLink binurl.BinaryLinks) bool { - return version == fmt.Sprintf("%d.%d", binLink.Version.Major, binLink.Version.Minor) - }) - maxPatchLinks, err := binurl.TakeMaxPatchVersion(links) - if err == nil { - // no return error: best effort on some archive - links = maxPatchLinks - } - maxDistrLinks, err := binurl.TakeMaxDistrVersion(links) - if err == nil { - // no return error: best effort on some archive - links = maxDistrLinks - } - - if len(links) == 0 { - return binurl.BinaryLinks{}, xerrors.Errorf("couldn't find suitable MongoDB binary for version '%s'", version) - } - - return links[0], nil -} - -// produceMongoDBBinaryPath looks up existing archives, finds the -// most suitable and extracts it, then returns MongoDB path for -// further use. Specify version parameter to narrow down desired archives -func (y *shardedMongoRecipe) produceMongoDBBinaryPath(version string) (string, error) { - links, err := y.archiveNamesToBinaryLinks() - if err != nil { - return "", xerrors.Errorf("cannot list MongoDB binaries to extract platform and version info: %w", err) - } - binLink, err := y.selectSuitableBinaryLink(links, version) - if err != nil { - return "", xerrors.Errorf("cannot find suitable MongoDB binary archive: %w", err) - } - destination := path.Join(MongoBinaryRoot, binLink.Tag) - source := fmt.Sprintf("%s.tgz", destination) - err = tarutil.UnpackArchive(yatest.WorkPath(source), yatest.WorkPath(destination)) - if err != nil { - return "", xerrors.Errorf("unable to unpack archive '%s' to '%s': %w", source, destination, err) - } - - dirEntries, err := os.ReadDir(destination) - if err != nil { - return "", xerrors.Errorf("cannot read destination archive directory: %w", err) - } - if len(dirEntries) > 1 { - return "", xerrors.Errorf("archive content is ambiguous: %s", - slices.Map(dirEntries, func(entry os.DirEntry) string { return entry.Name() }), - ) - } - return path.Join(destination, dirEntries[0].Name()), nil -} - -func (y *shardedMongoRecipe) startCluster(logger log.Logger, cfgID int, configFilePath string) ( - mongoshardedcluster.Cluster, - mongoshardedconfig.MongoShardedClusterConfig, - mongoshardedcluster.EnvironmentInfo, - error, -) { - var dummyLgr mongoshardedcluster.Cluster - var dummyCfg mongoshardedconfig.MongoShardedClusterConfig - var dummyEnvInfo mongoshardedcluster.EnvironmentInfo - absoluteConfigFilePath := yatest.SourcePath(configFilePath) - clusterConfig, err := mongoshardedconfig.GetConfigFromYaml(absoluteConfigFilePath) - if err != nil { - return dummyLgr, dummyCfg, dummyEnvInfo, xerrors.Errorf("Cannot parse config: %w", err) - } - - mongoBinaryPath, err := y.produceMongoDBBinaryPath(clusterConfig.MongoDBVersion) - if err != nil { - return dummyLgr, dummyCfg, dummyEnvInfo, xerrors.Errorf("failed unpacking any MongoDB binary from meta archive: %w", err) - } - - logger.Info("MongoDB binary selected", log.String("name", mongoBinaryPath)) - - mongoVersion, err := semver.ParseTolerant(clusterConfig.MongoDBVersion) - if err != nil { - return dummyLgr, dummyCfg, dummyEnvInfo, xerrors.Errorf("cannot parse config version: %w", err) - } - - libCurl440Path := yatest.WorkPath(path.Join(SharedObjectsPath, LibCurlSO440)) - libCurl450Path := yatest.WorkPath(path.Join(SharedObjectsPath, LibCurlSO450)) - LibIdnSOPath := yatest.WorkPath(path.Join(SharedObjectsPath, LibIdnSO)) - LibSslSO110Path := yatest.WorkPath(path.Join(SharedObjectsPath, LibSslSO100)) - LibCryptoSOPath := yatest.WorkPath(path.Join(SharedObjectsPath, LibCryptoSO)) - - // specifying curl dependency - ldPreload := []string{} - if mongoVersion.GTE(MongoVersion4Dot4) { - ldPreload = append(ldPreload, libCurl450Path, LibIdnSOPath, LibSslSO110Path, LibCryptoSOPath) - } else { - if y.isLaunchedInDistBuild() { - return dummyLgr, dummyCfg, dummyEnvInfo, xerrors.Errorf("MongoDB versions lower than 4.4 is not supported in recipes because distbuild uses Ubuntu2004 (see full list of binaries: https://www.mongodb.com/download-center/community/releases/archive)") - } - } - if mongoVersion.GTE(MongoVersion3) && mongoVersion.LT(MongoVersion4Dot4) { - ldPreload = append(ldPreload, libCurl440Path) - } - - envInfo := mongoshardedcluster.EnvironmentInfo{ - BinaryPath: yatest.WorkPath(mongoBinaryPath), - WorkspacePath: pathutil.MakeWorkPath(fmt.Sprintf("workspace_%d", cfgID)), - LogsPath: pathutil.MakeOutputPath(fmt.Sprintf("logs_%d", cfgID)), - LdPreload: ldPreload, - } - - cluster, err := mongoshardedcluster.StartCluster(logger, envInfo, clusterConfig) - if err != nil { - return dummyLgr, dummyCfg, dummyEnvInfo, xerrors.Errorf("cannot start sharded cluster: %w", err) - } - - allPids, _ := cluster.GetAllPids() - logger.Info("cluster has been successfully started!", - log.Ints("ports_mongos", slices.Map(cluster.MongoSList, func(s mongoshardedcluster.MongoS) int { - return s.Port - })), - log.Ints("all_pids", allPids), - ) - - return cluster, clusterConfig, envInfo, nil -} - -func (y *shardedMongoRecipe) Start() error { - logger, err := zap.New(zap.CLIConfig(log.InfoLevel)) - if err != nil { - return xerrors.Errorf("cannot create logger: %w", err) - } - - logger.Info("Starting MongoDB sharded cluster recipe...") - - configFilePaths, ok := os.LookupEnv("MONGO_SHARDED_CLUSTER_CONFIG") - if !ok { - return xerrors.Errorf("config file is not specified, you should specify it via env variable MONGO_SHARDED_CLUSTER_CONFIG " + - "in 'ya.make' file like this: ENV(MONGO_SHARDED_CLUSTER_CONFIG=\"cluster_config.yaml\"." + - "Note, that you should also add directive DATA(arcadia//cluster_config.yaml)") - } - logger.Infof("Using config files: %s", configFilePaths) - - configFilePathsArr := strings.Split(configFilePaths, ":") - if len(configFilePathsArr) == 0 { - return xerrors.Errorf("zero config files specified!") - } - - var rollbacks util.Rollbacks - defer rollbacks.Do() - envInfos := []mongoshardedcluster.EnvironmentInfo{} - for cfgID, configFilePath := range configFilePathsArr { - logger.Info("Process config", - log.Int("config_index", cfgID), - log.String("config_path", configFilePath), - ) - cluster, clusterConfig, envInfo, err := y.startCluster(logger, cfgID, configFilePath) - if err != nil { - return xerrors.Errorf("cannot start cluster #%d: %w", cfgID+1, err) - } - - rollbacks.Add(func() { - err = mongoshardedcluster.StopCluster(logger, envInfo) - if err != nil { - logger.Error("Cannot stop mongo sharded cluster", log.Error(err)) - } - }) - - err = y.publishEnvVariables(clusterConfig, cluster) - if err != nil { - return xerrors.Errorf("Unable to publish env variables: %w", err) - } - - envInfos = append(envInfos, envInfo) - } - - marshalledEnvInfo, err := json.Marshal(envInfos) - if err != nil { - return xerrors.Errorf("cannot marshal recipe environment info: %w", err) - } - - envInfoPath := yatest.WorkPath("env.info.json") - err = os.WriteFile(envInfoPath, marshalledEnvInfo, 0755) - if err != nil { - return xerrors.Errorf("cannot write recipe environment info: %w", err) - } - - rollbacks.Cancel() - return nil -} - -func (y *shardedMongoRecipe) Stop() error { - logger, err := zap.New(zap.CLIConfig(log.InfoLevel)) - if err != nil { - return xerrors.Errorf("cannot create logger: %w", err) - } - - logger.Infof("Stopping MongoDB sharded cluster recipe...") - - envInfoPath := yatest.WorkPath("env.info.json") - marshalledEnvInfo, err := os.ReadFile(envInfoPath) - if err != nil { - return xerrors.Errorf("cannot read recipe environment info from file: %w", err) - } - - var envInfos []mongoshardedcluster.EnvironmentInfo - err = json.Unmarshal(marshalledEnvInfo, &envInfos) - if err != nil { - return xerrors.Errorf("cannot unmarshal recipe environment info from file: %w", err) - } - - for i, envInfo := range envInfos { - err = mongoshardedcluster.StopCluster(logger, envInfo) - if err != nil { - logger.Error("Cannot stop sharded cluster", log.Int("index", i), log.Error(err)) - } - } - - return nil -} - -func main() { - recipe.Run(&shardedMongoRecipe{}) -} diff --git a/recipe/mongo/test/4.4/cluster_test.go b/recipe/mongo/test/4.4/cluster_test.go deleted file mode 100644 index 894fe80c..00000000 --- a/recipe/mongo/test/4.4/cluster_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package example - -import ( - "testing" - - "github.com/doublecloud/tross/recipe/mongo/pkg/util" -) - -func TestSample(t *testing.T) { - util.TestMongoShardedClusterRecipe(t) -} diff --git a/recipe/mongo/test/4.4/mongocluster.yaml b/recipe/mongo/test/4.4/mongocluster.yaml deleted file mode 100644 index a21c68c0..00000000 --- a/recipe/mongo/test/4.4/mongocluster.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: 4.4 -configReplicaSet: - amount: 1 - config: -shards: - amount: 1 - shardReplicaSet: - amount: 1 - config: -mongos: - security: - authorization: enabled -postSteps: - createAdminUser: - user: user - password: P@ssw0rd - authSource: db diff --git a/recipe/mongo/test/5.0/cluster_test.go b/recipe/mongo/test/5.0/cluster_test.go deleted file mode 100644 index 894fe80c..00000000 --- a/recipe/mongo/test/5.0/cluster_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package example - -import ( - "testing" - - "github.com/doublecloud/tross/recipe/mongo/pkg/util" -) - -func TestSample(t *testing.T) { - util.TestMongoShardedClusterRecipe(t) -} diff --git a/recipe/mongo/test/5.0/mongocluster.yaml b/recipe/mongo/test/5.0/mongocluster.yaml deleted file mode 100644 index 4afe7ef9..00000000 --- a/recipe/mongo/test/5.0/mongocluster.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: 5.0 -configReplicaSet: - amount: 1 - config: -shards: - amount: 1 - shardReplicaSet: - amount: 1 - config: -mongos: - security: - authorization: enabled -postSteps: - createAdminUser: - user: user - password: P@ssw0rd - authSource: db diff --git a/recipe/mongo/test/6.0/cluster_test.go b/recipe/mongo/test/6.0/cluster_test.go deleted file mode 100644 index 894fe80c..00000000 --- a/recipe/mongo/test/6.0/cluster_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package example - -import ( - "testing" - - "github.com/doublecloud/tross/recipe/mongo/pkg/util" -) - -func TestSample(t *testing.T) { - util.TestMongoShardedClusterRecipe(t) -} diff --git a/recipe/mongo/test/6.0/mongocluster.yaml b/recipe/mongo/test/6.0/mongocluster.yaml deleted file mode 100644 index 074ceae0..00000000 --- a/recipe/mongo/test/6.0/mongocluster.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: 6.0 -configReplicaSet: - amount: 1 - config: -shards: - amount: 1 - shardReplicaSet: - amount: 1 - config: -mongos: - security: - authorization: enabled -postSteps: - createAdminUser: - user: user - password: P@ssw0rd - authSource: db diff --git a/tests/canon/all_db_test.go b/tests/canon/all_db_test.go deleted file mode 100644 index 0700d5ce..00000000 --- a/tests/canon/all_db_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package canon - -import ( - "testing" - - "github.com/doublecloud/tross/pkg/providers/clickhouse" - "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/ydb" - "github.com/doublecloud/tross/pkg/providers/yt" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/stretchr/testify/require" -) - -func TestAll(t *testing.T) { - cases := All( - ydb.ProviderType, - yt.ProviderType, - mongo.ProviderType, - clickhouse.ProviderType, - mysql.ProviderType, - postgres.ProviderType, - ) - for _, tc := range cases { - t.Run(tc.String(), func(t *testing.T) { - require.NotEmpty(t, tc.Data) - snkr := validator.Referencer(t)() - require.NoError(t, snkr.Push(tc.Data)) - require.NoError(t, snkr.Close()) - }) - } -} diff --git a/tests/canon/clickhouse/README.md b/tests/canon/clickhouse/README.md deleted file mode 100644 index d56788f5..00000000 --- a/tests/canon/clickhouse/README.md +++ /dev/null @@ -1,19 +0,0 @@ -### local run - -Start docker for server: -```shell -docker run -d --name some-clickhouse-server -e CLICKHOUSE_USER=default -e CLICKHOUSE_DB=canon -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -p 18123:8123 -p 19000:9000 --ulimit nofile=262144:262144 clickhouse/clickhouse-server -``` - -pipe snapshot dump into client -```shell -cat transfer_manager/go/tests/canon/clickhouse/snapshot/data.sql | clickhouse-client --host localhost --port 19000 --user default --multiline --multiquery -``` - -Pass env variables to test: - -```shell -RECIPE_CLICKHOUSE_HTTP_PORT=18123 RECIPE_CLICKHOUSE_NATIVE_PORT=19000 -``` - -Enjoy diff --git a/tests/canon/clickhouse/canon_test.go b/tests/canon/clickhouse/canon_test.go deleted file mode 100644 index 7f14314e..00000000 --- a/tests/canon/clickhouse/canon_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package clickhouse - -import ( - "os" - "sort" - "strings" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse" - "github.com/doublecloud/tross/pkg/providers/clickhouse/columntypes" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" -) - -func getID(item abstract.ChangeItem) uint64 { - var index int - var found bool - for i, name := range item.ColumnNames { - if name == "id" { - index = i - found = true - break - } - } - - if !found { - return 0 - } - - id, ok := item.ColumnValues[index].(uint64) - if !ok { - return 0 - } - - return id -} - -func sortItems(item []abstract.ChangeItem) []abstract.ChangeItem { - sort.Slice(item, func(i, j int) bool { - return getID(item[i]) < getID(item[j]) - }) - return item -} - -func getBaseType(colSchema abstract.ColSchema) string { - return columntypes.BaseType(strings.TrimPrefix(colSchema.OriginalType, "ch:")) -} - -func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - Source := &model.ChSource{ - ShardsList: []model.ClickHouseShard{ - { - Name: "_", - Hosts: []string{ - "localhost", - }, - }, - }, - User: "default", - Password: "", - Database: "canon", - HTTPPort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_HTTP_PORT"), - NativePort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_NATIVE_PORT"), - } - Source.WithDefaults() - - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(Source), - validator.InitDone(t), - validator.ValuesTypeChecker, - validator.Canonizator(t, sortItems), - validator.TypesystemChecker(clickhouse.ProviderType, getBaseType), - ), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotOnly, - ) - _ = helpers.Activate(t, transfer) -} diff --git a/tests/canon/clickhouse/snapshot/data.sql b/tests/canon/clickhouse/snapshot/data.sql deleted file mode 100644 index 5cefdc1a..00000000 --- a/tests/canon/clickhouse/snapshot/data.sql +++ /dev/null @@ -1,162 +0,0 @@ -CREATE DATABASE IF NOT EXISTS canon; - -/*SET allow_experimental_object_type=1; -- Needs for JSON type */ - -CREATE TABLE canon.table -( - id UInt64, - - /* All data types */ - - col_Int8 Int8, - col_Int16 Int16, - col_Int32 Int32, - col_Int64 Int64, - /* col_Int128 Int128, -- Error: unhandled type Int128 */ - /* col_Int256 Int256, -- Error: unhandled type Int256 */ - - col_UInt8 UInt8, - col_UInt16 UInt16, - col_UInt32 UInt32, - col_UInt64 UInt64, - /* col_UInt128 UInt128, -- Error: unhandled type UInt128 */ - /* col_UInt256 UInt256, -- Error: unhandled type UInt256 */ - - col_Float32 Float32, - col_Float64 Float64, - - col_Decimal32 Decimal32(2), - col_Decimal64 Decimal64(3), - col_Decimal128 Decimal128(4), - col_Decimal256 Decimal256(5), - - /* col_Bool Bool, -- Error: unhandled type Bool */ - - col_String String, - - col_FixedString_1 FixedString(32), - col_FixedString_2 FixedString(64), - - col_UUID UUID, - - col_Date Date, - /* col_Date32 Date32, -- Error: unhandled type Date32 */ - col_DateTime DateTime('UTC'), - col_DateTime64_1 DateTime64(0, 'UTC'), - col_DateTime64_2 DateTime64(3, 'UTC'), - col_DateTime64_3 DateTime64(6, 'UTC'), - - col_enum_1 Enum('a' = -128, 'b' = 127), - col_enum_2 Enum('a' = -32768, 'b' = 32767), - - /* col_JSON JSON, -- Error: unhandled type 'a Int8', unhandled type 'a String' */ - - /* Nullable */ - - col_n_Int8 Nullable(Int8), - col_n_Int16 Nullable(Int16), - col_n_Int32 Nullable(Int32), - col_n_Int64 Nullable(Int64), - /* col_n_Int128 Nullable(Int128), -- Error: unhandled type Int128 */ - /* col_n_Int256 Nullable(Int256), -- Error: unhandled type Int256 */ - - col_n_UInt8 Nullable(UInt8), - col_n_UInt16 Nullable(UInt16), - col_n_UInt32 Nullable(UInt32), - col_n_UInt64 Nullable(UInt64), - /* col_n_UInt128 Nullable(UInt128), -- Error: unhandled type UInt128 */ - /* col_n_UInt256 Nullable(UInt256), -- Error: unhandled type UInt256 */ - - col_n_Float32 Nullable(Float32), - col_n_Float64 Nullable(Float64), - - col_n_Decimal32 Nullable(Decimal32(2)), - col_n_Decimal64 Nullable(Decimal64(3)), - /* col_n_Decimal128 Nullable(Decimal128(4)), -- Error: reflect: call of reflect.Value.Interface on zero Value */ - /* col_n_Decimal256 Nullable(Decimal256(5)), -- Error: Nullable(T): precision of Decimal exceeds max bound */ - - /* col_n_Bool Nullable(Bool), -- Error: unhandled type Bool */ - - col_n_String Nullable(String), - - col_n_FixedString_1 Nullable(FixedString(32)), - col_n_FixedString_2 Nullable(FixedString(64)), - - col_n_UUID Nullable(UUID), - - col_n_Date Nullable(Date), - /* col_n_Date32 Nullable(Date32), -- Error: unhandled type Date32 */ - col_n_DateTime Nullable(DateTime('UTC')), - col_n_DateTime64_1 Nullable(DateTime64(0, 'UTC')), - col_n_DateTime64_2 Nullable(DateTime64(3, 'UTC')), - col_n_DateTime64_3 Nullable(DateTime64(6, 'UTC')), - - col_n_enum_1 Nullable(Enum('a' = -128, 'b' = 127)), - col_n_enum_2 Nullable(Enum('a' = -32768, 'b' = 32767)), - - /* Arrays */ - - col_arr_Int8 Array(Nullable(Int8)), - col_arr_Int16 Array(Nullable(Int16)), - col_arr_Int32 Array(Nullable(Int32)), - col_arr_Int64 Array(Nullable(Int64)), - /* col_arr_Int128 Array(Nullable(Int128)), -- Error: unhandled type Int128 */ - /* col_arr_Int256 Array(Nullable(Int256)), -- Error: unhandled type Int256 */ - - col_arr_UInt8 Array(Nullable(UInt8)), - col_arr_UInt16 Array(Nullable(UInt16)), - col_arr_UInt32 Array(Nullable(UInt32)), - col_arr_UInt64 Array(Nullable(UInt64)), - /* col_arr_UInt128 Array(Nullable(UInt128)), -- Error: unhandled type UInt128 */ - /* col_arr_UInt256 Array(Nullable(UInt256)), -- Error: unhandled type UInt256 */ - - col_arr_Float32 Array(Nullable(Float32)), - col_arr_Float64 Array(Nullable(Float64)), - - col_arr_Decimal32 Array(Nullable(Decimal32(2))), - col_arr_Decimal64 Array(Nullable(Decimal64(3))), - /* col_arr_Decimal128 Array(Nullable(Decimal128(4))), -- Error: unsupported Array type */ - /* col_arr_Decimal256 Array(Nullable(Decimal256(5))), -- Error: Array(T): Nullable(T): precision of Decimal exceeds max bound */ - - /* col_arr_Bool Array(Nullable(Bool)), -- Error: unhandled type Bool */ - - col_arr_String Array(Nullable(String)), - - col_arr_FixedString_1 Array(Nullable(FixedString(32))), - col_arr_FixedString_2 Array(Nullable(FixedString(64))), - - col_arr_UUID Array(Nullable(UUID)), - - /* - Arrays with dates/datetimes reads incorrectly, with strange timezones (+3h/+2.5h) - col_arr_Date Array(Nullable(Date)), - col_arr_Date32 Array(Nullable(Date32)), -- Error: unhandled type Date32 - col_arr_DateTime Array(Nullable(DateTime('UTC'))), - col_arr_DateTime64_1 Array(Nullable(DateTime64(0, 'UTC'))), - col_arr_DateTime64_2 Array(Nullable(DateTime64(3, 'UTC'))), - col_arr_DateTime64_3 Array(Nullable(DateTime64(6, 'UTC'))), - */ - - col_arr_enum_1 Array(Nullable(Enum('a' = -128, 'b' = 127))), - col_arr_enum_2 Array(Nullable(Enum('a' = -32768, 'b' = 32767))) - - /* Maps */ - - /* - Map types are not supported, error: unhandled type Map(,) - col_map_string_n_string Map(String, Nullable(String)), - col_map_int_n_int Map(Int32, Nullable(Int32)), - col_map_fixedstring_n_fixedstring Map(FixedString(32), Nullable(FixedString(32))), - col_map_uuid_n_uuid Map(UUID, Nullable(UUID)), - col_map_date_n_date Map(Date, Nullable(Date)), - col_map_datetime_n_datetime Map(DateTime('UTC'), Nullable(DateTime('UTC'))), - col_map_date32_n_date32 Map(Date32, Nullable(Date32)), - col_map_enum_n_enum Map(Enum('a' = -128, 'b' = 127), Nullable(Enum('a' = -128, 'b' = 127))) - */ - -) -ENGINE = MergeTree ORDER BY (id); - -INSERT INTO canon.table VALUES (1, /* All data types */ -128, -32768, -2147483648, -9223372036854775808, /* -170141183460469231731687303715884105728, -- Error: unhandled type Int128 */ /* -57896044618658097711785492504343953926634992332820282019728792003956564819968, -- Error: unhandled type Int256 */ 0, 0, 0, 0, /* 0, -- Error: unhandled type UInt128 */ /* 0, -- Error: unhandled type UInt256 */ 1.175494351E-38, 2.2250738585072014E-308, 1.23, 1.23, 123456789012345678901234567890.23, 1234567890123456789012345678901234567890123456789012345678901234567890.23, /* false, -- Error: unhandled type Bool */ 'clickhouse', 'clickhouse', 'clickhouse', '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '1970-01-01', /* '1900-01-01', -- Error: unhandled type Date32 */ '1970-01-01 00:00:00', '1900-01-01 00:00:00', '1900-01-01 00:00:00', '1900-01-01 00:00:00', 'a', 'a', /* '{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}', -- Error: unhandled type 'a Int8', unhandled type 'a String' */ /* Nullable */ null, null, null, null, /* null, -- Error: unhandled type Int256 */ /* null, -- Error: unhandled type Int256 */ null, null, null, null, /* null, -- Error: unhandled type UInt256 */ /* null, -- Error: unhandled type UInt256 */ null, null, null, null, /* null, -- Error: reflect: call of reflect.Value.Interface on zero Value */ /* null, -- Error: Nullable(T): precision of Decimal exceeds max bound */ /* null, -- Error: unhandled type Bool */ null, null, null, null, null, /* null, -- Error: unhandled type Date32 */ null, null, null, null, null, null, /* Arrays */ [null, 1], [null, 1], [null, 1], [null, 1], /* [null, 1], -- Error: unhandled type Int128 */ /* [null, 1], -- Error: unhandled type Int256 */ [null, 1], [null, 1], [null, 1], [null, 1], /* [null, 1], -- Error: unhandled type UInt128 */ /* [null, 1], -- Error: unhandled type UInt256 */ [null, 1], [null, 1], [null, 1], [null, 1], /* [null, 1], -- Error: unsupported Array type */ /* [null, 1], -- Error: Array(T): Nullable(T): precision of Decimal exceeds max bound */ /* [null, false], -- Error: unhandled type Bool */ [null, '1'], [null, '1'], [null, '1'], [null, '61f0c404-5cb3-11e7-907b-a6006ad3dba0'], /* Arrays with dates/datetimes reads incorrectly, with strange timezones (+3h/+2.5h) [null, '1970-01-01'], [null, '1900-01-01'], -- Error: unhandled type Date32 [null, '1970-01-01 00:00:00'], [null, '1900-01-01 00:00:00'], [null, '1900-01-01 00:00:00'], [null, '1900-01-01 00:00:00'], */ [null, 'a'], [null, 'a'] /* Maps */ /* Map types are not supported, error: unhandled type Map(,) map('1', null, '2' ,'clickhouse'), map(1, null, 2, 222), map('1', null, '2', 'clickhouse'), map('61f0c404-5cb3-11e7-907b-a6006ad3dba0', null, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '61f0c404-5cb3-11e7-907b-a6006ad3dba0'), map('1970-01-01', null, '1970-01-01', '1970-01-01'), map('1970-01-01 00:00:00', null, '1970-01-01 00:00:00', '1970-01-01 00:00:00'), map('1900-01-01', null, '1900-01-01', '1900-01-01'), map('a', null, 'a', 'b') */); -INSERT INTO canon.table VALUES (2, /* All data types */ 127, 32767, 2147483647, 9223372036854775807, /* 170141183460469231731687303715884105727, -- Error: unhandled type Int128 */ /* 57896044618658097711785492504343953926634992332820282019728792003956564819967, -- Error: unhandled type Int256 */ 255, 65535, 4294967295, 18446744073709551615, /* 340282366920938463463374607431768211455, -- Error: unhandled type UInt128 */ /* 115792089237316195423570985008687907853269984665640564039457584007913129639935, -- Error: unhandled type UInt256 */ 3.02823466E+38, 1.7976931348623158E+308, 3.21, 3.21, 3.21, 3.21, /* true, -- Error: unhandled type Bool */ 'clickhouse', 'clickhouse', 'clickhouse', '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '2149-06-06', /* '2299-12-31', -- Error: unhandled type Date32 */ '2106-02-07 06:28:15', '2299-12-31 23:59:59', '2299-12-31 23:59:59.999', '2299-12-31 23:59:59.999999', 'b', 'b', /* '{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}', -- Error: unhandled type 'a Int8', unhandled type 'a String' */ /* Nullable */ 1, 1, 1, 1, /* 1, -- Error: unhandled type Int128 */ /* 1, -- Error: unhandled type Int256 */ 1, 1, 1, 1, /* 1, -- Error: unhandled type UInt128 */ /* 1, -- Error: unhandled type UInt256 */ 1, 1, 1, 1, /* 1, -- Error: reflect: call of reflect.Value.Interface on zero Value */ /* 1, -- Error: Nullable(T): precision of Decimal exceeds max bound */ /* true, -- Error: unhandled type Bool */ 'clickhouse', 'clickhouse', 'clickhouse', '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '2022-08-29', /* '2022-08-29', -- Error: unhandled type Date32 */ '2022-08-29 19:04:00', '2022-08-29 19:04:00', '2022-08-29 19:04:00', '2022-08-29 19:04:00', 'a', 'a', /* Arrays */ [0, 1], [0, 1], [0, 1], [0, 1], /* [0, 1], -- Error: unhandled type Int128 */ /* [0, 1], -- Error: unhandled type Int256 */ [0, 1], [0, 1], [0, 1], [0, 1], /* [0, 1], -- Error: unhandled type UInt128 */ /* [0, 1], -- Error: unhandled type UInt256 */ [0, 1], [0, 1], [0, 1], [0, 1], /* [0, 1], -- Error: unsupported Array type */ /* [0, 1], -- Error: Array(T): Nullable(T): precision of Decimal exceeds max bound */ /* [true, false], -- Error: unhandled type Bool */ ['0', '1'], ['0', '1'], ['0', '1'], ['61f0c404-5cb3-11e7-907b-a6006ad3dba0'], /* Arrays with dates/datetimes reads incorrectly, with strange timezones (+3h/+2.5h) ['1970-01-01'], ['1900-01-01'], -- Error: unhandled type Date32 ['1970-01-01 00:00:00'], ['1900-01-01 00:00:00'], ['1900-01-01 00:00:00'], ['1900-01-01 00:00:00'], */ ['b', 'a'], ['b', 'a'] /* Maps */ /* Map types are not supported, error: unhandled type Map(,) map('2' ,'clickhouse'), map(2, 222), map('2', 'clickhouse'), map('61f0c404-5cb3-11e7-907b-a6006ad3dba0', '61f0c404-5cb3-11e7-907b-a6006ad3dba0'), map('1970-01-01', '1970-01-01'), map('1970-01-01 00:00:00', '1970-01-01 00:00:00'), map('1900-01-01', '1900-01-01'), map('a', 'b') */); -INSERT INTO canon.table VALUES (3, /* All data types */ 2, 2, 2, 2, /* 2, -- Error: unhandled type Int128 */ /* 2, -- Error: unhandled type Int256 */ 2, 2, 2, 2, /* 2, -- Error: unhandled type UInt128 */ /* 2, -- Error: unhandled type UInt256 */ 2.2, 2.2, 2.10, 2.10, 2.10, 2.10, /* true, -- Error: unhandled type Bool */ 'clickhouse', 'clickhouse', 'clickhouse', '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '2022-08-29', /* '2022-08-29', -- Error: unhandled type Date32 */ '2022-08-29 19:04:00', '2022-08-29 19:04:00', '2022-08-29 19:04:00', '2022-08-29 19:04:00', 'b', 'b', /* '{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}', -- Error: unhandled type 'a Int8', unhandled type 'a String' */ /* Nullable */ 1, 1, 1, 1, /* 1, -- Error: unhandled type Int128 */ /* 1, -- Error: unhandled type Int256 */ 1, 1, 1, 1, /* 1, -- Error: unhandled type UInt128 */ /* 1, -- Error: unhandled type UInt256 */ 1, 1, 1, 1, /* 1, -- Error: reflect: call of reflect.Value.Interface on zero Value */ /* 1, -- Error: Nullable(T): precision of Decimal exceeds max bound */ /* true, -- Error: unhandled type Bool */ 'clickhouse', 'clickhouse', 'clickhouse', '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '2022-08-29', /* '2022-08-29', -- Error: unhandled type Date32 */ '2022-08-29 19:04:00', '2022-08-29 19:04:00', '2022-08-29 19:04:00', '2022-08-29 19:04:00', 'a', 'a', /* Arrays */ [0, 1], [0, 1], [0, 1], [0, 1], /* [0, 1], -- Error: unhandled type Int128 */ /* [0, 1], -- Error: unhandled type Int256 */ [0, 1], [0, 1], [0, 1], [0, 1], /* [0, 1], -- Error: unhandled type UInt128 */ /* [0, 1], -- Error: unhandled type UInt256 */ [0, 1], [0, 1], [0, 1], [0, 1], /* [0, 1], -- Error: unsupported Array type */ /* [0, 1], -- Error: Array(T): Nullable(T): precision of Decimal exceeds max bound */ /* [true, false], -- Error: unhandled type Bool */ ['0', '1'], ['0', '1'], ['0', '1'], ['61f0c404-5cb3-11e7-907b-a6006ad3dba0'], /* Arrays with dates/datetimes reads incorrectly, with strange timezones (+3h/+2.5h) ['2022-08-29'], ['2022-08-29'], -- Error: unhandled type Date32 ['2022-08-29 19:04:00'], ['2022-08-29 19:04:00'], ['2022-08-29 19:04:00'], ['2022-08-29 19:04:00'], */ ['b', 'a'], ['b', 'a'] /* Maps */ /* Map types are not supported, error: unhandled type Map(,) map('2' ,'clickhouse'), map(2, 222), map('2', 'clickhouse'), map('61f0c404-5cb3-11e7-907b-a6006ad3dba0', '61f0c404-5cb3-11e7-907b-a6006ad3dba0'), map('2022-08-29', '2022-08-29'), map('2022-08-29 19:04:00', '2022-08-29 19:04:00'), map('2022-08-29', '2022-08-29'), map('a', 'b') */); diff --git a/tests/canon/gotest/canondata/result.json b/tests/canon/gotest/canondata/result.json deleted file mode 100644 index 089b47af..00000000 --- a/tests/canon/gotest/canondata/result.json +++ /dev/null @@ -1,15055 +0,0 @@ -{ - "gotest.gotest.TestAll/ch.TestCanonSource_canon_0#01/canon.table": { - "Rows": [ - { - "Data": { - "col_Date": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "col_DateTime": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "col_DateTime64_1": { - "GoType": "time.Time", - "Val": "1900-01-01T00:00:00Z" - }, - "col_DateTime64_2": { - "GoType": "time.Time", - "Val": "1900-01-01T00:00:00Z" - }, - "col_DateTime64_3": { - "GoType": "time.Time", - "Val": "1900-01-01T00:00:00Z" - }, - "col_Decimal128": { - "GoType": "string", - "Val": "123456789012345678901234567890.23" - }, - "col_Decimal256": { - "GoType": "string", - "Val": "1234567890123456789012345678901234567890123456789012345678901234567890.23" - }, - "col_Decimal32": { - "GoType": "string", - "Val": "1.23" - }, - "col_Decimal64": { - "GoType": "string", - "Val": "1.23" - }, - "col_FixedString_1": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_FixedString_2": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_Float32": { - "GoType": "float32", - "Val": 0 - }, - "col_Float64": { - "GoType": "float64", - "Val": 2.2250738585072014e-308 - }, - "col_Int16": { - "GoType": "int16", - "Val": -32768 - }, - "col_Int32": { - "GoType": "int32", - "Val": -2147483648 - }, - "col_Int64": { - "GoType": "int64", - "Val": -9223372036854775808 - }, - "col_Int8": { - "GoType": "int8", - "Val": -128 - }, - "col_String": { - "GoType": "string", - "Val": "clickhouse" - }, - "col_UInt16": { - "GoType": "uint16", - "Val": 0 - }, - "col_UInt32": { - "GoType": "uint32", - "Val": 0 - }, - "col_UInt64": { - "GoType": "uint64", - "Val": 0 - }, - "col_UInt8": { - "GoType": "uint8", - "Val": 0 - }, - "col_UUID": { - "GoType": "string", - "Val": "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - }, - "col_arr_Decimal32": { - "GoType": "[]*string", - "Val": [ - null, - "1" - ] - }, - "col_arr_Decimal64": { - "GoType": "[]*string", - "Val": [ - null, - "1" - ] - }, - "col_arr_FixedString_1": { - "GoType": "[]*string", - "Val": [ - null, - "1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - ] - }, - "col_arr_FixedString_2": { - "GoType": "[]*string", - "Val": [ - null, - "1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - ] - }, - "col_arr_Float32": { - "GoType": "[]*float32", - "Val": [ - null, - 1 - ] - }, - "col_arr_Float64": { - "GoType": "[]*float64", - "Val": [ - null, - 1 - ] - }, - "col_arr_Int16": { - "GoType": "[]*int16", - "Val": [ - null, - 1 - ] - }, - "col_arr_Int32": { - "GoType": "[]*int32", - "Val": [ - null, - 1 - ] - }, - "col_arr_Int64": { - "GoType": "[]*int64", - "Val": [ - null, - 1 - ] - }, - "col_arr_Int8": { - "GoType": "[]*int8", - "Val": [ - null, - 1 - ] - }, - "col_arr_String": { - "GoType": "[]*string", - "Val": [ - null, - "1" - ] - }, - "col_arr_UInt16": { - "GoType": "[]*uint16", - "Val": [ - null, - 1 - ] - }, - "col_arr_UInt32": { - "GoType": "[]*uint32", - "Val": [ - null, - 1 - ] - }, - "col_arr_UInt64": { - "GoType": "[]*uint64", - "Val": [ - null, - 1 - ] - }, - "col_arr_UInt8": { - "GoType": "[]*uint8", - "Val": [ - null, - 1 - ] - }, - "col_arr_UUID": { - "GoType": "[]*string", - "Val": [ - null, - "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - ] - }, - "col_arr_enum_1": { - "GoType": "[]*string", - "Val": [ - null, - "a" - ] - }, - "col_arr_enum_2": { - "GoType": "[]*string", - "Val": [ - null, - "a" - ] - }, - "col_enum_1": { - "GoType": "string", - "Val": "a" - }, - "col_enum_2": { - "GoType": "string", - "Val": "a" - }, - "col_n_Date": { - "GoType": "", - "Val": null - }, - "col_n_DateTime": { - "GoType": "", - "Val": null - }, - "col_n_DateTime64_1": { - "GoType": "", - "Val": null - }, - "col_n_DateTime64_2": { - "GoType": "", - "Val": null - }, - "col_n_DateTime64_3": { - "GoType": "", - "Val": null - }, - "col_n_Decimal32": { - "GoType": "", - "Val": null - }, - "col_n_Decimal64": { - "GoType": "", - "Val": null - }, - "col_n_FixedString_1": { - "GoType": "", - "Val": null - }, - "col_n_FixedString_2": { - "GoType": "", - "Val": null - }, - "col_n_Float32": { - "GoType": "", - "Val": null - }, - "col_n_Float64": { - "GoType": "", - "Val": null - }, - "col_n_Int16": { - "GoType": "", - "Val": null - }, - "col_n_Int32": { - "GoType": "", - "Val": null - }, - "col_n_Int64": { - "GoType": "", - "Val": null - }, - "col_n_Int8": { - "GoType": "", - "Val": null - }, - "col_n_String": { - "GoType": "", - "Val": null - }, - "col_n_UInt16": { - "GoType": "", - "Val": null - }, - "col_n_UInt32": { - "GoType": "", - "Val": null - }, - "col_n_UInt64": { - "GoType": "", - "Val": null - }, - "col_n_UInt8": { - "GoType": "", - "Val": null - }, - "col_n_UUID": { - "GoType": "", - "Val": null - }, - "col_n_enum_1": { - "GoType": "", - "Val": null - }, - "col_n_enum_2": { - "GoType": "", - "Val": null - }, - "id": { - "GoType": "uint64", - "Val": 1 - } - } - }, - { - "Data": { - "col_Date": { - "GoType": "time.Time", - "Val": "2149-06-06T00:00:00Z" - }, - "col_DateTime": { - "GoType": "time.Time", - "Val": "2106-02-07T06:28:15Z" - }, - "col_DateTime64_1": { - "GoType": "time.Time", - "Val": "1715-06-13T00:25:25.290448384Z" - }, - "col_DateTime64_2": { - "GoType": "time.Time", - "Val": "1715-06-13T00:25:26.289448384Z" - }, - "col_DateTime64_3": { - "GoType": "time.Time", - "Val": "1715-06-13T00:25:26.290447384Z" - }, - "col_Decimal128": { - "GoType": "string", - "Val": "3.21" - }, - "col_Decimal256": { - "GoType": "string", - "Val": "3.21" - }, - "col_Decimal32": { - "GoType": "string", - "Val": "3.21" - }, - "col_Decimal64": { - "GoType": "string", - "Val": "3.21" - }, - "col_FixedString_1": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_FixedString_2": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_Float32": { - "GoType": "float32", - "Val": 0 - }, - "col_Float64": { - "GoType": "float64", - "Val": 1.7976931348623157e+308 - }, - "col_Int16": { - "GoType": "int16", - "Val": 32767 - }, - "col_Int32": { - "GoType": "int32", - "Val": 2147483647 - }, - "col_Int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "col_Int8": { - "GoType": "int8", - "Val": 127 - }, - "col_String": { - "GoType": "string", - "Val": "clickhouse" - }, - "col_UInt16": { - "GoType": "uint16", - "Val": 65535 - }, - "col_UInt32": { - "GoType": "uint32", - "Val": 4294967295 - }, - "col_UInt64": { - "GoType": "uint64", - "Val": 18446744073709551615 - }, - "col_UInt8": { - "GoType": "uint8", - "Val": 255 - }, - "col_UUID": { - "GoType": "string", - "Val": "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - }, - "col_arr_Decimal32": { - "GoType": "[]*string", - "Val": [ - "0", - "1" - ] - }, - "col_arr_Decimal64": { - "GoType": "[]*string", - "Val": [ - "0", - "1" - ] - }, - "col_arr_FixedString_1": { - "GoType": "[]*string", - "Val": [ - "0\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - ] - }, - "col_arr_FixedString_2": { - "GoType": "[]*string", - "Val": [ - "0\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - ] - }, - "col_arr_Float32": { - "GoType": "[]*float32", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Float64": { - "GoType": "[]*float64", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int16": { - "GoType": "[]*int16", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int32": { - "GoType": "[]*int32", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int64": { - "GoType": "[]*int64", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int8": { - "GoType": "[]*int8", - "Val": [ - 0, - 1 - ] - }, - "col_arr_String": { - "GoType": "[]*string", - "Val": [ - "0", - "1" - ] - }, - "col_arr_UInt16": { - "GoType": "[]*uint16", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UInt32": { - "GoType": "[]*uint32", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UInt64": { - "GoType": "[]*uint64", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UInt8": { - "GoType": "[]*uint8", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UUID": { - "GoType": "[]*string", - "Val": [ - "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - ] - }, - "col_arr_enum_1": { - "GoType": "[]*string", - "Val": [ - "b", - "a" - ] - }, - "col_arr_enum_2": { - "GoType": "[]*string", - "Val": [ - "b", - "a" - ] - }, - "col_enum_1": { - "GoType": "string", - "Val": "b" - }, - "col_enum_2": { - "GoType": "string", - "Val": "b" - }, - "col_n_Date": { - "GoType": "time.Time", - "Val": "2022-08-29T00:00:00Z" - }, - "col_n_DateTime": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_DateTime64_1": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_DateTime64_2": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_DateTime64_3": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_Decimal32": { - "GoType": "string", - "Val": "1" - }, - "col_n_Decimal64": { - "GoType": "string", - "Val": "1" - }, - "col_n_FixedString_1": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_n_FixedString_2": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_n_Float32": { - "GoType": "float32", - "Val": 1 - }, - "col_n_Float64": { - "GoType": "float64", - "Val": 1 - }, - "col_n_Int16": { - "GoType": "int16", - "Val": 1 - }, - "col_n_Int32": { - "GoType": "int32", - "Val": 1 - }, - "col_n_Int64": { - "GoType": "int64", - "Val": 1 - }, - "col_n_Int8": { - "GoType": "int8", - "Val": 1 - }, - "col_n_String": { - "GoType": "string", - "Val": "clickhouse" - }, - "col_n_UInt16": { - "GoType": "uint16", - "Val": 1 - }, - "col_n_UInt32": { - "GoType": "uint32", - "Val": 1 - }, - "col_n_UInt64": { - "GoType": "uint64", - "Val": 1 - }, - "col_n_UInt8": { - "GoType": "uint8", - "Val": 1 - }, - "col_n_UUID": { - "GoType": "string", - "Val": "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - }, - "col_n_enum_1": { - "GoType": "string", - "Val": "a" - }, - "col_n_enum_2": { - "GoType": "string", - "Val": "a" - }, - "id": { - "GoType": "uint64", - "Val": 2 - } - } - }, - { - "Data": { - "col_Date": { - "GoType": "time.Time", - "Val": "2022-08-29T00:00:00Z" - }, - "col_DateTime": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_DateTime64_1": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_DateTime64_2": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_DateTime64_3": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_Decimal128": { - "GoType": "string", - "Val": "2.1" - }, - "col_Decimal256": { - "GoType": "string", - "Val": "2.1" - }, - "col_Decimal32": { - "GoType": "string", - "Val": "2.1" - }, - "col_Decimal64": { - "GoType": "string", - "Val": "2.1" - }, - "col_FixedString_1": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_FixedString_2": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_Float32": { - "GoType": "float32", - "Val": 0 - }, - "col_Float64": { - "GoType": "float64", - "Val": 2.2 - }, - "col_Int16": { - "GoType": "int16", - "Val": 2 - }, - "col_Int32": { - "GoType": "int32", - "Val": 2 - }, - "col_Int64": { - "GoType": "int64", - "Val": 2 - }, - "col_Int8": { - "GoType": "int8", - "Val": 2 - }, - "col_String": { - "GoType": "string", - "Val": "clickhouse" - }, - "col_UInt16": { - "GoType": "uint16", - "Val": 2 - }, - "col_UInt32": { - "GoType": "uint32", - "Val": 2 - }, - "col_UInt64": { - "GoType": "uint64", - "Val": 2 - }, - "col_UInt8": { - "GoType": "uint8", - "Val": 2 - }, - "col_UUID": { - "GoType": "string", - "Val": "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - }, - "col_arr_Decimal32": { - "GoType": "[]*string", - "Val": [ - "0", - "1" - ] - }, - "col_arr_Decimal64": { - "GoType": "[]*string", - "Val": [ - "0", - "1" - ] - }, - "col_arr_FixedString_1": { - "GoType": "[]*string", - "Val": [ - "0\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - ] - }, - "col_arr_FixedString_2": { - "GoType": "[]*string", - "Val": [ - "0\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", - "1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - ] - }, - "col_arr_Float32": { - "GoType": "[]*float32", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Float64": { - "GoType": "[]*float64", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int16": { - "GoType": "[]*int16", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int32": { - "GoType": "[]*int32", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int64": { - "GoType": "[]*int64", - "Val": [ - 0, - 1 - ] - }, - "col_arr_Int8": { - "GoType": "[]*int8", - "Val": [ - 0, - 1 - ] - }, - "col_arr_String": { - "GoType": "[]*string", - "Val": [ - "0", - "1" - ] - }, - "col_arr_UInt16": { - "GoType": "[]*uint16", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UInt32": { - "GoType": "[]*uint32", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UInt64": { - "GoType": "[]*uint64", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UInt8": { - "GoType": "[]*uint8", - "Val": [ - 0, - 1 - ] - }, - "col_arr_UUID": { - "GoType": "[]*string", - "Val": [ - "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - ] - }, - "col_arr_enum_1": { - "GoType": "[]*string", - "Val": [ - "b", - "a" - ] - }, - "col_arr_enum_2": { - "GoType": "[]*string", - "Val": [ - "b", - "a" - ] - }, - "col_enum_1": { - "GoType": "string", - "Val": "b" - }, - "col_enum_2": { - "GoType": "string", - "Val": "b" - }, - "col_n_Date": { - "GoType": "time.Time", - "Val": "2022-08-29T00:00:00Z" - }, - "col_n_DateTime": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_DateTime64_1": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_DateTime64_2": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_DateTime64_3": { - "GoType": "time.Time", - "Val": "2022-08-29T19:04:00Z" - }, - "col_n_Decimal32": { - "GoType": "string", - "Val": "1" - }, - "col_n_Decimal64": { - "GoType": "string", - "Val": "1" - }, - "col_n_FixedString_1": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_n_FixedString_2": { - "GoType": "string", - "Val": "clickhouse\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" - }, - "col_n_Float32": { - "GoType": "float32", - "Val": 1 - }, - "col_n_Float64": { - "GoType": "float64", - "Val": 1 - }, - "col_n_Int16": { - "GoType": "int16", - "Val": 1 - }, - "col_n_Int32": { - "GoType": "int32", - "Val": 1 - }, - "col_n_Int64": { - "GoType": "int64", - "Val": 1 - }, - "col_n_Int8": { - "GoType": "int8", - "Val": 1 - }, - "col_n_String": { - "GoType": "string", - "Val": "clickhouse" - }, - "col_n_UInt16": { - "GoType": "uint16", - "Val": 1 - }, - "col_n_UInt32": { - "GoType": "uint32", - "Val": 1 - }, - "col_n_UInt64": { - "GoType": "uint64", - "Val": 1 - }, - "col_n_UInt8": { - "GoType": "uint8", - "Val": 1 - }, - "col_n_UUID": { - "GoType": "string", - "Val": "61f0c404-5cb3-11e7-907b-a6006ad3dba0" - }, - "col_n_enum_1": { - "GoType": "string", - "Val": "a" - }, - "col_n_enum_2": { - "GoType": "string", - "Val": "a" - }, - "id": { - "GoType": "uint64", - "Val": 3 - } - } - } - ], - "TableID": { - "Name": "table", - "Namespace": "canon" - }, - "TableSchema": [ - { - "expression": ":", - "fake_key": false, - "key": true, - "name": "id", - "original_type": "ch:UInt64", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "uint64" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Date", - "original_type": "ch:Date", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "date" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_DateTime", - "original_type": "ch:DateTime('UTC')", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "datetime" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_DateTime64_1", - "original_type": "ch:DateTime64(0, 'UTC')", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "timestamp" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_DateTime64_2", - "original_type": "ch:DateTime64(3, 'UTC')", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "timestamp" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_DateTime64_3", - "original_type": "ch:DateTime64(6, 'UTC')", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "timestamp" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Decimal128", - "original_type": "ch:Decimal(38, 4)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Decimal256", - "original_type": "ch:Decimal(76, 5)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Decimal32", - "original_type": "ch:Decimal(9, 2)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Decimal64", - "original_type": "ch:Decimal(18, 3)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_FixedString_1", - "original_type": "ch:FixedString(32)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "string" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_FixedString_2", - "original_type": "ch:FixedString(64)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "string" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Float32", - "original_type": "ch:Float32", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Float64", - "original_type": "ch:Float64", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "double" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Int16", - "original_type": "ch:Int16", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "int16" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Int32", - "original_type": "ch:Int32", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "int32" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Int64", - "original_type": "ch:Int64", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "int64" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_Int8", - "original_type": "ch:Int8", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "int8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_String", - "original_type": "ch:String", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "string" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_UInt16", - "original_type": "ch:UInt16", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "uint16" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_UInt32", - "original_type": "ch:UInt32", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "uint32" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_UInt64", - "original_type": "ch:UInt64", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "uint64" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_UInt8", - "original_type": "ch:UInt8", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "uint8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_UUID", - "original_type": "ch:UUID", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Decimal32", - "original_type": "ch:Array(Nullable(Decimal(9, 2)))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Decimal64", - "original_type": "ch:Array(Nullable(Decimal(18, 3)))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_FixedString_1", - "original_type": "ch:Array(Nullable(FixedString(32)))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_FixedString_2", - "original_type": "ch:Array(Nullable(FixedString(64)))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Float32", - "original_type": "ch:Array(Nullable(Float32))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Float64", - "original_type": "ch:Array(Nullable(Float64))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Int16", - "original_type": "ch:Array(Nullable(Int16))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Int32", - "original_type": "ch:Array(Nullable(Int32))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Int64", - "original_type": "ch:Array(Nullable(Int64))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_Int8", - "original_type": "ch:Array(Nullable(Int8))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_String", - "original_type": "ch:Array(Nullable(String))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_UInt16", - "original_type": "ch:Array(Nullable(UInt16))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_UInt32", - "original_type": "ch:Array(Nullable(UInt32))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_UInt64", - "original_type": "ch:Array(Nullable(UInt64))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_UInt8", - "original_type": "ch:Array(Nullable(UInt8))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_UUID", - "original_type": "ch:Array(Nullable(UUID))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_enum_1", - "original_type": "ch:Array(Nullable(Enum8('a' = -128, 'b' = 127)))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_arr_enum_2", - "original_type": "ch:Array(Nullable(Enum16('a' = -32768, 'b' = 32767)))", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_enum_1", - "original_type": "ch:Enum8('a' = -128, 'b' = 127)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "utf8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_enum_2", - "original_type": "ch:Enum16('a' = -32768, 'b' = 32767)", - "path": "", - "required": true, - "table_name": "table", - "table_schema": "canon", - "type": "utf8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Date", - "original_type": "ch:Nullable(Date)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "date" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_DateTime", - "original_type": "ch:Nullable(DateTime('UTC'))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "datetime" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_DateTime64_1", - "original_type": "ch:Nullable(DateTime64(0, 'UTC'))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "timestamp" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_DateTime64_2", - "original_type": "ch:Nullable(DateTime64(3, 'UTC'))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "timestamp" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_DateTime64_3", - "original_type": "ch:Nullable(DateTime64(6, 'UTC'))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "timestamp" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Decimal32", - "original_type": "ch:Nullable(Decimal(9, 2))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Decimal64", - "original_type": "ch:Nullable(Decimal(18, 3))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_FixedString_1", - "original_type": "ch:Nullable(FixedString(32))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "string" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_FixedString_2", - "original_type": "ch:Nullable(FixedString(64))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "string" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Float32", - "original_type": "ch:Nullable(Float32)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Float64", - "original_type": "ch:Nullable(Float64)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "double" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Int16", - "original_type": "ch:Nullable(Int16)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "int16" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Int32", - "original_type": "ch:Nullable(Int32)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "int32" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Int64", - "original_type": "ch:Nullable(Int64)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "int64" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_Int8", - "original_type": "ch:Nullable(Int8)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "int8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_String", - "original_type": "ch:Nullable(String)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "string" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_UInt16", - "original_type": "ch:Nullable(UInt16)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "uint16" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_UInt32", - "original_type": "ch:Nullable(UInt32)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "uint32" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_UInt64", - "original_type": "ch:Nullable(UInt64)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "uint64" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_UInt8", - "original_type": "ch:Nullable(UInt8)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "uint8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_UUID", - "original_type": "ch:Nullable(UUID)", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "any" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_enum_1", - "original_type": "ch:Nullable(Enum8('a' = -128, 'b' = 127))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "utf8" - }, - { - "expression": ":", - "fake_key": false, - "key": false, - "name": "col_n_enum_2", - "original_type": "ch:Nullable(Enum16('a' = -32768, 'b' = 32767))", - "path": "", - "required": false, - "table_name": "table", - "table_schema": "canon", - "type": "utf8" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_initial_canon_0#01/source.initial": { - "Rows": [ - { - "Data": { - "DECIMAL_": { - "GoType": "json.Number", - "Val": 2345678901 - }, - "DECIMAL_5": { - "GoType": "json.Number", - "Val": 23451 - }, - "DECIMAL_5_2": { - "GoType": "json.Number", - "Val": 231.45 - }, - "NUMERIC_": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "NUMERIC_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "NUMERIC_5_2": { - "GoType": "json.Number", - "Val": 123.45 - }, - "bigint5": { - "GoType": "int64", - "Val": 88 - }, - "bigint_": { - "GoType": "int64", - "Val": 8 - }, - "bigint_u": { - "GoType": "uint64", - "Val": 888 - }, - "binary5": { - "GoType": "[]uint8", - "Val": "nwAAAAA=" - }, - "binary_": { - "GoType": "[]uint8", - "Val": "nw==" - }, - "bit": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "bit16": { - "GoType": "[]uint8", - "Val": "AJ8=" - }, - "blob_": { - "GoType": "[]uint8", - "Val": "/w==" - }, - "bool1": { - "GoType": "int8", - "Val": 0 - }, - "bool2": { - "GoType": "int8", - "Val": 1 - }, - "char5": { - "GoType": "string", - "Val": "abc" - }, - "char_": { - "GoType": "string", - "Val": "a" - }, - "date_": { - "GoType": "time.Time", - "Val": "1000-01-01T00:00:00Z" - }, - "datetime0": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10Z" - }, - "datetime1": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.1Z" - }, - "datetime2": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.12Z" - }, - "datetime3": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.123Z" - }, - "datetime4": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.1234Z" - }, - "datetime5": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.12345Z" - }, - "datetime6": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.123456Z" - }, - "datetime_": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10Z" - }, - "double_": { - "GoType": "json.Number", - "Val": 2.34 - }, - "double_precision": { - "GoType": "json.Number", - "Val": 2.34 - }, - "enum_": { - "GoType": "string", - "Val": "x-small" - }, - "float_": { - "GoType": "json.Number", - "Val": 1.23 - }, - "float_53": { - "GoType": "json.Number", - "Val": 1.23 - }, - "int_": { - "GoType": "int32", - "Val": 9 - }, - "int_u": { - "GoType": "uint32", - "Val": 9999 - }, - "integer5": { - "GoType": "int32", - "Val": 999 - }, - "integer_": { - "GoType": "int32", - "Val": 99 - }, - "json_": { - "GoType": "map[string]interface {}", - "Val": { - "k1": "v1" - } - }, - "longblob_": { - "GoType": "[]uint8", - "Val": "q80=" - }, - "longtext_": { - "GoType": "string", - "Val": "my-longtext" - }, - "mediumblob_": { - "GoType": "[]uint8", - "Val": "q80=" - }, - "mediumint5": { - "GoType": "int32", - "Val": 11 - }, - "mediumint_": { - "GoType": "int32", - "Val": 1 - }, - "mediumint_u": { - "GoType": "uint32", - "Val": 111 - }, - "mediumtext_": { - "GoType": "string", - "Val": "my-mediumtext" - }, - "pk": { - "GoType": "uint32", - "Val": 1 - }, - "real_": { - "GoType": "json.Number", - "Val": 123.45 - }, - "real_10_2": { - "GoType": "json.Number", - "Val": 99999.99 - }, - "set_": { - "GoType": "string", - "Val": "a" - }, - "smallint5": { - "GoType": "int16", - "Val": 100 - }, - "smallint_": { - "GoType": "int16", - "Val": 1000 - }, - "smallint_u": { - "GoType": "uint16", - "Val": 10 - }, - "text_": { - "GoType": "string", - "Val": "my-text" - }, - "time0": { - "GoType": "string", - "Val": "04:05:06" - }, - "time1": { - "GoType": "string", - "Val": "04:05:06.1" - }, - "time2": { - "GoType": "string", - "Val": "04:05:06.12" - }, - "time3": { - "GoType": "string", - "Val": "04:05:06.123" - }, - "time4": { - "GoType": "string", - "Val": "04:05:06.1234" - }, - "time5": { - "GoType": "string", - "Val": "04:05:06.12345" - }, - "time6": { - "GoType": "string", - "Val": "04:05:06.123456" - }, - "time_": { - "GoType": "string", - "Val": "04:05:06" - }, - "timestamp0": { - "GoType": "time.Time", - "Val": "1999-10-19T10:23:54Z" - }, - "timestamp1": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.1Z" - }, - "timestamp2": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.12Z" - }, - "timestamp3": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.123Z" - }, - "timestamp4": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.1234Z" - }, - "timestamp5": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.12345Z" - }, - "timestamp6": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.123456Z" - }, - "timestamp_": { - "GoType": "time.Time", - "Val": "1999-01-01T00:00:01Z" - }, - "tinyblob_": { - "GoType": "[]uint8", - "Val": "n5+f" - }, - "tinyint1": { - "GoType": "int8", - "Val": 1 - }, - "tinyint1u": { - "GoType": "uint8", - "Val": 1 - }, - "tinyint_": { - "GoType": "int8", - "Val": 1 - }, - "tinyint_def": { - "GoType": "int8", - "Val": 22 - }, - "tinyint_u": { - "GoType": "uint8", - "Val": 255 - }, - "tinytext_": { - "GoType": "string", - "Val": "qwerty12345" - }, - "varbinary5": { - "GoType": "[]uint8", - "Val": "n58=" - }, - "varchar5": { - "GoType": "string", - "Val": "blab" - }, - "year4": { - "GoType": "string", - "Val": "2155" - }, - "year_": { - "GoType": "string", - "Val": "1901" - } - } - } - ], - "TableID": { - "Name": "initial", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "pk", - "original_type": "mysql:int(10) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bigint5", - "original_type": "mysql:bigint(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bigint_", - "original_type": "mysql:bigint(20)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bigint_u", - "original_type": "mysql:bigint(20) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "binary5", - "original_type": "mysql:binary(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "binary_", - "original_type": "mysql:binary(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bit", - "original_type": "mysql:bit(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bit16", - "original_type": "mysql:bit(16)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "blob_", - "original_type": "mysql:blob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bool1", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bool2", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "char5", - "original_type": "mysql:char(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "char_", - "original_type": "mysql:char(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime0", - "original_type": "mysql:datetime", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime1", - "original_type": "mysql:datetime(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime2", - "original_type": "mysql:datetime(2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime3", - "original_type": "mysql:datetime(3)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime4", - "original_type": "mysql:datetime(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime5", - "original_type": "mysql:datetime(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime6", - "original_type": "mysql:datetime(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime_", - "original_type": "mysql:datetime", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "date_", - "original_type": "mysql:date", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DECIMAL_", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DECIMAL_5", - "original_type": "mysql:decimal(5,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DECIMAL_5_2", - "original_type": "mysql:decimal(5,2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "double_", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "double_precision", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "enum_", - "original_type": "mysql:enum('x-small','small','medium','large','x-large')", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "float_", - "original_type": "mysql:float", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "float_53", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "integer5", - "original_type": "mysql:int(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "integer_", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int_", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int_u", - "original_type": "mysql:int(10) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "json_", - "original_type": "mysql:json", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "longblob_", - "original_type": "mysql:longblob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "longtext_", - "original_type": "mysql:longtext", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumblob_", - "original_type": "mysql:mediumblob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumint5", - "original_type": "mysql:mediumint(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumint_", - "original_type": "mysql:mediumint(9)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumint_u", - "original_type": "mysql:mediumint(8) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumtext_", - "original_type": "mysql:mediumtext", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "NUMERIC_", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "NUMERIC_5", - "original_type": "mysql:decimal(5,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "NUMERIC_5_2", - "original_type": "mysql:decimal(5,2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "real_", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "real_10_2", - "original_type": "mysql:double(10,2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "set_", - "original_type": "mysql:set('a','b','c','d')", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "smallint5", - "original_type": "mysql:smallint(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "smallint_", - "original_type": "mysql:smallint(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "smallint_u", - "original_type": "mysql:smallint(5) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "text_", - "original_type": "mysql:text", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time0", - "original_type": "mysql:time", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time1", - "original_type": "mysql:time(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time2", - "original_type": "mysql:time(2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time3", - "original_type": "mysql:time(3)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time4", - "original_type": "mysql:time(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time5", - "original_type": "mysql:time(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time6", - "original_type": "mysql:time(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp0", - "original_type": "mysql:timestamp", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp1", - "original_type": "mysql:timestamp(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp2", - "original_type": "mysql:timestamp(2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp3", - "original_type": "mysql:timestamp(3)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp4", - "original_type": "mysql:timestamp(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp5", - "original_type": "mysql:timestamp(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp6", - "original_type": "mysql:timestamp(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp_", - "original_type": "mysql:timestamp", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time_", - "original_type": "mysql:time", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyblob_", - "original_type": "mysql:tinyblob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint1", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint1u", - "original_type": "mysql:tinyint(1) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint_", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint_def", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint_u", - "original_type": "mysql:tinyint(3) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinytext_", - "original_type": "mysql:tinytext", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "varbinary5", - "original_type": "mysql:varbinary(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "varchar5", - "original_type": "mysql:varchar(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "year4", - "original_type": "mysql:year(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "year_", - "original_type": "mysql:year(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_initial_canon_0#03/source.initial": { - "Rows": [ - { - "Data": { - "DECIMAL_": { - "GoType": "json.Number", - "Val": 2345678901 - }, - "DECIMAL_5": { - "GoType": "json.Number", - "Val": 23451 - }, - "DECIMAL_5_2": { - "GoType": "json.Number", - "Val": 231.45 - }, - "NUMERIC_": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "NUMERIC_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "NUMERIC_5_2": { - "GoType": "json.Number", - "Val": 123.45 - }, - "bigint5": { - "GoType": "int64", - "Val": 88 - }, - "bigint_": { - "GoType": "int64", - "Val": 8 - }, - "bigint_u": { - "GoType": "uint64", - "Val": 888 - }, - "binary5": { - "GoType": "[]uint8", - "Val": "nw==" - }, - "binary_": { - "GoType": "[]uint8", - "Val": "nw==" - }, - "bit": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "bit16": { - "GoType": "[]uint8", - "Val": "AJ8=" - }, - "blob_": { - "GoType": "[]uint8", - "Val": "/w==" - }, - "bool1": { - "GoType": "int8", - "Val": 0 - }, - "bool2": { - "GoType": "int8", - "Val": 1 - }, - "char5": { - "GoType": "string", - "Val": "abc" - }, - "char_": { - "GoType": "string", - "Val": "a" - }, - "date_": { - "GoType": "time.Time", - "Val": "1000-01-01T00:00:00Z" - }, - "datetime0": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10Z" - }, - "datetime1": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.1Z" - }, - "datetime2": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.12Z" - }, - "datetime3": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.123Z" - }, - "datetime4": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.1234Z" - }, - "datetime5": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.12345Z" - }, - "datetime6": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10.123456Z" - }, - "datetime_": { - "GoType": "time.Time", - "Val": "2020-01-01T15:10:10Z" - }, - "double_": { - "GoType": "json.Number", - "Val": 2.34 - }, - "double_precision": { - "GoType": "json.Number", - "Val": 2.34 - }, - "enum_": { - "GoType": "string", - "Val": "x-small" - }, - "float_": { - "GoType": "json.Number", - "Val": 1.23 - }, - "float_53": { - "GoType": "json.Number", - "Val": 1.23 - }, - "int_": { - "GoType": "int32", - "Val": 9 - }, - "int_u": { - "GoType": "uint32", - "Val": 9999 - }, - "integer5": { - "GoType": "int32", - "Val": 999 - }, - "integer_": { - "GoType": "int32", - "Val": 99 - }, - "json_": { - "GoType": "map[string]interface {}", - "Val": { - "k1": "v1" - } - }, - "longblob_": { - "GoType": "[]uint8", - "Val": "q80=" - }, - "longtext_": { - "GoType": "string", - "Val": "my-longtext" - }, - "mediumblob_": { - "GoType": "[]uint8", - "Val": "q80=" - }, - "mediumint5": { - "GoType": "int32", - "Val": 11 - }, - "mediumint_": { - "GoType": "int32", - "Val": 1 - }, - "mediumint_u": { - "GoType": "uint32", - "Val": 111 - }, - "mediumtext_": { - "GoType": "string", - "Val": "my-mediumtext" - }, - "pk": { - "GoType": "uint32", - "Val": 1 - }, - "real_": { - "GoType": "json.Number", - "Val": 123.45 - }, - "real_10_2": { - "GoType": "json.Number", - "Val": 99999.99 - }, - "set_": { - "GoType": "string", - "Val": "a" - }, - "smallint5": { - "GoType": "int16", - "Val": 100 - }, - "smallint_": { - "GoType": "int16", - "Val": 1000 - }, - "smallint_u": { - "GoType": "uint16", - "Val": 10 - }, - "text_": { - "GoType": "string", - "Val": "my-text" - }, - "time0": { - "GoType": "string", - "Val": "04:05:06" - }, - "time1": { - "GoType": "string", - "Val": "04:05:06.1" - }, - "time2": { - "GoType": "string", - "Val": "04:05:06.12" - }, - "time3": { - "GoType": "string", - "Val": "04:05:06.123" - }, - "time4": { - "GoType": "string", - "Val": "04:05:06.1234" - }, - "time5": { - "GoType": "string", - "Val": "04:05:06.12345" - }, - "time6": { - "GoType": "string", - "Val": "04:05:06.123456" - }, - "time_": { - "GoType": "string", - "Val": "04:05:06" - }, - "timestamp0": { - "GoType": "time.Time", - "Val": "1999-10-19T10:23:54Z" - }, - "timestamp1": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.1Z" - }, - "timestamp2": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.12Z" - }, - "timestamp3": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.123Z" - }, - "timestamp4": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.1234Z" - }, - "timestamp5": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.12345Z" - }, - "timestamp6": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.123456Z" - }, - "timestamp_": { - "GoType": "time.Time", - "Val": "1999-01-01T00:00:01Z" - }, - "tinyblob_": { - "GoType": "[]uint8", - "Val": "n5+f" - }, - "tinyint1": { - "GoType": "int8", - "Val": 1 - }, - "tinyint1u": { - "GoType": "uint8", - "Val": 1 - }, - "tinyint_": { - "GoType": "int8", - "Val": 1 - }, - "tinyint_def": { - "GoType": "int8", - "Val": 22 - }, - "tinyint_u": { - "GoType": "uint8", - "Val": 255 - }, - "tinytext_": { - "GoType": "string", - "Val": "qwerty12345" - }, - "varbinary5": { - "GoType": "[]uint8", - "Val": "n58=" - }, - "varchar5": { - "GoType": "string", - "Val": "blab" - }, - "year4": { - "GoType": "string", - "Val": "2155" - }, - "year_": { - "GoType": "string", - "Val": "1901" - } - } - } - ], - "TableID": { - "Name": "initial", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "pk", - "original_type": "mysql:int(10) unsigned", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bool1", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bool2", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bit", - "original_type": "mysql:bit(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bit16", - "original_type": "mysql:bit(16)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint_", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint_def", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint_u", - "original_type": "mysql:tinyint(3) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint1", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyint1u", - "original_type": "mysql:tinyint(1) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "smallint_", - "original_type": "mysql:smallint(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "smallint5", - "original_type": "mysql:smallint(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "smallint_u", - "original_type": "mysql:smallint(5) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumint_", - "original_type": "mysql:mediumint(9)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumint5", - "original_type": "mysql:mediumint(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumint_u", - "original_type": "mysql:mediumint(8) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int_", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "integer_", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "integer5", - "original_type": "mysql:int(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int_u", - "original_type": "mysql:int(10) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bigint_", - "original_type": "mysql:bigint(20)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bigint5", - "original_type": "mysql:bigint(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bigint_u", - "original_type": "mysql:bigint(20) unsigned", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "real_", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "real_10_2", - "original_type": "mysql:double(10,2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "float_", - "original_type": "mysql:float", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "float_53", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "double_", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "double_precision", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "char_", - "original_type": "mysql:char(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "char5", - "original_type": "mysql:char(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "varchar5", - "original_type": "mysql:varchar(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "binary_", - "original_type": "mysql:binary(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "binary5", - "original_type": "mysql:binary(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "varbinary5", - "original_type": "mysql:varbinary(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinyblob_", - "original_type": "mysql:tinyblob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "tinytext_", - "original_type": "mysql:tinytext", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "blob_", - "original_type": "mysql:blob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "text_", - "original_type": "mysql:text", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumblob_", - "original_type": "mysql:mediumblob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "mediumtext_", - "original_type": "mysql:mediumtext", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "longblob_", - "original_type": "mysql:longblob", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "longtext_", - "original_type": "mysql:longtext", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "json_", - "original_type": "mysql:json", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "enum_", - "original_type": "mysql:enum('x-small','small','medium','large','x-large')", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "set_", - "original_type": "mysql:set('a','b','c','d')", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "year_", - "original_type": "mysql:year(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "year4", - "original_type": "mysql:year(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp_", - "original_type": "mysql:timestamp", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp0", - "original_type": "mysql:timestamp", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp1", - "original_type": "mysql:timestamp(1)", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp2", - "original_type": "mysql:timestamp(2)", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp3", - "original_type": "mysql:timestamp(3)", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp4", - "original_type": "mysql:timestamp(4)", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp5", - "original_type": "mysql:timestamp(5)", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp6", - "original_type": "mysql:timestamp(6)", - "path": "", - "required": true, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "date_", - "original_type": "mysql:date", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time_", - "original_type": "mysql:time", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time0", - "original_type": "mysql:time", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time1", - "original_type": "mysql:time(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time2", - "original_type": "mysql:time(2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time3", - "original_type": "mysql:time(3)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time4", - "original_type": "mysql:time(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time5", - "original_type": "mysql:time(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time6", - "original_type": "mysql:time(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime_", - "original_type": "mysql:datetime", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime0", - "original_type": "mysql:datetime", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime1", - "original_type": "mysql:datetime(1)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime2", - "original_type": "mysql:datetime(2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime3", - "original_type": "mysql:datetime(3)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime4", - "original_type": "mysql:datetime(4)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime5", - "original_type": "mysql:datetime(5)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "datetime6", - "original_type": "mysql:datetime(6)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "NUMERIC_", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "NUMERIC_5", - "original_type": "mysql:decimal(5,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "NUMERIC_5_2", - "original_type": "mysql:decimal(5,2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DECIMAL_", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DECIMAL_5", - "original_type": "mysql:decimal(5,0)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DECIMAL_5_2", - "original_type": "mysql:decimal(5,2)", - "path": "", - "required": false, - "table_name": "initial", - "table_schema": "source", - "type": "double" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_json_types_canon_0#01/source.json_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 0 - }, - "t_json": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "t_json": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "t_json": { - "GoType": "string", - "Val": "blablabla" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 3 - }, - "t_json": { - "GoType": "json.Number", - "Val": 1 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 4 - }, - "t_json": { - "GoType": "map[string]interface {}", - "Val": {} - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 5 - }, - "t_json": { - "GoType": "[]interface {}", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_json": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "json_types", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "json_types", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_json", - "original_type": "mysql:json", - "path": "", - "required": false, - "table_name": "json_types", - "table_schema": "source", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_json_types_canon_0#03/source.json_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 0 - }, - "t_json": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "t_json": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "t_json": { - "GoType": "string", - "Val": "blablabla" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 3 - }, - "t_json": { - "GoType": "json.Number", - "Val": 1 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 4 - }, - "t_json": { - "GoType": "map[string]interface {}", - "Val": {} - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 5 - }, - "t_json": { - "GoType": "[]interface {}", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_json": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "json_types", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "json_types", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_json", - "original_type": "mysql:json", - "path": "", - "required": false, - "table_name": "json_types", - "table_schema": "source", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_bit_canon_0#01/source.numeric_types_bit": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 11000 - }, - "t_bit": { - "GoType": "[]uint8", - "Val": "AA==" - }, - "t_bit1": { - "GoType": "[]uint8", - "Val": "AA==" - }, - "t_bit16": { - "GoType": "[]uint8", - "Val": "AAA=" - }, - "t_bit64": { - "GoType": "[]uint8", - "Val": "AAAAAAAAAAA=" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 11001 - }, - "t_bit": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "t_bit1": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "t_bit16": { - "GoType": "[]uint8", - "Val": "//8=" - }, - "t_bit64": { - "GoType": "[]uint8", - "Val": "//////////8=" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_bit": { - "GoType": "", - "Val": null - }, - "t_bit1": { - "GoType": "", - "Val": null - }, - "t_bit16": { - "GoType": "", - "Val": null - }, - "t_bit64": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_bit", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit", - "original_type": "mysql:bit(1)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit1", - "original_type": "mysql:bit(1)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit16", - "original_type": "mysql:bit(16)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit64", - "original_type": "mysql:bit(64)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_bit_canon_0#03/source.numeric_types_bit": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 11000 - }, - "t_bit": { - "GoType": "[]uint8", - "Val": "AA==" - }, - "t_bit1": { - "GoType": "[]uint8", - "Val": "AA==" - }, - "t_bit16": { - "GoType": "[]uint8", - "Val": "AAA=" - }, - "t_bit64": { - "GoType": "[]uint8", - "Val": "AAAAAAAAAAA=" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 11001 - }, - "t_bit": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "t_bit1": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "t_bit16": { - "GoType": "[]uint8", - "Val": "//8=" - }, - "t_bit64": { - "GoType": "[]uint8", - "Val": "//////////8=" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_bit": { - "GoType": "", - "Val": null - }, - "t_bit1": { - "GoType": "", - "Val": null - }, - "t_bit16": { - "GoType": "", - "Val": null - }, - "t_bit64": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_bit", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit", - "original_type": "mysql:bit(1)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit1", - "original_type": "mysql:bit(1)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit16", - "original_type": "mysql:bit(16)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit64", - "original_type": "mysql:bit(64)", - "path": "", - "required": false, - "table_name": "numeric_types_bit", - "table_schema": "source", - "type": "string" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_boolean_canon_0#01/source.numeric_types_boolean": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10000 - }, - "t_bool": { - "GoType": "int8", - "Val": 0 - }, - "t_boolean": { - "GoType": "int8", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10001 - }, - "t_bool": { - "GoType": "int8", - "Val": 1 - }, - "t_boolean": { - "GoType": "int8", - "Val": 1 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_bool": { - "GoType": "", - "Val": null - }, - "t_boolean": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_boolean", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_boolean", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bool", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "numeric_types_boolean", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_boolean", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "numeric_types_boolean", - "table_schema": "source", - "type": "int8" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_boolean_canon_0#03/source.numeric_types_boolean": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10000 - }, - "t_bool": { - "GoType": "int8", - "Val": 0 - }, - "t_boolean": { - "GoType": "int8", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10001 - }, - "t_bool": { - "GoType": "int8", - "Val": 1 - }, - "t_boolean": { - "GoType": "int8", - "Val": 1 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_bool": { - "GoType": "", - "Val": null - }, - "t_boolean": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_boolean", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "numeric_types_boolean", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_boolean", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "numeric_types_boolean", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bool", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "numeric_types_boolean", - "table_schema": "source", - "type": "int8" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_decimal_canon_0#01/source.numeric_types_decimal": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13000 - }, - "t_dec": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 123.45 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": 12345678901234567890123456789012345678901234567890123456789012345 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 1234567890 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13001 - }, - "t_dec": { - "GoType": "json.Number", - "Val": -1234567890 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": -1234567890 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": -12345 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": -123.45 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": -12345678901234567890123456789012345678901234567890123456789012345 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 0 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": -1234567890 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": -1234567890 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13002 - }, - "t_dec": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 0.0 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 0 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": 0 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13003 - }, - "t_dec": { - "GoType": "json.Number", - "Val": 999999999 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 9999999999 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 99999 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 999.99 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": 99999999999999999999999999999999999999999999999999999999999999999 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 9999999999 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": 9999999999 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 9999999999 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13004 - }, - "t_dec": { - "GoType": "json.Number", - "Val": -9999999999 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": -9999999999 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": -99999 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": -999.99 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": -99999999999999999999999999999999999999999999999999999999999999999 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 0 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": -9999999999 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": -9999999999 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_dec": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_decimal_65": { - "GoType": "", - "Val": null - }, - "t_decimal_u": { - "GoType": "", - "Val": null - }, - "t_fixed": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_decimal", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_dec", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5", - "original_type": "mysql:decimal(5,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5_2", - "original_type": "mysql:decimal(5,2)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_65", - "original_type": "mysql:decimal(65,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_u", - "original_type": "mysql:decimal(10,0) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_fixed", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_decimal_canon_0#03/source.numeric_types_decimal": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13000 - }, - "t_dec": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 123.45 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": 12345678901234567890123456789012345678901234567890123456789012345 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": 1234567890 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 1234567890 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13001 - }, - "t_dec": { - "GoType": "json.Number", - "Val": -1234567890 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": -1234567890 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": -12345 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": -123.45 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": -12345678901234567890123456789012345678901234567890123456789012345 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 0 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": -1234567890 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": -1234567890 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13002 - }, - "t_dec": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": 0 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 0 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": 0 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13003 - }, - "t_dec": { - "GoType": "json.Number", - "Val": 999999999 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 9999999999 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 99999 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 999.99 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": 99999999999999999999999999999999999999999999999999999999999999999 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 9999999999 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": 9999999999 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 9999999999 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13004 - }, - "t_dec": { - "GoType": "json.Number", - "Val": -9999999999 - }, - "t_decimal": { - "GoType": "json.Number", - "Val": -9999999999 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": -99999 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": -999.99 - }, - "t_decimal_65": { - "GoType": "json.Number", - "Val": -99999999999999999999999999999999999999999999999999999999999999999 - }, - "t_decimal_u": { - "GoType": "json.Number", - "Val": 0 - }, - "t_fixed": { - "GoType": "json.Number", - "Val": -9999999999 - }, - "t_numeric": { - "GoType": "json.Number", - "Val": -9999999999 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_dec": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_decimal_65": { - "GoType": "", - "Val": null - }, - "t_decimal_u": { - "GoType": "", - "Val": null - }, - "t_fixed": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_decimal", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5", - "original_type": "mysql:decimal(5,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5_2", - "original_type": "mysql:decimal(5,2)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_u", - "original_type": "mysql:decimal(10,0) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_dec", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_fixed", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric", - "original_type": "mysql:decimal(10,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_65", - "original_type": "mysql:decimal(65,0)", - "path": "", - "required": false, - "table_name": "numeric_types_decimal", - "table_schema": "source", - "type": "double" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_float_canon_0#01/source.numeric_types_float": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13005 - }, - "t_double": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "json.Number", - "Val": -340282000000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13006 - }, - "t_double": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "json.Number", - "Val": 340282000000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13007 - }, - "t_double": { - "GoType": "json.Number", - "Val": -100000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "", - "Val": null - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13008 - }, - "t_double": { - "GoType": "json.Number", - "Val": 100000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "", - "Val": null - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14000 - }, - "t_double": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_float": { - "GoType": "json.Number", - "Val": 0 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": -12345678901234568000000000000000000000000000000000000 - }, - "t_real": { - "GoType": "json.Number", - "Val": 1.45e-45 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 12345678.9 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14001 - }, - "t_double": { - "GoType": "json.Number", - "Val": -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_float": { - "GoType": "json.Number", - "Val": -340282000000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": 100000000000000000000000000000000000000000000000000000 - }, - "t_real": { - "GoType": "json.Number", - "Val": 1.175494351e-38 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 99999999.99 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14002 - }, - "t_double": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_float": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_real": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 123456.12 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14003 - }, - "t_double": { - "GoType": "json.Number", - "Val": 179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_float": { - "GoType": "json.Number", - "Val": 340000000000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": 340000000000000000000000000000000000000 - }, - "t_real": { - "GoType": "json.Number", - "Val": 179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_double": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "", - "Val": null - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_float", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double_precision", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float", - "original_type": "mysql:float", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_53", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_real", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_real_10_2", - "original_type": "mysql:double(10,2)", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_float_canon_0#03/source.numeric_types_float": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14000 - }, - "t_double": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_float": { - "GoType": "json.Number", - "Val": 0 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": -12345678901234568000000000000000000000000000000000000 - }, - "t_real": { - "GoType": "json.Number", - "Val": 1.45e-45 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 12345678.9 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14001 - }, - "t_double": { - "GoType": "json.Number", - "Val": -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_float": { - "GoType": "json.Number", - "Val": -340282350000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": 100000000000000000000000000000000000000000000000000000 - }, - "t_real": { - "GoType": "json.Number", - "Val": 1.175494351e-38 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 99999999.99 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14002 - }, - "t_double": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_float": { - "GoType": "json.Number", - "Val": 123456.125 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_real": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 123456.12 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 14003 - }, - "t_double": { - "GoType": "json.Number", - "Val": 179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_float": { - "GoType": "json.Number", - "Val": 340000000000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "json.Number", - "Val": 340000000000000000000000000000000000000 - }, - "t_real": { - "GoType": "json.Number", - "Val": 179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - }, - "t_real_10_2": { - "GoType": "json.Number", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13005 - }, - "t_double": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "json.Number", - "Val": -340282350000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13006 - }, - "t_double": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "json.Number", - "Val": 340282350000000000000000000000000000000 - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13007 - }, - "t_double": { - "GoType": "json.Number", - "Val": -100000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "", - "Val": null - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 13008 - }, - "t_double": { - "GoType": "json.Number", - "Val": 100000000000000000000000000000000000000000000000000000000000000000 - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "", - "Val": null - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_double": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float": { - "GoType": "", - "Val": null - }, - "t_float_53": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_real_10_2": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_float", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_real", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_real_10_2", - "original_type": "mysql:double(10,2)", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float", - "original_type": "mysql:float", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_53", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double_precision", - "original_type": "mysql:double", - "path": "", - "required": false, - "table_name": "numeric_types_float", - "table_schema": "source", - "type": "double" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_int_canon_0#01/source.numeric_types_int": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 12000 - }, - "t_bigint": { - "GoType": "int64", - "Val": -9223372036854775807 - }, - "t_bigint5": { - "GoType": "int64", - "Val": -99999 - }, - "t_bigint_u": { - "GoType": "uint64", - "Val": 0 - }, - "t_int": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_int_u": { - "GoType": "uint32", - "Val": 0 - }, - "t_integer": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_integer5": { - "GoType": "int32", - "Val": -99999 - }, - "t_mediumint": { - "GoType": "int32", - "Val": -8388608 - }, - "t_mediumint5": { - "GoType": "int32", - "Val": -99999 - }, - "t_mediumint_u": { - "GoType": "uint32", - "Val": 0 - }, - "t_smallint": { - "GoType": "int16", - "Val": -32768 - }, - "t_smallint5": { - "GoType": "int16", - "Val": -32768 - }, - "t_smallint_u": { - "GoType": "uint16", - "Val": 0 - }, - "t_tinyint": { - "GoType": "int8", - "Val": -128 - }, - "t_tinyint1": { - "GoType": "int8", - "Val": -1 - }, - "t_tinyint1u": { - "GoType": "uint8", - "Val": 0 - }, - "t_tinyint_default0": { - "GoType": "int8", - "Val": -128 - }, - "t_tinyint_u": { - "GoType": "uint8", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 12001 - }, - "t_bigint": { - "GoType": "int64", - "Val": -9223372036854775807 - }, - "t_bigint5": { - "GoType": "int64", - "Val": -99999 - }, - "t_bigint_u": { - "GoType": "uint64", - "Val": 18446744073709551615 - }, - "t_int": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_int_u": { - "GoType": "uint32", - "Val": 0 - }, - "t_integer": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_integer5": { - "GoType": "int32", - "Val": 99999 - }, - "t_mediumint": { - "GoType": "int32", - "Val": 8388607 - }, - "t_mediumint5": { - "GoType": "int32", - "Val": 99999 - }, - "t_mediumint_u": { - "GoType": "uint32", - "Val": 16777215 - }, - "t_smallint": { - "GoType": "int16", - "Val": 32767 - }, - "t_smallint5": { - "GoType": "int16", - "Val": 32767 - }, - "t_smallint_u": { - "GoType": "uint16", - "Val": 65535 - }, - "t_tinyint": { - "GoType": "int8", - "Val": 127 - }, - "t_tinyint1": { - "GoType": "int8", - "Val": 1 - }, - "t_tinyint1u": { - "GoType": "uint8", - "Val": 1 - }, - "t_tinyint_default0": { - "GoType": "int8", - "Val": 127 - }, - "t_tinyint_u": { - "GoType": "uint8", - "Val": 255 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigint5": { - "GoType": "", - "Val": null - }, - "t_bigint_u": { - "GoType": "", - "Val": null - }, - "t_int": { - "GoType": "", - "Val": null - }, - "t_int_u": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_integer5": { - "GoType": "", - "Val": null - }, - "t_mediumint": { - "GoType": "", - "Val": null - }, - "t_mediumint5": { - "GoType": "", - "Val": null - }, - "t_mediumint_u": { - "GoType": "", - "Val": null - }, - "t_smallint": { - "GoType": "", - "Val": null - }, - "t_smallint5": { - "GoType": "", - "Val": null - }, - "t_smallint_u": { - "GoType": "", - "Val": null - }, - "t_tinyint": { - "GoType": "", - "Val": null - }, - "t_tinyint1": { - "GoType": "", - "Val": null - }, - "t_tinyint1u": { - "GoType": "", - "Val": null - }, - "t_tinyint_default0": { - "GoType": "", - "Val": null - }, - "t_tinyint_u": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_int", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint", - "original_type": "mysql:bigint(20)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint5", - "original_type": "mysql:bigint(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint_u", - "original_type": "mysql:bigint(20) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_integer", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_integer5", - "original_type": "mysql:int(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int_u", - "original_type": "mysql:int(10) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_mediumint", - "original_type": "mysql:mediumint(9)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_mediumint5", - "original_type": "mysql:mediumint(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_mediumint_u", - "original_type": "mysql:mediumint(8) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint", - "original_type": "mysql:smallint(6)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint5", - "original_type": "mysql:smallint(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint_u", - "original_type": "mysql:smallint(5) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint1", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint1u", - "original_type": "mysql:tinyint(1) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint_default0", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint_u", - "original_type": "mysql:tinyint(3) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint8" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_numeric_types_int_canon_0#03/source.numeric_types_int": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 12000 - }, - "t_bigint": { - "GoType": "int64", - "Val": -9223372036854775807 - }, - "t_bigint5": { - "GoType": "int64", - "Val": -99999 - }, - "t_bigint_u": { - "GoType": "uint64", - "Val": 0 - }, - "t_int": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_int_u": { - "GoType": "uint32", - "Val": 0 - }, - "t_integer": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_integer5": { - "GoType": "int32", - "Val": -99999 - }, - "t_mediumint": { - "GoType": "int32", - "Val": -8388608 - }, - "t_mediumint5": { - "GoType": "int32", - "Val": -99999 - }, - "t_mediumint_u": { - "GoType": "uint32", - "Val": 0 - }, - "t_smallint": { - "GoType": "int16", - "Val": -32768 - }, - "t_smallint5": { - "GoType": "int16", - "Val": -32768 - }, - "t_smallint_u": { - "GoType": "uint16", - "Val": 0 - }, - "t_tinyint": { - "GoType": "int8", - "Val": -128 - }, - "t_tinyint1": { - "GoType": "int8", - "Val": -1 - }, - "t_tinyint1u": { - "GoType": "uint8", - "Val": 0 - }, - "t_tinyint_default0": { - "GoType": "int8", - "Val": -128 - }, - "t_tinyint_u": { - "GoType": "uint8", - "Val": 0 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 12001 - }, - "t_bigint": { - "GoType": "int64", - "Val": -9223372036854775807 - }, - "t_bigint5": { - "GoType": "int64", - "Val": -99999 - }, - "t_bigint_u": { - "GoType": "uint64", - "Val": 18446744073709551615 - }, - "t_int": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_int_u": { - "GoType": "uint32", - "Val": 0 - }, - "t_integer": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_integer5": { - "GoType": "int32", - "Val": 99999 - }, - "t_mediumint": { - "GoType": "int32", - "Val": 8388607 - }, - "t_mediumint5": { - "GoType": "int32", - "Val": 99999 - }, - "t_mediumint_u": { - "GoType": "uint32", - "Val": 16777215 - }, - "t_smallint": { - "GoType": "int16", - "Val": 32767 - }, - "t_smallint5": { - "GoType": "int16", - "Val": 32767 - }, - "t_smallint_u": { - "GoType": "uint16", - "Val": 65535 - }, - "t_tinyint": { - "GoType": "int8", - "Val": 127 - }, - "t_tinyint1": { - "GoType": "int8", - "Val": 1 - }, - "t_tinyint1u": { - "GoType": "uint8", - "Val": 1 - }, - "t_tinyint_default0": { - "GoType": "int8", - "Val": 127 - }, - "t_tinyint_u": { - "GoType": "uint8", - "Val": 255 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigint5": { - "GoType": "", - "Val": null - }, - "t_bigint_u": { - "GoType": "", - "Val": null - }, - "t_int": { - "GoType": "", - "Val": null - }, - "t_int_u": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_integer5": { - "GoType": "", - "Val": null - }, - "t_mediumint": { - "GoType": "", - "Val": null - }, - "t_mediumint5": { - "GoType": "", - "Val": null - }, - "t_mediumint_u": { - "GoType": "", - "Val": null - }, - "t_smallint": { - "GoType": "", - "Val": null - }, - "t_smallint5": { - "GoType": "", - "Val": null - }, - "t_smallint_u": { - "GoType": "", - "Val": null - }, - "t_tinyint": { - "GoType": "", - "Val": null - }, - "t_tinyint1": { - "GoType": "", - "Val": null - }, - "t_tinyint1u": { - "GoType": "", - "Val": null - }, - "t_tinyint_default0": { - "GoType": "", - "Val": null - }, - "t_tinyint_u": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types_int", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint_default0", - "original_type": "mysql:tinyint(4)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint_u", - "original_type": "mysql:tinyint(3) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint1", - "original_type": "mysql:tinyint(1)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tinyint1u", - "original_type": "mysql:tinyint(1) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint", - "original_type": "mysql:smallint(6)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint5", - "original_type": "mysql:smallint(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint_u", - "original_type": "mysql:smallint(5) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_mediumint", - "original_type": "mysql:mediumint(9)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_mediumint5", - "original_type": "mysql:mediumint(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_mediumint_u", - "original_type": "mysql:mediumint(8) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_integer", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_integer5", - "original_type": "mysql:int(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int_u", - "original_type": "mysql:int(10) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint", - "original_type": "mysql:bigint(20)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint5", - "original_type": "mysql:bigint(5)", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint_u", - "original_type": "mysql:bigint(20) unsigned", - "path": "", - "required": false, - "table_name": "numeric_types_int", - "table_schema": "source", - "type": "uint64" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_string_types_emoji_canon_0#01/source.string_types_emoji": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10000 - }, - "t_char": { - "GoType": "string", - "Val": "❌" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_char": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "string_types_emoji", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": false, - "table_name": "string_types_emoji", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_char", - "original_type": "mysql:char(1)", - "path": "", - "required": false, - "table_name": "string_types_emoji", - "table_schema": "source", - "type": "utf8" - } - ] - }, - "gotest.gotest.TestAll/mysql.TestCanonSource_string_types_emoji_canon_0#03/source.string_types_emoji": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10000 - }, - "t_char": { - "GoType": "string", - "Val": "❌" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 801640048 - }, - "t_char": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "string_types_emoji", - "Namespace": "source" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "mysql:int(11)", - "path": "", - "required": true, - "table_name": "string_types_emoji", - "table_schema": "source", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_char", - "original_type": "mysql:char(1)", - "path": "", - "required": false, - "table_name": "string_types_emoji", - "table_schema": "source", - "type": "utf8" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_array_types_canon_0#01/public.array_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "arr_bl": { - "GoType": "[]interface {}", - "Val": [ - true, - true - ] - }, - "arr_c": { - "GoType": "[]interface {}", - "Val": [ - "1", - "1" - ] - }, - "arr_character_": { - "GoType": "[]interface {}", - "Val": [ - "abcd", - "abcd" - ] - }, - "arr_character_varying_": { - "GoType": "[]interface {}", - "Val": [ - "varc", - "varc" - ] - }, - "arr_d": { - "GoType": "[]interface {}", - "Val": [ - 3.14e-100, - 3.14e-100 - ] - }, - "arr_date_": { - "GoType": "[]interface {}", - "Val": [ - "1999-01-08T00:00:00Z", - "1999-01-08T00:00:00Z" - ] - }, - "arr_decimal_": { - "GoType": "[]interface {}", - "Val": [ - 123456.0, - 123456.0 - ] - }, - "arr_decimal_5": { - "GoType": "[]interface {}", - "Val": [ - 12345.0, - 12345.0 - ] - }, - "arr_decimal_5_2": { - "GoType": "[]interface {}", - "Val": [ - 123.67, - 123.67 - ] - }, - "arr_enum": { - "GoType": "[]interface {}", - "Val": [ - "VALUE_ONE", - "VALUE_THREE" - ] - }, - "arr_f": { - "GoType": "[]interface {}", - "Val": [ - 1.45e-10, - 1.45e-10 - ] - }, - "arr_i": { - "GoType": "[]interface {}", - "Val": [ - 1, - 1 - ] - }, - "arr_id": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_int": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_it": { - "GoType": "[]interface {}", - "Val": [ - "192.168.100.128/25", - "192.168.100.128/25" - ] - }, - "arr_numeric_": { - "GoType": "[]interface {}", - "Val": [ - 1.2676506002282294e+30, - 1.2676506002282294e+16 - ] - }, - "arr_numeric_5": { - "GoType": "[]interface {}", - "Val": [ - 12345.0, - 12345.0 - ] - }, - "arr_numeric_5_2": { - "GoType": "[]interface {}", - "Val": [ - 123.67, - 123.67 - ] - }, - "arr_oid_": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_real_": { - "GoType": "[]interface {}", - "Val": [ - 1.45e-10, - 1.45e-10 - ] - }, - "arr_si": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_str": { - "GoType": "[]interface {}", - "Val": [ - "varchar_example", - "varchar_example" - ] - }, - "arr_t": { - "GoType": "[]interface {}", - "Val": [ - "text_example", - "text_example" - ] - }, - "arr_time1": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06.1", - "04:05:06.1" - ] - }, - "arr_time3": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06.123", - "04:05:06.123" - ] - }, - "arr_time6": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06.123456", - "04:05:06.123456" - ] - }, - "arr_time_": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06", - "04:05:06" - ] - }, - "arr_time_with_time_zone_": { - "GoType": "[]interface {}", - "Val": [ - "00:51:02.746572-08", - "00:51:02.746572-08" - ] - }, - "arr_timestamp": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54Z", - "2004-10-19T10:23:54Z" - ] - }, - "arr_timestamp1": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54.9Z", - "2004-10-19T10:23:54.9Z" - ] - }, - "arr_timestamp3": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54.987Z", - "2004-10-19T10:23:54.987Z" - ] - }, - "arr_timestamp6": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54.987654Z", - "2004-10-19T10:23:54.987654Z" - ] - }, - "arr_timestamptz_": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54+02:00", - "2004-10-19T10:23:54+02:00" - ] - }, - "arr_timetz1": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25.5-04", - "13:30:25.5-04" - ] - }, - "arr_timetz3": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25.575-04", - "13:30:25.575-04" - ] - }, - "arr_timetz6": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25.575401-04", - "13:30:25.575401-04" - ] - }, - "arr_timetz_": { - "GoType": "[]interface {}", - "Val": [ - "00:51:02.746572-08", - "00:51:02.746572-08" - ] - }, - "arr_timetz__": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25-04", - "13:30:25-04" - ] - }, - "arr_tst": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T11:23:54+02:00", - "2004-10-19T11:23:54+02:00" - ] - }, - "arr_uid": { - "GoType": "[]interface {}", - "Val": [ - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11" - ] - } - } - } - ], - "TableID": { - "Name": "array_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "array_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_bl", - "original_type": "pg:boolean[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_si", - "original_type": "pg:smallint[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_int", - "original_type": "pg:integer[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_id", - "original_type": "pg:bigint[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_oid_", - "original_type": "pg:oid[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_real_", - "original_type": "pg:real[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_d", - "original_type": "pg:double precision[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_c", - "original_type": "pg:character(1)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_str", - "original_type": "pg:character varying(256)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_character_", - "original_type": "pg:character(4)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_character_varying_", - "original_type": "pg:character varying(5)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamptz_", - "original_type": "pg:timestamp with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_tst", - "original_type": "pg:timestamp with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz_", - "original_type": "pg:time with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time_with_time_zone_", - "original_type": "pg:time with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_uid", - "original_type": "pg:uuid[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_it", - "original_type": "pg:inet[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_f", - "original_type": "pg:double precision[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_i", - "original_type": "pg:integer[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_t", - "original_type": "pg:text[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_date_", - "original_type": "pg:date[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time_", - "original_type": "pg:time without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time1", - "original_type": "pg:time(1) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time3", - "original_type": "pg:time(3) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time6", - "original_type": "pg:time(6) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz__", - "original_type": "pg:time with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz1", - "original_type": "pg:time(1) with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz3", - "original_type": "pg:time(3) with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz6", - "original_type": "pg:time(6) with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp1", - "original_type": "pg:timestamp(1) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp3", - "original_type": "pg:timestamp(3) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp6", - "original_type": "pg:timestamp(6) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp", - "original_type": "pg:timestamp without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_numeric_", - "original_type": "pg:numeric[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_numeric_5", - "original_type": "pg:numeric(5,0)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_numeric_5_2", - "original_type": "pg:numeric(5,2)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_decimal_", - "original_type": "pg:numeric[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_decimal_5", - "original_type": "pg:numeric(5,0)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_decimal_5_2", - "original_type": "pg:numeric(5,2)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_enum", - "original_type": "pg:my_enum_type[]", - "path": "", - "properties": { - "pg:enum_all_values": [ - "VALUE_ONE", - "VALUE_TWO", - "VALUE_THREE" - ] - }, - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_array_types_canon_0#03/public.array_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "arr_bl": { - "GoType": "[]interface {}", - "Val": [ - true, - true - ] - }, - "arr_c": { - "GoType": "[]interface {}", - "Val": [ - "1", - "1" - ] - }, - "arr_character_": { - "GoType": "[]interface {}", - "Val": [ - "abcd", - "abcd" - ] - }, - "arr_character_varying_": { - "GoType": "[]interface {}", - "Val": [ - "varc", - "varc" - ] - }, - "arr_d": { - "GoType": "[]interface {}", - "Val": [ - 3.14e-100, - 3.14e-100 - ] - }, - "arr_date_": { - "GoType": "[]interface {}", - "Val": [ - "1999-01-08T00:00:00Z", - "1999-01-08T00:00:00Z" - ] - }, - "arr_decimal_": { - "GoType": "[]interface {}", - "Val": [ - 123456.0, - 123456.0 - ] - }, - "arr_decimal_5": { - "GoType": "[]interface {}", - "Val": [ - 12345.0, - 12345.0 - ] - }, - "arr_decimal_5_2": { - "GoType": "[]interface {}", - "Val": [ - 123.67, - 123.67 - ] - }, - "arr_enum": { - "GoType": "[]interface {}", - "Val": [ - "VALUE_ONE", - "VALUE_THREE" - ] - }, - "arr_f": { - "GoType": "[]interface {}", - "Val": [ - 1.45e-10, - 1.45e-10 - ] - }, - "arr_i": { - "GoType": "[]interface {}", - "Val": [ - 1, - 1 - ] - }, - "arr_id": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_int": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_it": { - "GoType": "[]interface {}", - "Val": [ - "192.168.100.128/25", - "192.168.100.128/25" - ] - }, - "arr_numeric_": { - "GoType": "[]interface {}", - "Val": [ - 1.2676506002282294e+30, - 1.2676506002282294e+16 - ] - }, - "arr_numeric_5": { - "GoType": "[]interface {}", - "Val": [ - 12345.0, - 12345.0 - ] - }, - "arr_numeric_5_2": { - "GoType": "[]interface {}", - "Val": [ - 123.67, - 123.67 - ] - }, - "arr_oid_": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_real_": { - "GoType": "[]interface {}", - "Val": [ - 1.45e-10, - 1.45e-10 - ] - }, - "arr_si": { - "GoType": "[]interface {}", - "Val": [ - 1, - 2 - ] - }, - "arr_str": { - "GoType": "[]interface {}", - "Val": [ - "varchar_example", - "varchar_example" - ] - }, - "arr_t": { - "GoType": "[]interface {}", - "Val": [ - "text_example", - "text_example" - ] - }, - "arr_time1": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06.1", - "04:05:06.1" - ] - }, - "arr_time3": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06.123", - "04:05:06.123" - ] - }, - "arr_time6": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06.123456", - "04:05:06.123456" - ] - }, - "arr_time_": { - "GoType": "[]interface {}", - "Val": [ - "04:05:06", - "04:05:06" - ] - }, - "arr_time_with_time_zone_": { - "GoType": "[]interface {}", - "Val": [ - "00:51:02.746572-08", - "00:51:02.746572-08" - ] - }, - "arr_timestamp": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54Z", - "2004-10-19T10:23:54Z" - ] - }, - "arr_timestamp1": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54.9Z", - "2004-10-19T10:23:54.9Z" - ] - }, - "arr_timestamp3": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54.987Z", - "2004-10-19T10:23:54.987Z" - ] - }, - "arr_timestamp6": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54.987654Z", - "2004-10-19T10:23:54.987654Z" - ] - }, - "arr_timestamptz_": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T10:23:54+02:00", - "2004-10-19T10:23:54+02:00" - ] - }, - "arr_timetz1": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25.5-04", - "13:30:25.5-04" - ] - }, - "arr_timetz3": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25.575-04", - "13:30:25.575-04" - ] - }, - "arr_timetz6": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25.575401-04", - "13:30:25.575401-04" - ] - }, - "arr_timetz_": { - "GoType": "[]interface {}", - "Val": [ - "00:51:02.746572-08", - "00:51:02.746572-08" - ] - }, - "arr_timetz__": { - "GoType": "[]interface {}", - "Val": [ - "13:30:25-04", - "13:30:25-04" - ] - }, - "arr_tst": { - "GoType": "[]interface {}", - "Val": [ - "2004-10-19T11:23:54+02:00", - "2004-10-19T11:23:54+02:00" - ] - }, - "arr_uid": { - "GoType": "[]interface {}", - "Val": [ - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", - "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11" - ] - } - } - } - ], - "TableID": { - "Name": "array_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "array_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_bl", - "original_type": "pg:boolean[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_si", - "original_type": "pg:smallint[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_int", - "original_type": "pg:integer[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_id", - "original_type": "pg:bigint[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_oid_", - "original_type": "pg:oid[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_real_", - "original_type": "pg:real[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_d", - "original_type": "pg:double precision[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_c", - "original_type": "pg:character(1)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_str", - "original_type": "pg:character varying(256)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_character_", - "original_type": "pg:character(4)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_character_varying_", - "original_type": "pg:character varying(5)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamptz_", - "original_type": "pg:timestamp with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_tst", - "original_type": "pg:timestamp with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz_", - "original_type": "pg:time with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time_with_time_zone_", - "original_type": "pg:time with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_uid", - "original_type": "pg:uuid[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_it", - "original_type": "pg:inet[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_f", - "original_type": "pg:double precision[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_i", - "original_type": "pg:integer[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_t", - "original_type": "pg:text[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_date_", - "original_type": "pg:date[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time_", - "original_type": "pg:time without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time1", - "original_type": "pg:time(1) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time3", - "original_type": "pg:time(3) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_time6", - "original_type": "pg:time(6) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz__", - "original_type": "pg:time with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz1", - "original_type": "pg:time(1) with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz3", - "original_type": "pg:time(3) with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timetz6", - "original_type": "pg:time(6) with time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp1", - "original_type": "pg:timestamp(1) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp3", - "original_type": "pg:timestamp(3) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp6", - "original_type": "pg:timestamp(6) without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_timestamp", - "original_type": "pg:timestamp without time zone[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_numeric_", - "original_type": "pg:numeric[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_numeric_5", - "original_type": "pg:numeric(5,0)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_numeric_5_2", - "original_type": "pg:numeric(5,2)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_decimal_", - "original_type": "pg:numeric[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_decimal_5", - "original_type": "pg:numeric(5,0)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_decimal_5_2", - "original_type": "pg:numeric(5,2)[]", - "path": "", - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "arr_enum", - "original_type": "pg:my_enum_type[]", - "path": "", - "properties": { - "pg:enum_all_values": [ - "VALUE_ONE", - "VALUE_TWO", - "VALUE_THREE" - ] - }, - "required": false, - "table_name": "array_types", - "table_schema": "public", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_date_types_canon_0#01/public.date_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "t_date": { - "GoType": "time.Time", - "Val": "1999-01-08T00:00:00Z" - }, - "t_interval": { - "GoType": "string", - "Val": "1 day 01:00:00.000000" - }, - "t_time": { - "GoType": "string", - "Val": "04:05:06" - }, - "t_time_1": { - "GoType": "string", - "Val": "04:05:06.1" - }, - "t_time_3": { - "GoType": "string", - "Val": "04:05:06.123" - }, - "t_time_6": { - "GoType": "string", - "Val": "04:05:06.123456" - }, - "t_time_with_time_zone_": { - "GoType": "string", - "Val": "00:51:02.746572-08" - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54+02:00" - }, - "t_timestamp_1": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.9+02:00" - }, - "t_timestamp_3": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.987+02:00" - }, - "t_timestamp_6": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.987654+02:00" - }, - "t_timestamptz": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54+02:00" - }, - "t_timetz": { - "GoType": "string", - "Val": "00:51:02.746572-08" - }, - "t_timetz_1": { - "GoType": "string", - "Val": "13:30:25.5-04" - }, - "t_timetz_3": { - "GoType": "string", - "Val": "13:30:25.575-04" - }, - "t_timetz_6": { - "GoType": "string", - "Val": "13:30:25.575401-04" - }, - "t_tst": { - "GoType": "time.Time", - "Val": "2004-10-19T11:23:54+02:00" - } - } - } - ], - "TableID": { - "Name": "date_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "date_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamptz", - "original_type": "pg:timestamp with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tst", - "original_type": "pg:timestamp with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz", - "original_type": "pg:time with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_with_time_zone_", - "original_type": "pg:time with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_interval", - "original_type": "pg:interval", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_date", - "original_type": "pg:date", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time", - "original_type": "pg:time without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_1", - "original_type": "pg:time(1) without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_3", - "original_type": "pg:time(3) without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_6", - "original_type": "pg:time(6) without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz_1", - "original_type": "pg:time(1) with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz_3", - "original_type": "pg:time(3) with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz_6", - "original_type": "pg:time(6) with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp_1", - "original_type": "pg:timestamp(1) without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp_3", - "original_type": "pg:timestamp(3) without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp_6", - "original_type": "pg:timestamp(6) without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp", - "original_type": "pg:timestamp without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_date_types_canon_0#03/public.date_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "t_date": { - "GoType": "time.Time", - "Val": "1999-01-08T00:00:00Z" - }, - "t_interval": { - "GoType": "string", - "Val": "1 day 01:00:00" - }, - "t_time": { - "GoType": "string", - "Val": "04:05:06" - }, - "t_time_1": { - "GoType": "string", - "Val": "04:05:06.1" - }, - "t_time_3": { - "GoType": "string", - "Val": "04:05:06.123" - }, - "t_time_6": { - "GoType": "string", - "Val": "04:05:06.123456" - }, - "t_time_with_time_zone_": { - "GoType": "string", - "Val": "00:51:02.746572-08" - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54+02:00" - }, - "t_timestamp_1": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.9+02:00" - }, - "t_timestamp_3": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.987+02:00" - }, - "t_timestamp_6": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54.987654+02:00" - }, - "t_timestamptz": { - "GoType": "time.Time", - "Val": "2004-10-19T10:23:54+02:00" - }, - "t_timetz": { - "GoType": "string", - "Val": "00:51:02.746572-08" - }, - "t_timetz_1": { - "GoType": "string", - "Val": "13:30:25.5-04" - }, - "t_timetz_3": { - "GoType": "string", - "Val": "13:30:25.575-04" - }, - "t_timetz_6": { - "GoType": "string", - "Val": "13:30:25.575401-04" - }, - "t_tst": { - "GoType": "time.Time", - "Val": "2004-10-19T11:23:54+02:00" - } - } - } - ], - "TableID": { - "Name": "date_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "date_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamptz", - "original_type": "pg:timestamp with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tst", - "original_type": "pg:timestamp with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz", - "original_type": "pg:time with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_with_time_zone_", - "original_type": "pg:time with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_interval", - "original_type": "pg:interval", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_date", - "original_type": "pg:date", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time", - "original_type": "pg:time without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_1", - "original_type": "pg:time(1) without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_3", - "original_type": "pg:time(3) without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_time_6", - "original_type": "pg:time(6) without time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz_1", - "original_type": "pg:time(1) with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz_3", - "original_type": "pg:time(3) with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timetz_6", - "original_type": "pg:time(6) with time zone", - "path": "", - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp_1", - "original_type": "pg:timestamp(1) without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp_3", - "original_type": "pg:timestamp(3) without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp_6", - "original_type": "pg:timestamp(6) without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp", - "original_type": "pg:timestamp without time zone", - "path": "", - "properties": { - "pg:database_timezone": "Europe/Berlin" - }, - "required": false, - "table_name": "date_types", - "table_schema": "public", - "type": "timestamp" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_geom_types_canon_0#01/public.geom_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "t_box": { - "GoType": "string", - "Val": "(3,3),(1,1)" - }, - "t_circle": { - "GoType": "string", - "Val": "<(1,1),10>" - }, - "t_line": { - "GoType": "string", - "Val": "{1,-1,1}" - }, - "t_lseg": { - "GoType": "string", - "Val": "(1,2),(2,3)" - }, - "t_path": { - "GoType": "string", - "Val": "[(1,1),(2,2),(2,3)]" - }, - "t_point": { - "GoType": "string", - "Val": "(1,2)" - }, - "t_polygon": { - "GoType": "string", - "Val": "((1,1),(2,2),(2,3),(1,1))" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "t_box": { - "GoType": "", - "Val": null - }, - "t_circle": { - "GoType": "", - "Val": null - }, - "t_line": { - "GoType": "", - "Val": null - }, - "t_lseg": { - "GoType": "", - "Val": null - }, - "t_path": { - "GoType": "", - "Val": null - }, - "t_point": { - "GoType": "", - "Val": null - }, - "t_polygon": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "geom_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "geom_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_point", - "original_type": "pg:point", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_line", - "original_type": "pg:line", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_lseg", - "original_type": "pg:lseg", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_box", - "original_type": "pg:box", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_path", - "original_type": "pg:path", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_polygon", - "original_type": "pg:polygon", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_circle", - "original_type": "pg:circle", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_geom_types_canon_0#03/public.geom_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 3 - }, - "t_box": { - "GoType": "string", - "Val": "(3,3),(1,1)" - }, - "t_circle": { - "GoType": "string", - "Val": "<(1,1),10>" - }, - "t_line": { - "GoType": "string", - "Val": "{1,-1,1}" - }, - "t_lseg": { - "GoType": "string", - "Val": "(1,2),(2,3)" - }, - "t_path": { - "GoType": "string", - "Val": "[(1,1),(2,2),(2,3)]" - }, - "t_point": { - "GoType": "string", - "Val": "(1,2)" - }, - "t_polygon": { - "GoType": "string", - "Val": "((1,1),(2,2),(2,3),(1,1))" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 4 - }, - "t_box": { - "GoType": "", - "Val": null - }, - "t_circle": { - "GoType": "", - "Val": null - }, - "t_line": { - "GoType": "", - "Val": null - }, - "t_lseg": { - "GoType": "", - "Val": null - }, - "t_path": { - "GoType": "", - "Val": null - }, - "t_point": { - "GoType": "", - "Val": null - }, - "t_polygon": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "geom_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "geom_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_point", - "original_type": "pg:point", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_line", - "original_type": "pg:line", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_lseg", - "original_type": "pg:lseg", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_box", - "original_type": "pg:box", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_path", - "original_type": "pg:path", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_polygon", - "original_type": "pg:polygon", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_circle", - "original_type": "pg:circle", - "path": "", - "required": false, - "table_name": "geom_types", - "table_schema": "public", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_numeric_types_canon_0#01/public.numeric_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "t_bigint": { - "GoType": "int64", - "Val": 1 - }, - "t_bigserial": { - "GoType": "int64", - "Val": 3372036854775807 - }, - "t_boolean": { - "GoType": "bool", - "Val": true - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 123456.0 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 12345.0 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 123.23 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_float_11": { - "GoType": "json.Number", - "Val": 2.802596928649634e-45 - }, - "t_float_4": { - "GoType": "json.Number", - "Val": 1.401298464324817e-45 - }, - "t_float_8": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_integer": { - "GoType": "int32", - "Val": -8388605 - }, - "t_money": { - "GoType": "string", - "Val": "$99.23" - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 1.2676506002282294e+30 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": 12345.0 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": 123.67 - }, - "t_oid": { - "GoType": "int64", - "Val": 2 - }, - "t_real": { - "GoType": "json.Number", - "Val": 1.401298464324817e-45 - }, - "t_serial": { - "GoType": "int32", - "Val": 0 - }, - "t_smallint": { - "GoType": "int16", - "Val": -32768 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "t_bigint": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_bigserial": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_boolean": { - "GoType": "bool", - "Val": true - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 12345.0 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 123.23 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_float_11": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_float_4": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_float_8": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_integer": { - "GoType": "int32", - "Val": 8388605 - }, - "t_money": { - "GoType": "string", - "Val": "$99.23" - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": 12345.0 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": 123.67 - }, - "t_oid": { - "GoType": "int64", - "Val": 2 - }, - "t_real": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_serial": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_smallint": { - "GoType": "int16", - "Val": 32767 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 3 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigserial": { - "GoType": "int64", - "Val": 1 - }, - "t_boolean": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float_11": { - "GoType": "", - "Val": null - }, - "t_float_4": { - "GoType": "", - "Val": null - }, - "t_float_8": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_money": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 0.0 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": 0.0 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": 0.0 - }, - "t_oid": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_serial": { - "GoType": "int32", - "Val": 1 - }, - "t_smallint": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 4 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigserial": { - "GoType": "int64", - "Val": 2 - }, - "t_boolean": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float_11": { - "GoType": "", - "Val": null - }, - "t_float_4": { - "GoType": "", - "Val": null - }, - "t_float_8": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_money": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "json.Number", - "Val": -123456.123456789 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": -12345.0 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": -123.67 - }, - "t_oid": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_serial": { - "GoType": "int32", - "Val": 2 - }, - "t_smallint": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 5 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigserial": { - "GoType": "int64", - "Val": 3 - }, - "t_boolean": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float_11": { - "GoType": "", - "Val": null - }, - "t_float_4": { - "GoType": "", - "Val": null - }, - "t_float_8": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_money": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "", - "Val": null - }, - "t_numeric_5": { - "GoType": "", - "Val": null - }, - "t_numeric_5_2": { - "GoType": "", - "Val": null - }, - "t_oid": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_serial": { - "GoType": "int32", - "Val": 3 - }, - "t_smallint": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_boolean", - "original_type": "pg:boolean", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint", - "original_type": "pg:smallint", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_integer", - "original_type": "pg:integer", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint", - "original_type": "pg:bigint", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_oid", - "original_type": "pg:oid", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal", - "original_type": "pg:numeric", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5", - "original_type": "pg:numeric(5,0)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5_2", - "original_type": "pg:numeric(5,2)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric", - "original_type": "pg:numeric", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric_5", - "original_type": "pg:numeric(5,0)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric_5_2", - "original_type": "pg:numeric(5,2)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_real", - "original_type": "pg:real", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_4", - "original_type": "pg:real", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_8", - "original_type": "pg:double precision", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_11", - "original_type": "pg:real", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double_precision", - "original_type": "pg:double precision", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_serial", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigserial", - "original_type": "pg:bigint", - "path": "", - "required": true, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_money", - "original_type": "pg:money", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "utf8" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_numeric_types_canon_0#03/public.numeric_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 6 - }, - "t_bigint": { - "GoType": "int64", - "Val": 1 - }, - "t_bigserial": { - "GoType": "int64", - "Val": 3372036854775807 - }, - "t_boolean": { - "GoType": "bool", - "Val": true - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 123.23 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_float_11": { - "GoType": "json.Number", - "Val": 2.8026e-45 - }, - "t_float_4": { - "GoType": "json.Number", - "Val": 1.4013e-45 - }, - "t_float_8": { - "GoType": "json.Number", - "Val": 5e-324 - }, - "t_integer": { - "GoType": "int32", - "Val": -8388605 - }, - "t_money": { - "GoType": "string", - "Val": "$99.23" - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 1267650600228229401496703205376 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": 123.67 - }, - "t_oid": { - "GoType": "json.Number", - "Val": 2 - }, - "t_real": { - "GoType": "json.Number", - "Val": 1.4013e-45 - }, - "t_serial": { - "GoType": "int32", - "Val": 0 - }, - "t_smallint": { - "GoType": "int16", - "Val": -32768 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 7 - }, - "t_bigint": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_bigserial": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_boolean": { - "GoType": "bool", - "Val": true - }, - "t_decimal": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_decimal_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "t_decimal_5_2": { - "GoType": "json.Number", - "Val": 123.23 - }, - "t_double_precision": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_float_11": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_float_4": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_float_8": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_integer": { - "GoType": "int32", - "Val": 8388605 - }, - "t_money": { - "GoType": "string", - "Val": "$99.23" - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 123456.123456789 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": 12345 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": 123.67 - }, - "t_oid": { - "GoType": "json.Number", - "Val": 2 - }, - "t_real": { - "GoType": "json.Number", - "Val": 123456 - }, - "t_serial": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_smallint": { - "GoType": "int16", - "Val": 32767 - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 8 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigserial": { - "GoType": "int64", - "Val": 4 - }, - "t_boolean": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float_11": { - "GoType": "", - "Val": null - }, - "t_float_4": { - "GoType": "", - "Val": null - }, - "t_float_8": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_money": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "json.Number", - "Val": 0.0 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": 0 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": 0.0 - }, - "t_oid": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_serial": { - "GoType": "int32", - "Val": 4 - }, - "t_smallint": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 9 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigserial": { - "GoType": "int64", - "Val": 5 - }, - "t_boolean": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float_11": { - "GoType": "", - "Val": null - }, - "t_float_4": { - "GoType": "", - "Val": null - }, - "t_float_8": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_money": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "json.Number", - "Val": -123456.123456789 - }, - "t_numeric_5": { - "GoType": "json.Number", - "Val": -12345 - }, - "t_numeric_5_2": { - "GoType": "json.Number", - "Val": -123.67 - }, - "t_oid": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_serial": { - "GoType": "int32", - "Val": 5 - }, - "t_smallint": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 10 - }, - "t_bigint": { - "GoType": "", - "Val": null - }, - "t_bigserial": { - "GoType": "int64", - "Val": 6 - }, - "t_boolean": { - "GoType": "", - "Val": null - }, - "t_decimal": { - "GoType": "", - "Val": null - }, - "t_decimal_5": { - "GoType": "", - "Val": null - }, - "t_decimal_5_2": { - "GoType": "", - "Val": null - }, - "t_double_precision": { - "GoType": "", - "Val": null - }, - "t_float_11": { - "GoType": "", - "Val": null - }, - "t_float_4": { - "GoType": "", - "Val": null - }, - "t_float_8": { - "GoType": "", - "Val": null - }, - "t_integer": { - "GoType": "", - "Val": null - }, - "t_money": { - "GoType": "", - "Val": null - }, - "t_numeric": { - "GoType": "", - "Val": null - }, - "t_numeric_5": { - "GoType": "", - "Val": null - }, - "t_numeric_5_2": { - "GoType": "", - "Val": null - }, - "t_oid": { - "GoType": "", - "Val": null - }, - "t_real": { - "GoType": "", - "Val": null - }, - "t_serial": { - "GoType": "int32", - "Val": 6 - }, - "t_smallint": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "numeric_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_boolean", - "original_type": "pg:boolean", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_smallint", - "original_type": "pg:smallint", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_integer", - "original_type": "pg:integer", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigint", - "original_type": "pg:bigint", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_oid", - "original_type": "pg:oid", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal", - "original_type": "pg:numeric", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5", - "original_type": "pg:numeric(5,0)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_decimal_5_2", - "original_type": "pg:numeric(5,2)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric", - "original_type": "pg:numeric", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric_5", - "original_type": "pg:numeric(5,0)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_numeric_5_2", - "original_type": "pg:numeric(5,2)", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_real", - "original_type": "pg:real", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_4", - "original_type": "pg:real", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_8", - "original_type": "pg:double precision", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float_11", - "original_type": "pg:real", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double_precision", - "original_type": "pg:double precision", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_serial", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bigserial", - "original_type": "pg:bigint", - "path": "", - "required": true, - "table_name": "numeric_types", - "table_schema": "public", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_money", - "original_type": "pg:money", - "path": "", - "required": false, - "table_name": "numeric_types", - "table_schema": "public", - "type": "utf8" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_text_types_canon_0#01/public.text_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "t_bit_1": { - "GoType": "string", - "Val": "1" - }, - "t_bit_8": { - "GoType": "string", - "Val": "10101111" - }, - "t_bytea": { - "GoType": "[]uint8", - "Val": "yv66vg==" - }, - "t_char": { - "GoType": "string", - "Val": "1" - }, - "t_character_": { - "GoType": "string", - "Val": "abcd" - }, - "t_character_varying_": { - "GoType": "string", - "Val": "varc" - }, - "t_text": { - "GoType": "string", - "Val": "text_example" - }, - "t_varbit_8": { - "GoType": "string", - "Val": "10101110" - }, - "t_varchar_256": { - "GoType": "string", - "Val": "varchar_example" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "t_bit_1": { - "GoType": "", - "Val": null - }, - "t_bit_8": { - "GoType": "", - "Val": null - }, - "t_bytea": { - "GoType": "", - "Val": null - }, - "t_char": { - "GoType": "", - "Val": null - }, - "t_character_": { - "GoType": "", - "Val": null - }, - "t_character_varying_": { - "GoType": "", - "Val": null - }, - "t_text": { - "GoType": "", - "Val": null - }, - "t_varbit_8": { - "GoType": "", - "Val": null - }, - "t_varchar_256": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "text_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "text_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_text", - "original_type": "pg:text", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_char", - "original_type": "pg:character(1)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_varchar_256", - "original_type": "pg:character varying(256)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_character_", - "original_type": "pg:character(4)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_character_varying_", - "original_type": "pg:character varying(5)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit_1", - "original_type": "pg:bit(1)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit_8", - "original_type": "pg:bit(8)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_varbit_8", - "original_type": "pg:bit varying(8)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bytea", - "original_type": "pg:bytea", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "string" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_text_types_canon_0#03/public.text_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 3 - }, - "t_bit_1": { - "GoType": "string", - "Val": "1" - }, - "t_bit_8": { - "GoType": "string", - "Val": "10101111" - }, - "t_bytea": { - "GoType": "[]uint8", - "Val": "yv66vg==" - }, - "t_char": { - "GoType": "string", - "Val": "1" - }, - "t_character_": { - "GoType": "string", - "Val": "abcd" - }, - "t_character_varying_": { - "GoType": "string", - "Val": "varc" - }, - "t_text": { - "GoType": "string", - "Val": "text_example" - }, - "t_varbit_8": { - "GoType": "string", - "Val": "10101110" - }, - "t_varchar_256": { - "GoType": "string", - "Val": "varchar_example" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 4 - }, - "t_bit_1": { - "GoType": "", - "Val": null - }, - "t_bit_8": { - "GoType": "", - "Val": null - }, - "t_bytea": { - "GoType": "", - "Val": null - }, - "t_char": { - "GoType": "", - "Val": null - }, - "t_character_": { - "GoType": "", - "Val": null - }, - "t_character_varying_": { - "GoType": "", - "Val": null - }, - "t_text": { - "GoType": "", - "Val": null - }, - "t_varbit_8": { - "GoType": "", - "Val": null - }, - "t_varchar_256": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "text_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "text_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_text", - "original_type": "pg:text", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_char", - "original_type": "pg:character(1)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_varchar_256", - "original_type": "pg:character varying(256)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_character_", - "original_type": "pg:character(4)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_character_varying_", - "original_type": "pg:character varying(5)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit_1", - "original_type": "pg:bit(1)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bit_8", - "original_type": "pg:bit(8)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_varbit_8", - "original_type": "pg:bit varying(8)", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bytea", - "original_type": "pg:bytea", - "path": "", - "required": false, - "table_name": "text_types", - "table_schema": "public", - "type": "string" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_wtf_types_canon_0#01/public.wtf_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 1 - }, - "j": { - "GoType": "map[string]interface {}", - "Val": { - "k": "v", - "kf": 1.2, - "ki": 42, - "kn": null, - "ks": "Ho Ho Ho my name's \"SANTA CLAWS\"" - } - }, - "jb": { - "GoType": "string", - "Val": "\"String in quotes\"" - }, - "t_cidr": { - "GoType": "string", - "Val": "10.1.0.0/16" - }, - "t_citext": { - "GoType": "string", - "Val": "Tom" - }, - "t_hstore": { - "GoType": "map[string]interface {}", - "Val": { - "a": "1", - "b": "2" - } - }, - "t_iner": { - "GoType": "string", - "Val": "192.168.1.5/32" - }, - "t_macaddr": { - "GoType": "string", - "Val": "08:00:2b:01:02:03" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 2 - }, - "j": { - "GoType": "", - "Val": null - }, - "jb": { - "GoType": "", - "Val": null - }, - "t_cidr": { - "GoType": "", - "Val": null - }, - "t_citext": { - "GoType": "", - "Val": null - }, - "t_hstore": { - "GoType": "", - "Val": null - }, - "t_iner": { - "GoType": "", - "Val": null - }, - "t_macaddr": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "wtf_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "wtf_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_hstore", - "original_type": "pg:hstore", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_iner", - "original_type": "pg:inet", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_cidr", - "original_type": "pg:cidr", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_macaddr", - "original_type": "pg:macaddr", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_citext", - "original_type": "pg:citext", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "j", - "original_type": "pg:json", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "jb", - "original_type": "pg:jsonb", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/pg.TestCanonSource_wtf_types_canon_0#03/public.wtf_types": { - "Rows": [ - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 3 - }, - "j": { - "GoType": "map[string]interface {}", - "Val": { - "k": "v", - "kf": 1.2, - "ki": 42, - "kn": null, - "ks": "Ho Ho Ho my name's \"SANTA CLAWS\"" - } - }, - "jb": { - "GoType": "string", - "Val": "\"String in quotes\"" - }, - "t_cidr": { - "GoType": "string", - "Val": "10.1.0.0/16" - }, - "t_citext": { - "GoType": "string", - "Val": "Tom" - }, - "t_hstore": { - "GoType": "map[string]interface {}", - "Val": { - "a": "1", - "b": "2" - } - }, - "t_iner": { - "GoType": "string", - "Val": "192.168.1.5" - }, - "t_macaddr": { - "GoType": "string", - "Val": "08:00:2b:01:02:03" - } - } - }, - { - "Data": { - "__primary_key": { - "GoType": "int32", - "Val": 4 - }, - "j": { - "GoType": "", - "Val": null - }, - "jb": { - "GoType": "", - "Val": null - }, - "t_cidr": { - "GoType": "", - "Val": null - }, - "t_citext": { - "GoType": "", - "Val": null - }, - "t_hstore": { - "GoType": "", - "Val": null - }, - "t_iner": { - "GoType": "", - "Val": null - }, - "t_macaddr": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "wtf_types", - "Namespace": "public" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__primary_key", - "original_type": "pg:integer", - "path": "", - "required": true, - "table_name": "wtf_types", - "table_schema": "public", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_hstore", - "original_type": "pg:hstore", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_iner", - "original_type": "pg:inet", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_cidr", - "original_type": "pg:cidr", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_macaddr", - "original_type": "pg:macaddr", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_citext", - "original_type": "pg:citext", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "j", - "original_type": "pg:json", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "jb", - "original_type": "pg:jsonb", - "path": "", - "required": false, - "table_name": "wtf_types", - "table_schema": "public", - "type": "any" - } - ] - }, - "gotest.gotest.TestAll/ydb.TestCanonSource_canon_0#01/.canon_table": { - "Rows": [ - { - "Data": { - "Bool_": { - "GoType": "bool", - "Val": true - }, - "Date_": { - "GoType": "time.Time", - "Val": "2020-02-02T00:00:00Z" - }, - "Datetime_": { - "GoType": "time.Time", - "Val": "2020-02-02T10:02:22Z" - }, - "Decimal_": { - "GoType": "string", - "Val": "234.000000000" - }, - "Double_": { - "GoType": "float64", - "Val": 2.2 - }, - "DyNumber_": { - "GoType": "string", - "Val": ".123e3" - }, - "Float_": { - "GoType": "float32", - "Val": 1.1 - }, - "Int16_": { - "GoType": "int16", - "Val": 2 - }, - "Int32_": { - "GoType": "int32", - "Val": 3 - }, - "Int64_": { - "GoType": "int64", - "Val": 4 - }, - "Int8_": { - "GoType": "int8", - "Val": 1 - }, - "Interval_": { - "GoType": "time.Duration", - "Val": 123000 - }, - "JsonDocument_": { - "GoType": "map[string]interface {}", - "Val": {} - }, - "Json_": { - "GoType": "map[string]interface {}", - "Val": {} - }, - "String_": { - "GoType": "[]uint8", - "Val": "AQ==" - }, - "Timestamp_": { - "GoType": "time.Time", - "Val": "2020-02-02T10:02:22Z" - }, - "Uint16_": { - "GoType": "uint16", - "Val": 6 - }, - "Uint32_": { - "GoType": "uint32", - "Val": 7 - }, - "Uint64_": { - "GoType": "uint64", - "Val": 8 - }, - "Uint8_": { - "GoType": "uint8", - "Val": 5 - }, - "Utf8_": { - "GoType": "string", - "Val": "my_utf8_string" - }, - "id": { - "GoType": "uint64", - "Val": 1 - } - } - }, - { - "Data": { - "Bool_": { - "GoType": "", - "Val": null - }, - "Date_": { - "GoType": "time.Time", - "Val": "2020-02-02T00:00:00Z" - }, - "Datetime_": { - "GoType": "time.Time", - "Val": "2020-02-02T10:02:22Z" - }, - "Decimal_": { - "GoType": "", - "Val": null - }, - "Double_": { - "GoType": "", - "Val": null - }, - "DyNumber_": { - "GoType": "", - "Val": null - }, - "Float_": { - "GoType": "", - "Val": null - }, - "Int16_": { - "GoType": "", - "Val": null - }, - "Int32_": { - "GoType": "", - "Val": null - }, - "Int64_": { - "GoType": "", - "Val": null - }, - "Int8_": { - "GoType": "", - "Val": null - }, - "Interval_": { - "GoType": "", - "Val": null - }, - "JsonDocument_": { - "GoType": "", - "Val": null - }, - "Json_": { - "GoType": "", - "Val": null - }, - "String_": { - "GoType": "", - "Val": null - }, - "Timestamp_": { - "GoType": "time.Time", - "Val": "2020-02-02T10:02:22Z" - }, - "Uint16_": { - "GoType": "", - "Val": null - }, - "Uint32_": { - "GoType": "", - "Val": null - }, - "Uint64_": { - "GoType": "", - "Val": null - }, - "Uint8_": { - "GoType": "", - "Val": null - }, - "Utf8_": { - "GoType": "", - "Val": null - }, - "id": { - "GoType": "uint64", - "Val": 801640048 - } - } - } - ], - "TableID": { - "Name": "canon_table", - "Namespace": "" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "id", - "original_type": "ydb:Uint64", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Bool_", - "original_type": "ydb:Bool", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Int8_", - "original_type": "ydb:Int8", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Int16_", - "original_type": "ydb:Int16", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Int32_", - "original_type": "ydb:Int32", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Int64_", - "original_type": "ydb:Int64", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Uint8_", - "original_type": "ydb:Uint8", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Uint16_", - "original_type": "ydb:Uint16", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Uint32_", - "original_type": "ydb:Uint32", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Uint64_", - "original_type": "ydb:Uint64", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Float_", - "original_type": "ydb:Float", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "float" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Double_", - "original_type": "ydb:Double", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Decimal_", - "original_type": "ydb:Decimal", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DyNumber_", - "original_type": "ydb:DyNumber", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "String_", - "original_type": "ydb:String", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Utf8_", - "original_type": "ydb:Utf8", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Json_", - "original_type": "ydb:Json", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "JsonDocument_", - "original_type": "ydb:JsonDocument", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Date_", - "original_type": "ydb:Date", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Datetime_", - "original_type": "ydb:Datetime", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Timestamp_", - "original_type": "ydb:Timestamp", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Interval_", - "original_type": "ydb:Interval", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "interval" - } - ] - }, - "gotest.gotest.TestAll/yt.TestCanonSourceWithDataObjects_canon_0/.some_table": { - "Rows": [ - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 0 - }, - "t_bool": { - "GoType": "bool", - "Val": false - }, - "t_date": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": null - }, - "t_double": { - "GoType": "json.Number", - "Val": 0 - }, - "t_float": { - "GoType": "float32", - "Val": 0 - }, - "t_int16": { - "GoType": "int16", - "Val": -32768 - }, - "t_int32": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_int64": { - "GoType": "int64", - "Val": -9223372036854775808 - }, - "t_int8": { - "GoType": "int8", - "Val": -128 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": -4291747199999999000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": null - }, - "t_opt_int64": { - "GoType": "", - "Val": null - }, - "t_string": { - "GoType": "[]uint8", - "Val": "" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldInt16", - 100 - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 0 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 0 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 0 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 0 - }, - "t_utf8": { - "GoType": "string", - "Val": "" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldInt16", - 100 - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 0, - 100 - ] - }, - "t_yson": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 1 - }, - "t_bool": { - "GoType": "bool", - "Val": true - }, - "t_date": { - "GoType": "time.Time", - "Val": "2021-12-27T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "2021-12-27T11:20:30Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": [ - [ - "my_key", - 100 - ] - ] - }, - "t_double": { - "GoType": "json.Number", - "Val": 2.2 - }, - "t_float": { - "GoType": "float32", - "Val": 2.2 - }, - "t_int16": { - "GoType": "int16", - "Val": -2000 - }, - "t_int32": { - "GoType": "int32", - "Val": -200000 - }, - "t_int64": { - "GoType": "int64", - "Val": -20000000000 - }, - "t_int8": { - "GoType": "int8", - "Val": 10 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": 60000000000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": [ - -1.01 - ] - }, - "t_opt_int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_string": { - "GoType": "[]uint8", - "Val": "VGVzdCBieXRlIHN0cmluZyAy" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldFloat32", - 100.01 - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2021-12-27T11:20:30.502383Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 2000 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 2000000 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 20000000000 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 20 - }, - "t_utf8": { - "GoType": "string", - "Val": "Test utf8 string 2" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldFloat32", - 100.01 - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 1, - 100.01 - ] - }, - "t_yson": { - "GoType": "[]interface {}", - "Val": [ - 100, - 200, - 300 - ] - } - } - }, - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 2 - }, - "t_bool": { - "GoType": "bool", - "Val": false - }, - "t_date": { - "GoType": "time.Time", - "Val": "2105-12-31T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "2105-12-31T23:59:59Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": [ - [ - "key1", - 1 - ], - [ - "key2", - 20 - ], - [ - "key3", - 300 - ] - ] - }, - "t_double": { - "GoType": "json.Number", - "Val": 42 - }, - "t_float": { - "GoType": "float32", - "Val": 42 - }, - "t_int16": { - "GoType": "int16", - "Val": 32767 - }, - "t_int32": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_int8": { - "GoType": "int8", - "Val": 127 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": 4291747199999999000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": [ - -1.01, - 2, - 1294.21 - ] - }, - "t_opt_int64": { - "GoType": "", - "Val": null - }, - "t_string": { - "GoType": "[]uint8", - "Val": "VGVzdCBieXRlIHN0cmluZyAz" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldString", - "100" - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2105-12-31T23:59:59Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 32767 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 2147483647 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 9223372036854775807 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 255 - }, - "t_utf8": { - "GoType": "string", - "Val": "Test utf8 string 3" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldString", - "magotan" - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 2, - "magotan" - ] - }, - "t_yson": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "some_table", - "Namespace": "" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "t_int8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int16", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int32", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int64", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint16", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint32", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint64", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "float" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bool", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_string", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_utf8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_date", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_datetime", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_interval", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "interval" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_yson", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_opt_int64", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_list", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_struct", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tuple", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_variant_named", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_variant_unnamed", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_dict", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tagged", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": true, - "name": "row_idx", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int64" - } - ] - }, - "gotest.gotest.TestAll/yt.TestCanonSourceWithDirInDataObjects_canon_0/.nested_dir3/some_table2": { - "Rows": [ - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 0 - }, - "t_bool": { - "GoType": "bool", - "Val": false - }, - "t_date": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": null - }, - "t_double": { - "GoType": "json.Number", - "Val": 0 - }, - "t_float": { - "GoType": "float32", - "Val": 0 - }, - "t_int16": { - "GoType": "int16", - "Val": -32768 - }, - "t_int32": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_int64": { - "GoType": "int64", - "Val": -9223372036854775808 - }, - "t_int8": { - "GoType": "int8", - "Val": -128 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": -4291747199999999000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": null - }, - "t_opt_int64": { - "GoType": "", - "Val": null - }, - "t_string": { - "GoType": "[]uint8", - "Val": "" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldInt16", - 100 - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 0 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 0 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 0 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 0 - }, - "t_utf8": { - "GoType": "string", - "Val": "" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldInt16", - 100 - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 0, - 100 - ] - }, - "t_yson": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 1 - }, - "t_bool": { - "GoType": "bool", - "Val": true - }, - "t_date": { - "GoType": "time.Time", - "Val": "2021-12-27T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "2021-12-27T11:20:30Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": [ - [ - "my_key", - 100 - ] - ] - }, - "t_double": { - "GoType": "json.Number", - "Val": 2.2 - }, - "t_float": { - "GoType": "float32", - "Val": 2.2 - }, - "t_int16": { - "GoType": "int16", - "Val": -2000 - }, - "t_int32": { - "GoType": "int32", - "Val": -200000 - }, - "t_int64": { - "GoType": "int64", - "Val": -20000000000 - }, - "t_int8": { - "GoType": "int8", - "Val": 10 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": 60000000000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": [ - -1.01 - ] - }, - "t_opt_int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_string": { - "GoType": "[]uint8", - "Val": "VGVzdCBieXRlIHN0cmluZyAy" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldFloat32", - 100.01 - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2021-12-27T11:20:30.502383Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 2000 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 2000000 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 20000000000 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 20 - }, - "t_utf8": { - "GoType": "string", - "Val": "Test utf8 string 2" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldFloat32", - 100.01 - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 1, - 100.01 - ] - }, - "t_yson": { - "GoType": "[]interface {}", - "Val": [ - 100, - 200, - 300 - ] - } - } - }, - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 2 - }, - "t_bool": { - "GoType": "bool", - "Val": false - }, - "t_date": { - "GoType": "time.Time", - "Val": "2105-12-31T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "2105-12-31T23:59:59Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": [ - [ - "key1", - 1 - ], - [ - "key2", - 20 - ], - [ - "key3", - 300 - ] - ] - }, - "t_double": { - "GoType": "json.Number", - "Val": 42 - }, - "t_float": { - "GoType": "float32", - "Val": 42 - }, - "t_int16": { - "GoType": "int16", - "Val": 32767 - }, - "t_int32": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_int8": { - "GoType": "int8", - "Val": 127 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": 4291747199999999000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": [ - -1.01, - 2, - 1294.21 - ] - }, - "t_opt_int64": { - "GoType": "", - "Val": null - }, - "t_string": { - "GoType": "[]uint8", - "Val": "VGVzdCBieXRlIHN0cmluZyAz" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldString", - "100" - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2105-12-31T23:59:59Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 32767 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 2147483647 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 9223372036854775807 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 255 - }, - "t_utf8": { - "GoType": "string", - "Val": "Test utf8 string 3" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldString", - "magotan" - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 2, - "magotan" - ] - }, - "t_yson": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "nested_dir3/some_table2", - "Namespace": "" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "t_int8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int16", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int32", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int64", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint16", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint32", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint64", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "float" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bool", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_string", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_utf8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_date", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_datetime", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_interval", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "interval" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_yson", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_opt_int64", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_list", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_struct", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tuple", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_variant_named", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_variant_unnamed", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_dict", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tagged", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": true, - "name": "row_idx", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int64" - } - ] - }, - "gotest.gotest.TestAll/yt.TestCanonSource_canon_0/.test_table": { - "Rows": [ - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 0 - }, - "t_bool": { - "GoType": "bool", - "Val": false - }, - "t_date": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": null - }, - "t_double": { - "GoType": "json.Number", - "Val": 0 - }, - "t_float": { - "GoType": "float32", - "Val": 0 - }, - "t_int16": { - "GoType": "int16", - "Val": -32768 - }, - "t_int32": { - "GoType": "int32", - "Val": -2147483648 - }, - "t_int64": { - "GoType": "int64", - "Val": -9223372036854775808 - }, - "t_int8": { - "GoType": "int8", - "Val": -128 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": -4291747199999999000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": null - }, - "t_opt_int64": { - "GoType": "", - "Val": null - }, - "t_string": { - "GoType": "[]uint8", - "Val": "" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldInt16", - 100 - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "1970-01-01T00:00:00Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 0 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 0 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 0 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 0 - }, - "t_utf8": { - "GoType": "string", - "Val": "" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldInt16", - 100 - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 0, - 100 - ] - }, - "t_yson": { - "GoType": "", - "Val": null - } - } - }, - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 1 - }, - "t_bool": { - "GoType": "bool", - "Val": true - }, - "t_date": { - "GoType": "time.Time", - "Val": "2021-12-27T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "2021-12-27T11:20:30Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": [ - [ - "my_key", - 100 - ] - ] - }, - "t_double": { - "GoType": "json.Number", - "Val": 2.2 - }, - "t_float": { - "GoType": "float32", - "Val": 2.2 - }, - "t_int16": { - "GoType": "int16", - "Val": -2000 - }, - "t_int32": { - "GoType": "int32", - "Val": -200000 - }, - "t_int64": { - "GoType": "int64", - "Val": -20000000000 - }, - "t_int8": { - "GoType": "int8", - "Val": 10 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": 60000000000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": [ - -1.01 - ] - }, - "t_opt_int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_string": { - "GoType": "[]uint8", - "Val": "VGVzdCBieXRlIHN0cmluZyAy" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldFloat32", - 100.01 - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2021-12-27T11:20:30.502383Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 2000 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 2000000 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 20000000000 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 20 - }, - "t_utf8": { - "GoType": "string", - "Val": "Test utf8 string 2" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldFloat32", - 100.01 - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 1, - 100.01 - ] - }, - "t_yson": { - "GoType": "[]interface {}", - "Val": [ - 100, - 200, - 300 - ] - } - } - }, - { - "Data": { - "row_idx": { - "GoType": "int64", - "Val": 2 - }, - "t_bool": { - "GoType": "bool", - "Val": false - }, - "t_date": { - "GoType": "time.Time", - "Val": "2105-12-31T00:00:00Z" - }, - "t_datetime": { - "GoType": "time.Time", - "Val": "2105-12-31T23:59:59Z" - }, - "t_dict": { - "GoType": "[]interface {}", - "Val": [ - [ - "key1", - 1 - ], - [ - "key2", - 20 - ], - [ - "key3", - 300 - ] - ] - }, - "t_double": { - "GoType": "json.Number", - "Val": 42 - }, - "t_float": { - "GoType": "float32", - "Val": 42 - }, - "t_int16": { - "GoType": "int16", - "Val": 32767 - }, - "t_int32": { - "GoType": "int32", - "Val": 2147483647 - }, - "t_int64": { - "GoType": "int64", - "Val": 9223372036854775807 - }, - "t_int8": { - "GoType": "int8", - "Val": 127 - }, - "t_interval": { - "GoType": "time.Duration", - "Val": 4291747199999999000 - }, - "t_list": { - "GoType": "[]interface {}", - "Val": [ - -1.01, - 2, - 1294.21 - ] - }, - "t_opt_int64": { - "GoType": "", - "Val": null - }, - "t_string": { - "GoType": "[]uint8", - "Val": "VGVzdCBieXRlIHN0cmluZyAz" - }, - "t_struct": { - "GoType": "map[string]interface {}", - "Val": { - "fieldFloat32": 100.01, - "fieldInt16": 100, - "fieldString": "abc" - } - }, - "t_tagged": { - "GoType": "[]interface {}", - "Val": [ - "fieldString", - "100" - ] - }, - "t_timestamp": { - "GoType": "time.Time", - "Val": "2105-12-31T23:59:59Z" - }, - "t_tuple": { - "GoType": "[]interface {}", - "Val": [ - -5, - 300.03, - "my data" - ] - }, - "t_uint16": { - "GoType": "uint16", - "Val": 32767 - }, - "t_uint32": { - "GoType": "uint32", - "Val": 2147483647 - }, - "t_uint64": { - "GoType": "uint64", - "Val": 9223372036854775807 - }, - "t_uint8": { - "GoType": "uint8", - "Val": 255 - }, - "t_utf8": { - "GoType": "string", - "Val": "Test utf8 string 3" - }, - "t_variant_named": { - "GoType": "[]interface {}", - "Val": [ - "fieldString", - "magotan" - ] - }, - "t_variant_unnamed": { - "GoType": "[]interface {}", - "Val": [ - 2, - "magotan" - ] - }, - "t_yson": { - "GoType": "", - "Val": null - } - } - } - ], - "TableID": { - "Name": "test_table", - "Namespace": "" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "t_int8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int16", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int32", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_int64", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint16", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint32", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_uint64", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_float", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "float" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_double", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_bool", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_string", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_utf8", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_date", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_datetime", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_timestamp", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_interval", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "interval" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_yson", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_opt_int64", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_list", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_struct", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tuple", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_variant_named", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_variant_unnamed", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_dict", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "t_tagged", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": true, - "name": "row_idx", - "original_type": "", - "path": "", - "required": true, - "table_name": "", - "table_schema": "", - "type": "int64" - } - ] - } -} diff --git a/tests/canon/mongo/README.md b/tests/canon/mongo/README.md deleted file mode 100644 index 31a8fd6e..00000000 --- a/tests/canon/mongo/README.md +++ /dev/null @@ -1,19 +0,0 @@ -### local run - -Start docker for server: -```shell -docker run -d --name some-mongo -p 27117:27017 \ - -e MONGO_INITDB_ROOT_USERNAME=user \ - -e MONGO_INITDB_ROOT_PASSWORD=password \ - -e MONGO_INITDB_DATABASE=canondb \ - mongo -``` - -end then: - -```shell -CGO_ENABLED=0; -DB0_MONGO_LOCAL_PASSWORD=password; -DB0_MONGO_LOCAL_PORT=27117; -DB0_MONGO_LOCAL_USER=user; -``` diff --git a/tests/canon/mongo/canon_docs.go b/tests/canon/mongo/canon_docs.go deleted file mode 100644 index a933cee9..00000000 --- a/tests/canon/mongo/canon_docs.go +++ /dev/null @@ -1,138 +0,0 @@ -package mongo - -import ( - "context" - "math" - "time" - - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func oid(hex string) primitive.ObjectID { - res, err := primitive.ObjectIDFromHex(hex) - if err != nil { - panic(err) - } - return res -} - -type IncrementUpdate struct{ Filter, Update interface{} } - -var ( - SnapshotDocuments = []interface{}{ - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b7")}, {Key: "x", Value: 4}, {Key: "y", Value: 7}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b8")}, {Key: "id", Value: bson.D{{Key: "z", Value: 2}, {Key: "incept", Value: bson.D{{Key: "y", Value: "asdf"}}}}}}, - bson.D{{Key: "_id", Value: bson.D{{Key: "x", Value: 0}, {Key: "y", Value: 1}}}}, - bson.D{{Key: "_id", Value: "omg"}}, - bson.D{{Key: "_id", Value: 8147}}, - bson.D{{Key: "_id", Value: int32(2499)}}, - bson.D{{Key: "_id", Value: int64(100502)}}, - bson.D{{Key: "_id", Value: true}}, - bson.D{{Key: "_id", Value: primitive.NewDecimal128(10101, 0)}}, - bson.D{{Key: "_id", Value: primitive.Timestamp{T: 1234526243, I: 1}}}, - bson.D{{Key: "_id", Value: primitive.NewDateTimeFromTime(time.Unix(848920, 4940200))}}, - bson.D{{Key: "_id", Value: primitive.Binary{Data: []byte("hello, world!")}}}, - bson.D{{Key: "_id", Value: primitive.Symbol("sex symbol")}}, - bson.D{{Key: "_id", Value: primitive.JavaScript("function(){ return \"omfg\" }")}, {Key: "what", Value: "is this?"}}, - bson.D{{Key: "_id", Value: primitive.Null{}}}, - bson.D{{Key: "_id", Value: primitive.DBPointer{DB: "dbdb", Pointer: oid("62e98ebcb28dbd2fffdf12b7")}}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b9")}, {Key: "string", Value: "2354"}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c0")}, {Key: "int", Value: 402}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c1")}, {Key: "double", Value: 1.27}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c3")}, {Key: "timestamp", Value: primitive.Timestamp{ - T: 222, - I: 1188888811, - }}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c4")}, {Key: "nil", Value: nil}}, - } - // this piece is valid only for repackable hetero transfer - ExtraSnapshotDocuments = []interface{}{ - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b4")}, {Key: "nan", Value: math.NaN()}, {Key: "inf+", Value: math.Inf(1)}, {Key: "inf-", Value: math.Inf(-1)}}, - } - IncrementDocuments = []interface{}{ - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c5")}, {Key: "s", Value: 1}, {Key: "t", Value: 2}}, - bson.D{{Key: "_id", Value: bson.D{{Key: "u", Value: 8}, {Key: "v", Value: 9}}}}, - bson.D{{Key: "_id", Value: "woof"}}, - bson.D{{Key: "_id", Value: 123}}, - bson.D{{Key: "_id", Value: int32(8440)}}, - bson.D{{Key: "_id", Value: int64(100503)}}, - bson.D{{Key: "_id", Value: false}}, - bson.D{{Key: "_id", Value: primitive.NewDecimal128(10101, 1)}}, - bson.D{{Key: "_id", Value: primitive.Timestamp{T: 1234526243, I: 2}}}, - bson.D{{Key: "_id", Value: primitive.NewDateTimeFromTime(time.Unix(74271040, 412))}}, - bson.D{{Key: "_id", Value: primitive.Binary{Data: []byte("Привет, мир!")}}}, - bson.D{{Key: "_id", Value: primitive.Symbol("no homo 🍆")}}, - bson.D{{Key: "_id", Value: primitive.DBPointer{DB: "ddbb", Pointer: oid("62e98ebcb28dbd2fffdf12c4")}}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c6")}, {Key: "id", Value: "x"}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c7")}, {Key: "id", Value: oid("62e98ebcb28dbd2fffdf12ba")}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c8")}, {Key: "id", Value: primitive.JavaScript("function(){}")}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c9")}, {Key: "decimal", Value: primitive.NewDecimal128(243994, 912994009199)}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12ca")}, {Key: "string", Value: "3"}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12cb")}, {Key: "int", Value: 5}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12cc")}, {Key: "double", Value: 2.56}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12ce")}, {Key: "timestamp", Value: primitive.Timestamp{ - T: 5555555, - I: 333, - }}}, - bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12cf")}, {Key: "nil", Value: primitive.Null{}}}, // primitive.Null is the same as nil - } - // Update existing - IncrementUpdates = []IncrementUpdate{ - { - Filter: bson.D{{Key: "_id", Value: "omg"}}, - Update: bson.D{{Key: "$set", Value: bson.D{{Key: "_id", Value: "omg"}, {Key: "omgval", Value: 44}}}}, - }, - { - Filter: bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b7")}}, - Update: bson.D{{Key: "$set", Value: bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b7")}, {Key: "x", Value: 29}}}}, - }, - } -) - -func DropCollection(ctx context.Context, Source *mongocommon.MongoSource, databaseName string, collectionName string) error { - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - if err != nil { - return err - } - defer client.Close(ctx) - collection := client.Database(databaseName).Collection(collectionName) - if err := collection.Drop(ctx); err != nil { - return err - } - return nil -} - -func InsertDocs(ctx context.Context, Source *mongocommon.MongoSource, databaseName string, collectionName string, docs ...any) error { - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - if err != nil { - return err - } - defer client.Close(ctx) - collection := client.Database(databaseName).Collection(collectionName) - insertManyOpts := new(options.InsertManyOptions). - SetBypassDocumentValidation(true) - - _, err = collection.InsertMany(ctx, docs, insertManyOpts) - return err -} - -func UpdateDocs(ctx context.Context, Source *mongocommon.MongoSource, databaseName string, collectionName string, docs ...IncrementUpdate) error { - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - if err != nil { - return err - } - defer client.Close(ctx) - collection := client.Database(databaseName).Collection(collectionName) - updateManyOpts := new(options.UpdateOptions). - SetBypassDocumentValidation(true) - for _, updDescr := range docs { - _, err = collection.UpdateMany(ctx, updDescr.Filter, updDescr.Update, updateManyOpts) - if err != nil { - return err - } - } - return nil -} diff --git a/tests/canon/mongo/canon_test.go b/tests/canon/mongo/canon_test.go deleted file mode 100644 index 282ea824..00000000 --- a/tests/canon/mongo/canon_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package mongo - -import ( - "context" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - databaseName := "canondb" - t.Run("vanilla hetero case", func(t *testing.T) { - snapshotPlusIncrementScenario(t, databaseName, "hetero_repack", false, false) - }) - t.Run("strange hetero case", func(t *testing.T) { - snapshotPlusIncrementScenario(t, databaseName, "hetero_no_repack", false, true) - }) -} - -func snapshotPlusIncrementScenario(t *testing.T, databaseName, collectionName string, isHomo, preventJSONRepack bool) { - Source := &mongocommon.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongocommon.MongoCollection{ - {DatabaseName: databaseName, CollectionName: collectionName}, - }, - IsHomo: isHomo, - PreventJSONRepack: preventJSONRepack, - } - Source.WithDefaults() - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - )) - }() - - ctx := context.Background() - require.NoError(t, DropCollection(ctx, Source, databaseName, collectionName)) - - require.NoError(t, InsertDocs(ctx, Source, databaseName, collectionName, SnapshotDocuments...)) - if !preventJSONRepack { - require.NoError(t, InsertDocs(ctx, Source, databaseName, collectionName, ExtraSnapshotDocuments...)) - } - - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(Source), - validator.InitDone(t), - validator.ValuesTypeChecker, - validator.Referencer(t), - validator.TypesystemChecker(mongocommon.ProviderType, func(colSchema abstract.ColSchema) string { - return strings.TrimPrefix(colSchema.OriginalType, "mongo:") - }), - ), - Cleanup: server.Drop, - }, - abstract.TransferTypeSnapshotAndIncrement, - ) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - time.Sleep(1 * time.Second) - - require.NoError(t, InsertDocs(ctx, Source, databaseName, collectionName, IncrementDocuments...)) - - require.NoError(t, UpdateDocs(ctx, Source, databaseName, collectionName, IncrementUpdates...)) - - time.Sleep(4 * mongocommon.DefaultBatchFlushInterval) -} diff --git a/tests/canon/mongo/gotest/canondata/result.json b/tests/canon/mongo/gotest/canondata/result.json deleted file mode 100644 index d18d924c..00000000 --- a/tests/canon/mongo/gotest/canondata/result.json +++ /dev/null @@ -1,1368 +0,0 @@ -{ - "gotest.gotest.TestCanonSource/strange_hetero_case/canondb.hetero_no_repack": { - "Rows": [ - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b7" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "x": 4, - "y": 7 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b8" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": { - "incept": { - "y": "asdf" - }, - "z": 2 - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "[{\"Key\":\"x\",\"Value\":0},{\"Key\":\"y\",\"Value\":1}]" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "omg" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "8147" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "2499" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "100502" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "true" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1.86330561888540180873216E-6153\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"T\":1234526243,\"I\":1}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1970-01-10T19:48:40.004Z\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "68656c6c6f2c20776f726c6421" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"sex symbol\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"function(){ return \\\"omfg\\\" }\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "what": "is this?" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "null" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"DB\":\"dbdb\",\"Pointer\":\"62e98ebcb28dbd2fffdf12b7\"}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b9" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "string": "2354" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c0" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "int": 402 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c1" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "double": 1.27 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c3" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "timestamp": { - "$timestamp": { - "i": 1188888811, - "t": 222 - } - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c4" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "nil": null - } - } - } - } - ], - "TableID": { - "Name": "hetero_no_repack", - "Namespace": "canondb" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "_id", - "original_type": "mongo:bson_id", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "document", - "original_type": "mongo:bson", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - } - ] - }, - "gotest.gotest.TestCanonSource/strange_hetero_case/canondb.hetero_no_repack#01": { - "Rows": [ - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b7" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "x": 29, - "y": 7 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "omg" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "omgval": 44 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12cf" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "nil": null - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12ce" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "timestamp": { - "$timestamp": { - "i": 333, - "t": 5555555 - } - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12cc" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "double": 2.56 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12cb" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "int": 5 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12ca" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "string": "3" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c9" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "decimal": { - "$numberDecimal": "4.500894873521601331003503E-6152" - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c8" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": { - "$code": "function(){}" - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c7" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": { - "$oid": "62e98ebcb28dbd2fffdf12ba" - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c6" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": "x" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"DB\":\"ddbb\",\"Pointer\":\"62e98ebcb28dbd2fffdf12c4\"}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"no homo 🍆\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "d09fd180d0b8d0b2d0b5d1822c20d0bcd0b8d18021" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1972-05-09T14:50:40Z\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"T\":1234526243,\"I\":2}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1.86330561888540180873217E-6153\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "false" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "100503" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "8440" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "123" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "woof" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "[{\"Key\":\"u\",\"Value\":8},{\"Key\":\"v\",\"Value\":9}]" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c5" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "s": 1, - "t": 2 - } - } - } - } - ], - "TableID": { - "Name": "hetero_no_repack", - "Namespace": "canondb" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "_id", - "original_type": "mongo:bson_id", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "document", - "original_type": "mongo:bson", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - } - ] - }, - "gotest.gotest.TestCanonSource/vanilla_hetero_case/canondb.hetero_repack": { - "Rows": [ - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b7" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "x": 4, - "y": 7 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b8" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": { - "incept": { - "y": "asdf" - }, - "z": 2 - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "[{\"Key\":\"x\",\"Value\":0},{\"Key\":\"y\",\"Value\":1}]" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "omg" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "8147" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "2499" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "100502" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "true" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1.86330561888540180873216E-6153\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"T\":1234526243,\"I\":1}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1970-01-10T19:48:40.004Z\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "68656c6c6f2c20776f726c6421" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"sex symbol\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"function(){ return \\\"omfg\\\" }\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "what": "is this?" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "null" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"DB\":\"dbdb\",\"Pointer\":\"62e98ebcb28dbd2fffdf12b7\"}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b9" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "string": "2354" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c0" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "int": 402 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c1" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "double": 1.27 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c3" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "timestamp": { - "$timestamp": { - "i": 1188888811, - "t": 222 - } - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c4" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "nil": null - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b4" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "inf+": { - "$numberDouble": "Infinity" - }, - "inf-": { - "$numberDouble": "-Infinity" - }, - "nan": { - "$numberDouble": "NaN" - } - } - } - } - } - ], - "TableID": { - "Name": "hetero_repack", - "Namespace": "canondb" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "_id", - "original_type": "mongo:bson_id", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "document", - "original_type": "mongo:bson", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - } - ] - }, - "gotest.gotest.TestCanonSource/vanilla_hetero_case/canondb.hetero_repack#01": { - "Rows": [ - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12b7" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "x": 29, - "y": 7 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "omg" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "omgval": 44 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12cf" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "nil": null - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12ce" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "timestamp": { - "$timestamp": { - "i": 333, - "t": 5555555 - } - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12cc" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "double": 2.56 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12cb" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "int": 5 - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12ca" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "string": "3" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c9" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "decimal": { - "$numberDecimal": "4.500894873521601331003503E-6152" - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c8" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": { - "$code": "function(){}" - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c7" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": { - "$oid": "62e98ebcb28dbd2fffdf12ba" - } - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c6" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "id": "x" - } - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"DB\":\"ddbb\",\"Pointer\":\"62e98ebcb28dbd2fffdf12c4\"}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"no homo 🍆\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "d09fd180d0b8d0b2d0b5d1822c20d0bcd0b8d18021" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1972-05-09T14:50:40Z\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "{\"T\":1234526243,\"I\":2}" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "\"1.86330561888540180873217E-6153\"" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "false" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "100503" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "8440" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "123" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "woof" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "[{\"Key\":\"u\",\"Value\":8},{\"Key\":\"v\",\"Value\":9}]" - }, - "document": { - "GoType": "mongo.DValue", - "Val": {} - } - } - }, - { - "Data": { - "_id": { - "GoType": "string", - "Val": "62e98ebcb28dbd2fffdf12c5" - }, - "document": { - "GoType": "mongo.DValue", - "Val": { - "s": 1, - "t": 2 - } - } - } - } - ], - "TableID": { - "Name": "hetero_repack", - "Namespace": "canondb" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "_id", - "original_type": "mongo:bson_id", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "document", - "original_type": "mongo:bson", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - } - ] - } -} diff --git a/tests/canon/mysql/canon_sql.go b/tests/canon/mysql/canon_sql.go deleted file mode 100644 index 726b8e93..00000000 --- a/tests/canon/mysql/canon_sql.go +++ /dev/null @@ -1,45 +0,0 @@ -package mysql - -import _ "embed" - -var ( - //go:embed dump/initial_data.sql - initial []byte - - //go:embed dump/date_types.sql - date_ []byte - //go:embed dump/json_types.sql - json_ []byte - //go:embed dump/numeric_types_bit.sql - numericBit_ []byte - //go:embed dump/numeric_types_boolean.sql - numericBoolean_ []byte - //go:embed dump/numeric_types_decimal.sql - numericDecimal_ []byte - //go:embed dump/numeric_types_float.sql - numericFloat_ []byte - //go:embed dump/numeric_types_int.sql - numericInt_ []byte - //go:embed dump/spatial_types.sql - spatial_ []byte - //go:embed dump/string_types.sql - string_ []byte - //go:embed dump/string_types_emoji.sql - stringEmoji_ []byte -) - -var ( - TableSQLs = map[string]string{ - "initial": string(initial), - "date_types": string(date_), - "json_types": string(json_), - "numeric_types_bit": string(numericBit_), - "numeric_types_boolean": string(numericBoolean_), - "numeric_types_decimal": string(numericDecimal_), - "numeric_types_float": string(numericFloat_), - "numeric_types_int": string(numericInt_), - "spatial_types": string(spatial_), - "string_types": string(string_), - "string_types_emoji": string(stringEmoji_), - } -) diff --git a/tests/canon/mysql/canon_test.go b/tests/canon/mysql/canon_test.go deleted file mode 100644 index 6f0b5cec..00000000 --- a/tests/canon/mysql/canon_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package mysql - -import ( - "database/sql" - _ "embed" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/dbaas/ycdbaas" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func execBatch(t *testing.T, conn *sql.DB, sqlCommands string) { - arr := strings.Split(sqlCommands, ";") - for _, command := range arr { - if command == "" || command == "\n" { - continue - } - _, err := conn.Exec(command) - require.NoError(t, err) - } -} - -func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - Source := &mysql.MysqlSource{ - ClusterID: os.Getenv("CLUSTER_ID"), - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - AllowDecimalAsFloat: true, - NoTracking: false, - } - if Source.ClusterID != "" { - _ = ycdbaas.InitializeInternalCloud("https://gw.db.yandex-team.ru/", "https://gw.db.yandex-team.ru:443/iam/v1/", os.Getenv("DBAAS_TOKEN")) - Source.Host = "" - } - Source.WithDefaults() - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - )) - }() - - tableCase := func(tableName string) func(t *testing.T) { - - return func(t *testing.T) { - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - conn, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = conn.Exec(fmt.Sprintf(`drop table %s`, tableName)) - require.NoError(t, err) - execBatch(t, conn, TableSQLs[tableName]) - - counterStorage, counterSinkFactory := validator.NewCounter() - transfer := helpers.MakeTransfer( - tableName, - Source, - &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(Source), - validator.InitDone(t), - validator.Canonizator(t), - validator.TypesystemChecker(mysql.ProviderType, func(colSchema abstract.ColSchema) string { - return mysql.ClearOriginalType(colSchema) - }, - ), - counterSinkFactory, - ), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotAndIncrement, - ) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{Source.Database + "." + tableName}} - worker := helpers.Activate(t, transfer) - - _, err = conn.Exec(fmt.Sprintf(`truncate table %s`, tableName)) - require.NoError(t, err) - counterStorage.Truncate(abstract.TableID{ - Namespace: Source.Database, - Name: tableName, - }) - execBatch(t, conn, TableSQLs[tableName]) - - srcStorage, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCount(t, Source.Database, tableName, srcStorage, counterStorage, time.Second*60)) - - defer worker.Close(t) - } - } - t.Run("initial", tableCase("initial")) - t.Run("date_types", tableCase("date_types")) - t.Run("json_types", tableCase("json_types")) - t.Run("numeric_types_bit", tableCase("numeric_types_bit")) - t.Run("numeric_types_boolean", tableCase("numeric_types_boolean")) - t.Run("numeric_types_decimal", tableCase("numeric_types_decimal")) - t.Run("numeric_types_float", tableCase("numeric_types_float")) - t.Run("numeric_types_int", tableCase("numeric_types_int")) - t.Run("string_types", tableCase("string_types")) - t.Run("string_types_emoji", tableCase("string_types_emoji")) - //t.Run("spatial_types", tableCase("spatial_types")) -} diff --git a/tests/canon/mysql/canondata/result.json b/tests/canon/mysql/canondata/result.json deleted file mode 100644 index 46b43f29..00000000 --- a/tests/canon/mysql/canondata/result.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "mysql.mysql.TestCanonSource/date_types/canon_0#01": { - "checksum": "63cdaf6fea8ae94d655a6725b6884f06", - "size": 375880, - "uri": "https://storage.yandex-team.ru/get-devtools/1937027/556f28ba793a36122a21906b5be32fd3b7c556e9/resource.tar.gz#mysql.mysql.TestCanonSource_date_types_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/date_types/canon_0#03": { - "checksum": "553cf9cb21651376814c8a54af60c25a", - "size": 375654, - "uri": "https://storage.yandex-team.ru/get-devtools/1937027/556f28ba793a36122a21906b5be32fd3b7c556e9/resource.tar.gz#mysql.mysql.TestCanonSource_date_types_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/initial/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_initial_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/initial/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_initial_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/json_types/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_json_types_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/json_types/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_json_types_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_bit/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_bit_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_bit/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_bit_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_boolean/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_boolean_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_boolean/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_boolean_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_decimal/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_decimal_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_decimal/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_decimal_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_float/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_float_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_float/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_float_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_int/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_int_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/numeric_types_int/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_numeric_types_int_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/string_types/canon_0#01": { - "checksum": "9625a825a89c306a345dd363347a79ba", - "size": 411782, - "uri": "https://storage.yandex-team.ru/get-devtools/1937027/556f28ba793a36122a21906b5be32fd3b7c556e9/resource.tar.gz#mysql.mysql.TestCanonSource_string_types_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/string_types/canon_0#03": { - "checksum": "13148de832c344a4760d8afdb07ee0db", - "size": 411702, - "uri": "https://storage.yandex-team.ru/get-devtools/1937027/556f28ba793a36122a21906b5be32fd3b7c556e9/resource.tar.gz#mysql.mysql.TestCanonSource_string_types_canon_0#03/extracted" - }, - "mysql.mysql.TestCanonSource/string_types_emoji/canon_0#01": { - "uri": "file://mysql.mysql.TestCanonSource_string_types_emoji_canon_0#01/extracted" - }, - "mysql.mysql.TestCanonSource/string_types_emoji/canon_0#03": { - "uri": "file://mysql.mysql.TestCanonSource_string_types_emoji_canon_0#03/extracted" - } -} diff --git a/tests/canon/mysql/dump/date_types.sql b/tests/canon/mysql/dump/date_types.sql deleted file mode 100644 index 59a0cc0b..00000000 --- a/tests/canon/mysql/dump/date_types.sql +++ /dev/null @@ -1,77 +0,0 @@ --- MySQL permits you to store a “zero” value of '0000-00-00' as a “dummy date.” - --- For DATETIME, DATE, and TIMESTAMP types, MySQL interprets dates specified with ambiguous year values using these rules: --- Year values in the range 00-69 become 2000-2069. --- Year values in the range 70-99 become 1970-1999. - -create table if not exists date_types -( - __primary_key INT PRIMARY KEY, - - -- - - t_date DATE, -- The supported range is '1000-01-01' to '9999-12-31' - - t_year YEAR, -- https://dev.mysql.com/doc/refman/8.0/en/year.html - t_year4 YEAR(4), -- it's alias to year - - t_timestamp TIMESTAMP DEFAULT '0000-00-00', - t_timestamp0 TIMESTAMP(0), - t_timestamp1 TIMESTAMP(1), - t_timestamp2 TIMESTAMP(2), - t_timestamp3 TIMESTAMP(3), - t_timestamp4 TIMESTAMP(4), - t_timestamp5 TIMESTAMP(5), - t_timestamp6 TIMESTAMP(6), - - t_time TIME, -- TIME values may range from '-38:59:59' to '38:59:59' - t_time0 TIME(0), - t_time1 TIME(1), - t_time2 TIME(2), - t_time3 TIME(3), - t_time4 TIME(4), - t_time5 TIME(5), - t_time6 TIME(6), - - t_datetime DATETIME, -- The supported range is '1000-01-01 00:00:00' to '9999-12-31 23:59:59'. - t_datetime0 DATETIME(0), - t_datetime1 DATETIME(1), - t_datetime2 DATETIME(2), - t_datetime3 DATETIME(3), - t_datetime4 DATETIME(4), - t_datetime5 DATETIME(5), - t_datetime6 DATETIME(6) -); - -INSERT INTO date_types (__primary_key, t_date) VALUES (10000, '1000-01-01'); -INSERT INTO date_types (__primary_key, t_date) VALUES (10001, '9999-12-31'); - -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11000, '1901', '1901'); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11001, '2155', '2155'); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11002, 1901, 1901); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11003, 1901, 1901); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11004, '0', '0'); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11005, '69', '69'); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11006, '70', '70'); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11007, '99', '99'); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11008, 0, 0); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11009, 69, 69); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11010, 70, 70); -INSERT INTO date_types (__primary_key, t_year, t_year4) VALUES (11011, 99, 99); - -INSERT INTO date_types (__primary_key, t_timestamp, t_timestamp0, t_timestamp1, t_timestamp2, t_timestamp3, t_timestamp4, t_timestamp5, t_timestamp6) VALUES (12000, - '1970-01-01 00:00:01','1970-01-01 00:00:01','1970-01-01 00:00:01.1','1970-01-01 00:00:01.12','1970-01-01 00:00:01.123','1970-01-01 00:00:01.1234','1970-01-01 00:00:01.12345','1970-01-01 00:00:01.123456'); -INSERT INTO date_types (__primary_key, t_timestamp, t_timestamp0, t_timestamp1, t_timestamp2, t_timestamp3, t_timestamp4, t_timestamp5, t_timestamp6) VALUES (12001, - '2038-01-19 03:14:07','2038-01-19 03:14:07','2038-01-19 03:14:07.1','2038-01-19 03:14:07.12','2038-01-19 03:14:07.123','2038-01-19 03:14:07.1234','2038-01-19 03:14:07.12345','2038-01-19 03:14:07.123456'); - -INSERT INTO date_types (__primary_key, t_time, t_time0, t_time1, t_time2, t_time3, t_time4, t_time5, t_time6) VALUES (13000, - '-38:59:59','-38:59:59','-38:59:59.1','-38:59:59.12','-38:59:59.123','-38:59:59.1234','-38:59:59.12345','-38:59:59.123456'); -INSERT INTO date_types (__primary_key, t_time, t_time0, t_time1, t_time2, t_time3, t_time4, t_time5, t_time6) VALUES (13001, - '38:59:59','38:59:59','38:59:59.1','38:59:59.12','38:59:59.123','38:59:59.1234','38:59:59.12345','38:59:59.123456'); -INSERT INTO date_types (__primary_key, t_time, t_time0, t_time1, t_time2, t_time3, t_time4, t_time5, t_time6) VALUES (13002, - '000:00:00','000:00:00','000:00:00.1','000:00:00.12','000:00:00.123','000:00:00.1234','000:00:00.12345','000:00:00.123456'); - -INSERT INTO date_types (__primary_key, t_datetime, t_datetime0, t_datetime1, t_datetime2, t_datetime3, t_datetime4, t_datetime5, t_datetime6) VALUES (14000, - '1000-01-01 00:00:00','1000-01-01 00:00:00','1000-01-01 00:00:00.1','1000-01-01 00:00:00.12','1000-01-01 00:00:00.123','1000-01-01 00:00:00.1234','1000-01-01 00:00:00.12345','1000-01-01 00:00:00.123456'); -INSERT INTO date_types (__primary_key, t_datetime, t_datetime0, t_datetime1, t_datetime2, t_datetime3, t_datetime4, t_datetime5, t_datetime6) VALUES (14001, - '9999-12-31 23:59:59','9999-12-31 23:59:59','9999-12-31 23:59:59.1','9999-12-31 23:59:59.12','9999-12-31 23:59:59.123','9999-12-31 23:59:59.1234','9999-12-31 23:59:59.12345','9999-12-31 23:59:59.123456'); diff --git a/tests/canon/mysql/dump/initial_data.sql b/tests/canon/mysql/dump/initial_data.sql deleted file mode 100644 index 9ed40e17..00000000 --- a/tests/canon/mysql/dump/initial_data.sql +++ /dev/null @@ -1,249 +0,0 @@ -CREATE TABLE IF NOT EXISTS initial -( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - -- float_10_2 FLOAT(10, 2), - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), - time1 TIME(1), - time2 TIME(2), - time3 TIME(3), - time4 TIME(4), - time5 TIME(5), - time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- SPATIAL TYPES - - -- LINESTRING_ GEOMETRY, - -- POLYGON_ GEOMETRY, - -- MULTIPOINT_ GEOMETRY, - -- MULTILINESTRING_ GEOMETRY, - -- MULTIPOLYGON_ GEOMETRY, - -- GEOMETRYCOLLECTION_ GEOMETRY, - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - - - - - -INSERT INTO initial (`pk`,`bool1`,`bool2`,`bit`,`bit16`,`tinyint_`,`tinyint_def`,`tinyint_u`,`tinyint1`,`tinyint1u`,`smallint_`,`smallint5`,`smallint_u`,`mediumint_`,`mediumint5`,`mediumint_u`,`int_`,`integer_`,`integer5`,`int_u`,`bigint_`,`bigint5`,`bigint_u`,`real_`,`real_10_2`,`float_`,`float_53`,`double_`,`double_precision`,`char_`,`char5`,`varchar5`,`binary_`,`binary5`,`varbinary5`,`tinyblob_`,`tinytext_`,`blob_`,`text_`,`mediumblob_`,`mediumtext_`,`longblob_`,`longtext_`,`json_`,`enum_`,`set_`,`year_`,`year4`,`timestamp_`,`timestamp0`,`timestamp1`,`timestamp2`,`timestamp3`,`timestamp4`,`timestamp5`,`timestamp6`,`date_`,`time_`,`time0`,`time1`,`time2`,`time3`,`time4`,`time5`,`time6`,`datetime_`,`datetime0`,`datetime1`,`datetime2`,`datetime3`,`datetime4`,`datetime5`,`datetime6`,`NUMERIC_`,`NUMERIC_5`,`NUMERIC_5_2`,`DECIMAL_`,`DECIMAL_5`,`DECIMAL_5_2`) -VALUES -( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - -- FLOAT(10, 2) - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - '04:05:06.12345', -- TIME(5) - '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/canon/mysql/dump/json_types.sql b/tests/canon/mysql/dump/json_types.sql deleted file mode 100644 index 84cdb031..00000000 --- a/tests/canon/mysql/dump/json_types.sql +++ /dev/null @@ -1,18 +0,0 @@ -create table if not exists json_types -( - __primary_key INT PRIMARY KEY, - - -- - - t_json JSON -); - -INSERT INTO json_types (__primary_key, t_json) VALUES (0, '{"a": "b"}'); -INSERT INTO json_types (__primary_key, t_json) VALUES (1, '["a", "b"]'); -INSERT INTO json_types (__primary_key, t_json) VALUES (2, '"blablabla"'); -INSERT INTO json_types (__primary_key, t_json) VALUES (3, '1'); -INSERT INTO json_types (__primary_key, t_json) VALUES (4, '{}'); -INSERT INTO json_types (__primary_key, t_json) VALUES (5, '[]'); - --- null case -INSERT INTO json_types (__primary_key) VALUES (801640048); diff --git a/tests/canon/mysql/dump/numeric_types.sql b/tests/canon/mysql/dump/numeric_types.sql deleted file mode 100644 index 73afaf7f..00000000 --- a/tests/canon/mysql/dump/numeric_types.sql +++ /dev/null @@ -1,114 +0,0 @@ -create table if not exists numeric_types -( - __primary_key INT PRIMARY KEY, - - -- bool - - t_boolean BOOLEAN, - t_bool BOOL, - - -- bit - -- https://dev.mysql.com/doc/refman/8.0/en/bit-type.html - -- A type of BIT(M) enables storage of M-bit values. M can range from 1 to 64. - - t_bit BIT, - t_bit1 BIT(1), - t_bit16 BIT(16), - t_bit64 BIT(64), - - -- int - -- limits: https://dev.mysql.com/doc/refman/8.0/en/integer-types.html - - t_tinyint TINYINT, - t_tinyint_default0 TINYINT DEFAULT 0, - t_tinyint_u TINYINT UNSIGNED, - - t_tinyint1 TINYINT(1), - t_tinyint1u TINYINT(1) UNSIGNED, - - t_smallint SMALLINT, - t_smallint5 SMALLINT(5), - t_smallint_u SMALLINT UNSIGNED, - - t_mediumint MEDIUMINT, - t_mediumint5 MEDIUMINT(5), - t_mediumint_u MEDIUMINT UNSIGNED, - - t_int INT, - t_integer INTEGER, - t_integer5 INTEGER(5), -- doc: For example, INT(4) specifies an INT with a display width of four digits - t_int_u INT UNSIGNED, - - t_bigint BIGINT, - t_bigint5 BIGINT(5), - t_bigint_u BIGINT UNSIGNED, - - -- float - -- no limits in doc: https://dev.mysql.com/doc/refman/8.0/en/floating-point-types.html - - t_real REAL, - t_real_10_2 REAL(10, 2), - - t_float FLOAT, - t_float_53 FLOAT(53), -- doc: A precision from 0 to 23 results in a 4-byte single-precision FLOAT column. A precision from 24 to 53 results in an 8-byte double-precision DOUBLE column. - - t_double DOUBLE, - t_double_precision DOUBLE PRECISION, - - -- numeric - -- https://dev.mysql.com/doc/refman/8.0/en/fixed-point-types.html - - t_decimal DECIMAL, - t_decimal_5 DECIMAL(5), - t_decimal_5_2 DECIMAL(5,2), - t_decimal_u DECIMAL UNSIGNED, - - t_dec DEC, - t_fixed FIXED, - t_numeric NUMERIC, - - t_decimal_65 DECIMAL(65) -); - -INSERT INTO numeric_types (__primary_key, t_boolean, t_bool) VALUES (10000, false, false); -INSERT INTO numeric_types (__primary_key, t_boolean, t_bool) VALUES (10001, true, true); - - -INSERT INTO numeric_types (__primary_key, t_bit, t_bit1, t_bit16, t_bit64) VALUES (11000, b'0', b'0', b'0000000000000000', b'0000000000000000000000000000000000000000000000000000000000000000'); -INSERT INTO numeric_types (__primary_key, t_bit, t_bit1, t_bit16, t_bit64) VALUES (11001, b'1', b'1', b'1111111111111111', b'1111111111111111111111111111111111111111111111111111111111111111'); - - -INSERT INTO numeric_types (__primary_key, t_tinyint,t_tinyint_default0,t_tinyint_u,t_tinyint1,t_tinyint1u,t_smallint,t_smallint5,t_smallint_u,t_mediumint,t_mediumint5,t_mediumint_u,t_int, t_integer, t_integer5,t_int_u,t_bigint, t_bigint5,t_bigint_u) VALUES (12000, - -128, -128, 0, -1, 0, -32768, -32768, 0, -8388608, -99999, 0, -2147483648, -2147483648,-99999, 0, -9223372036854775807, -99999, 0); -INSERT INTO numeric_types (__primary_key, t_tinyint,t_tinyint_default0,t_tinyint_u,t_tinyint1,t_tinyint1u,t_smallint,t_smallint5,t_smallint_u,t_mediumint,t_mediumint5,t_mediumint_u,t_int, t_integer, t_integer5,t_int_u,t_bigint, t_bigint5,t_bigint_u) VALUES (12001, - 127, 127, 256, 1, 1, 32767, 32767, 65535, 8388607, 99999, 16777215, 2147483647, 2147483647, 99999, 0, -9223372036854775807, -99999, 18446744073709551615); - - -INSERT INTO numeric_types (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14000, - 1.45e-45,12345678.90, 3.14e-324,-12345678901234567890123456789012345678901234567890123, 3.14e-324, 3.14e-324); -INSERT INTO numeric_types (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14001, - 1.175494351E-38, 99999999.99, -1.7976931348623157E+308, 99999999999999999999999999999999999999999999999999999, -1.7976931348623157E+308, -1.7976931348623157E+308); -INSERT INTO numeric_types (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14002, - 123456.123456789012345, 123456.123456789012345, 123456.123456789012345, 123456.123456789012345, 123456.123456789012345, 123456.123456789012345); -INSERT INTO numeric_types (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14003, - 1.7976931348623157E+308, 00000000.00, 3.4E+38, 3.4E+38, 1.7976931348623157E+308, 1.7976931348623157E+308); -INSERT INTO numeric_types (__primary_key, t_float) VALUES (13005, -340282346638528859811704183484516925440.0000000000000000); -INSERT INTO numeric_types (__primary_key, t_float) VALUES (13006, 340282346638528859811704183484516925440.0000000000000000); -INSERT INTO numeric_types (__primary_key, t_double) VALUES (13007, -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.0000000000000000); -INSERT INTO numeric_types (__primary_key, t_double) VALUES (13008, 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.0000000000000000); - - -INSERT INTO numeric_types (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13000, - 1234567890,12345, 123.45, 1234567890, 1234567890,1234567890,1234567890,12345678901234567890123456789012345678901234567890123456789012345); -INSERT INTO numeric_types (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13001, - -1234567890,-12345, -123.45, 0, -1234567890,-1234567890,-1234567890,-12345678901234567890123456789012345678901234567890123456789012345); -INSERT INTO numeric_types (__primary_key,t_decimal,t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec,t_fixed,t_numeric, t_decimal_65) VALUES (13002, - 0, 0, 0, 0, 0, 0, 0, 0); -INSERT INTO numeric_types (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13003, - 9999999999,99999, 999.99, 99999999999, 999999999,9999999999,9999999999,99999999999999999999999999999999999999999999999999999999999999999); -INSERT INTO numeric_types (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13004, - -9999999999,-99999, -999.99, 0, -9999999999,-9999999999,-9999999999,-99999999999999999999999999999999999999999999999999999999999999999); - - --- null case -INSERT INTO numeric_types (__primary_key, t_tinyint_default0) VALUES (801640048, NULL); diff --git a/tests/canon/mysql/dump/numeric_types_bit.sql b/tests/canon/mysql/dump/numeric_types_bit.sql deleted file mode 100644 index 632f42d0..00000000 --- a/tests/canon/mysql/dump/numeric_types_bit.sql +++ /dev/null @@ -1,20 +0,0 @@ -create table if not exists numeric_types_bit -( - __primary_key INT PRIMARY KEY, - - -- bit - -- https://dev.mysql.com/doc/refman/8.0/en/bit-type.html - -- A type of BIT(M) enables storage of M-bit values. M can range from 1 to 64. - - t_bit BIT, - t_bit1 BIT(1), - t_bit16 BIT(16), - t_bit64 BIT(64) -); - - -INSERT INTO numeric_types_bit (__primary_key, t_bit, t_bit1, t_bit16, t_bit64) VALUES (11000, b'0', b'0', b'0000000000000000', b'0000000000000000000000000000000000000000000000000000000000000000'); -INSERT INTO numeric_types_bit (__primary_key, t_bit, t_bit1, t_bit16, t_bit64) VALUES (11001, b'1', b'1', b'1111111111111111', b'1111111111111111111111111111111111111111111111111111111111111111'); - --- null case -INSERT INTO numeric_types_bit (__primary_key) VALUES (801640048); diff --git a/tests/canon/mysql/dump/numeric_types_boolean.sql b/tests/canon/mysql/dump/numeric_types_boolean.sql deleted file mode 100644 index e2c62bf0..00000000 --- a/tests/canon/mysql/dump/numeric_types_boolean.sql +++ /dev/null @@ -1,16 +0,0 @@ -create table if not exists numeric_types_boolean -( - __primary_key INT PRIMARY KEY, - - -- bool - - t_boolean BOOLEAN, - t_bool BOOL -); - - -INSERT INTO numeric_types_boolean (__primary_key, t_boolean, t_bool) VALUES (10000, false, false); -INSERT INTO numeric_types_boolean (__primary_key, t_boolean, t_bool) VALUES (10001, true, true); - --- null case -INSERT INTO numeric_types_boolean (__primary_key) VALUES (801640048); diff --git a/tests/canon/mysql/dump/numeric_types_decimal.sql b/tests/canon/mysql/dump/numeric_types_decimal.sql deleted file mode 100644 index 82c4ef6b..00000000 --- a/tests/canon/mysql/dump/numeric_types_decimal.sql +++ /dev/null @@ -1,34 +0,0 @@ -create table if not exists numeric_types_decimal -( - __primary_key INT PRIMARY KEY, - - -- numeric - -- https://dev.mysql.com/doc/refman/8.0/en/fixed-point-types.html - - t_decimal DECIMAL, - t_decimal_5 DECIMAL(5), - t_decimal_5_2 DECIMAL(5,2), - t_decimal_u DECIMAL UNSIGNED, - - t_dec DEC, - t_fixed FIXED, - t_numeric NUMERIC, - - t_decimal_65 DECIMAL(65) -); - - -INSERT INTO numeric_types_decimal (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13000, - 1234567890,12345, 123.45, 1234567890, 1234567890,1234567890,1234567890,12345678901234567890123456789012345678901234567890123456789012345); -INSERT INTO numeric_types_decimal (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13001, - -1234567890,-12345, -123.45, 0, -1234567890,-1234567890,-1234567890,-12345678901234567890123456789012345678901234567890123456789012345); -INSERT INTO numeric_types_decimal (__primary_key,t_decimal,t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec,t_fixed,t_numeric, t_decimal_65) VALUES (13002, - 0, 0, 0, 0, 0, 0, 0, 0); -INSERT INTO numeric_types_decimal (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13003, - 9999999999,99999, 999.99, 99999999999, 999999999,9999999999,9999999999,99999999999999999999999999999999999999999999999999999999999999999); -INSERT INTO numeric_types_decimal (__primary_key,t_decimal, t_decimal_5,t_decimal_5_2,t_decimal_u,t_dec, t_fixed, t_numeric, t_decimal_65) VALUES (13004, - -9999999999,-99999, -999.99, 0, -9999999999,-9999999999,-9999999999,-99999999999999999999999999999999999999999999999999999999999999999); - - --- null case -INSERT INTO numeric_types_decimal (__primary_key) VALUES (801640048); diff --git a/tests/canon/mysql/dump/numeric_types_float.sql b/tests/canon/mysql/dump/numeric_types_float.sql deleted file mode 100644 index 36b5cb7e..00000000 --- a/tests/canon/mysql/dump/numeric_types_float.sql +++ /dev/null @@ -1,34 +0,0 @@ -create table if not exists numeric_types_float -( - __primary_key INT PRIMARY KEY, - - -- float - -- no limits in doc: https://dev.mysql.com/doc/refman/8.0/en/floating-point-types.html - - t_real REAL, - t_real_10_2 REAL(10, 2), - - t_float FLOAT, - t_float_53 FLOAT(53), -- doc: A precision from 0 to 23 results in a 4-byte single-precision FLOAT column. A precision from 24 to 53 results in an 8-byte double-precision DOUBLE column. - - t_double DOUBLE, - t_double_precision DOUBLE PRECISION -); - - -INSERT INTO numeric_types_float (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14000, - 1.45e-45,12345678.90, 3.14e-324,-12345678901234567890123456789012345678901234567890123, 3.14e-324, 3.14e-324); -INSERT INTO numeric_types_float (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14001, - 1.175494351E-38, 99999999.99, -1.7976931348623157E+308, 99999999999999999999999999999999999999999999999999999, -1.7976931348623157E+308, -1.7976931348623157E+308); -INSERT INTO numeric_types_float (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14002, - 123456.123456789012345, 123456.123456789012345, 123456.123456789012345, 123456.123456789012345, 123456.123456789012345, 123456.123456789012345); -INSERT INTO numeric_types_float (__primary_key, t_real, t_real_10_2, t_float, t_float_53, t_double, t_double_precision) VALUES (14003, - 1.7976931348623157E+308, 00000000.00, 3.4E+38, 3.4E+38, 1.7976931348623157E+308, 1.7976931348623157E+308); -INSERT INTO numeric_types_float (__primary_key, t_float) VALUES (13005, -340282346638528859811704183484516925440.0000000000000000); -INSERT INTO numeric_types_float (__primary_key, t_float) VALUES (13006, 340282346638528859811704183484516925440.0000000000000000); -INSERT INTO numeric_types_float (__primary_key, t_double) VALUES (13007, -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.0000000000000000); -INSERT INTO numeric_types_float (__primary_key, t_double) VALUES (13008, 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.0000000000000000); - - --- null case -INSERT INTO numeric_types_float (__primary_key) VALUES (801640048); diff --git a/tests/canon/mysql/dump/numeric_types_int.sql b/tests/canon/mysql/dump/numeric_types_int.sql deleted file mode 100644 index fb0fefe0..00000000 --- a/tests/canon/mysql/dump/numeric_types_int.sql +++ /dev/null @@ -1,41 +0,0 @@ -create table if not exists numeric_types_int -( - __primary_key INT PRIMARY KEY, - - -- int - -- limits: https://dev.mysql.com/doc/refman/8.0/en/integer-types.html - - t_tinyint TINYINT, - t_tinyint_default0 TINYINT DEFAULT 0, - t_tinyint_u TINYINT UNSIGNED, - - t_tinyint1 TINYINT(1), - t_tinyint1u TINYINT(1) UNSIGNED, - - t_smallint SMALLINT, - t_smallint5 SMALLINT(5), - t_smallint_u SMALLINT UNSIGNED, - - t_mediumint MEDIUMINT, - t_mediumint5 MEDIUMINT(5), - t_mediumint_u MEDIUMINT UNSIGNED, - - t_int INT, - t_integer INTEGER, - t_integer5 INTEGER(5), -- doc: For example, INT(4) specifies an INT with a display width of four digits - t_int_u INT UNSIGNED, - - t_bigint BIGINT, - t_bigint5 BIGINT(5), - t_bigint_u BIGINT UNSIGNED -); - - -INSERT INTO numeric_types_int (__primary_key, t_tinyint,t_tinyint_default0,t_tinyint_u,t_tinyint1,t_tinyint1u,t_smallint,t_smallint5,t_smallint_u,t_mediumint,t_mediumint5,t_mediumint_u,t_int, t_integer, t_integer5,t_int_u,t_bigint, t_bigint5,t_bigint_u) VALUES (12000, - -128, -128, 0, -1, 0, -32768, -32768, 0, -8388608, -99999, 0, -2147483648, -2147483648,-99999, 0, -9223372036854775807, -99999, 0); -INSERT INTO numeric_types_int (__primary_key, t_tinyint,t_tinyint_default0,t_tinyint_u,t_tinyint1,t_tinyint1u,t_smallint,t_smallint5,t_smallint_u,t_mediumint,t_mediumint5,t_mediumint_u,t_int, t_integer, t_integer5,t_int_u,t_bigint, t_bigint5,t_bigint_u) VALUES (12001, - 127, 127, 256, 1, 1, 32767, 32767, 65535, 8388607, 99999, 16777215, 2147483647, 2147483647, 99999, 0, -9223372036854775807, -99999, 18446744073709551615); - - --- null case -INSERT INTO numeric_types_int (__primary_key, t_tinyint_default0) VALUES (801640048, NULL); diff --git a/tests/canon/mysql/dump/spatial_types.sql b/tests/canon/mysql/dump/spatial_types.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/canon/mysql/dump/string_types.sql b/tests/canon/mysql/dump/string_types.sql deleted file mode 100644 index 5920b370..00000000 --- a/tests/canon/mysql/dump/string_types.sql +++ /dev/null @@ -1,80 +0,0 @@ -create table if not exists string_types -( - __primary_key INT PRIMARY KEY, - - -- - - t_char CHAR, - t_char5 CHAR(5), - - t_varchar5 VARCHAR(5), - - t_binary BINARY, - t_binary5 BINARY(5), - - t_varbinary5 VARBINARY(5), - - t_tinyblob TINYBLOB, - t_tinytext TINYTEXT, - t_blob BLOB, - t_text TEXT, - t_mediumblob MEDIUMBLOB, - t_mediumtext MEDIUMTEXT, - t_longblob LONGBLOB, - t_longtext LONGTEXT, - t_enum ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - t_set SET('a', 'b', 'c', 'd') -); - -INSERT INTO string_types (__primary_key,t_char) VALUES (10000,'a'); - -INSERT INTO string_types (__primary_key,t_char5) VALUES (11000,'a'); -INSERT INTO string_types (__primary_key,t_char5) VALUES (11001,'abcde'); - -INSERT INTO string_types (__primary_key,t_varchar5) VALUES (12000,'a'); -INSERT INTO string_types (__primary_key,t_varchar5) VALUES (12001,'abcde'); - -INSERT INTO string_types (__primary_key,t_binary) VALUES (13000,X'9f'); -INSERT INTO string_types (__primary_key,t_binary) VALUES (13001,X'fd'); - -INSERT INTO string_types (__primary_key,t_binary5) VALUES (14000,X'9f'); -INSERT INTO string_types (__primary_key,t_binary5) VALUES (14001,X'fd'); -INSERT INTO string_types (__primary_key,t_binary5) VALUES (14002,X'9f9f9f9f9f'); -INSERT INTO string_types (__primary_key,t_binary5) VALUES (14003,X'fdfdfdfdfd'); - -INSERT INTO string_types (__primary_key,t_varbinary5) VALUES (15000,X'9f'); -INSERT INTO string_types (__primary_key,t_varbinary5) VALUES (15001,X'fd'); -INSERT INTO string_types (__primary_key,t_varbinary5) VALUES (15002,X'9f9f9f9f9f'); -INSERT INTO string_types (__primary_key,t_varbinary5) VALUES (15003,X'fdfdfdfdfd'); - -INSERT INTO string_types (__primary_key,t_tinyblob) VALUES (16000,X'9f'); -INSERT INTO string_types (__primary_key,t_tinyblob) VALUES (16001,X'fd'); -INSERT INTO string_types (__primary_key,t_tinytext) VALUES (16002,'blablabla'); - -INSERT INTO string_types (__primary_key,t_blob) VALUES (17000,X'9f'); -INSERT INTO string_types (__primary_key,t_blob) VALUES (17001,X'fd'); -INSERT INTO string_types (__primary_key,t_blob) VALUES (17002,X'9f9f9f9f9f'); -INSERT INTO string_types (__primary_key,t_blob) VALUES (17003,X'fdfdfdfdfd'); - -INSERT INTO string_types (__primary_key,t_text) VALUES (18000,'blablabla'); - -INSERT INTO string_types (__primary_key,t_mediumblob) VALUES (19000,X'9f'); -INSERT INTO string_types (__primary_key,t_mediumblob) VALUES (19001,X'fd'); -INSERT INTO string_types (__primary_key,t_mediumblob) VALUES (19002,X'9f9f9f9f9f'); -INSERT INTO string_types (__primary_key,t_mediumblob) VALUES (19003,X'fdfdfdfdfd'); - -INSERT INTO string_types (__primary_key,t_mediumtext) VALUES (20000,'blablabla'); - -INSERT INTO string_types (__primary_key,t_longblob) VALUES (21000,X'9f'); -INSERT INTO string_types (__primary_key,t_longblob) VALUES (21001,X'fd'); -INSERT INTO string_types (__primary_key,t_longblob) VALUES (21002,X'9f9f9f9f9f'); -INSERT INTO string_types (__primary_key,t_longblob) VALUES (21003,X'fdfdfdfdfd'); - -INSERT INTO string_types (__primary_key,t_longtext) VALUES (22000,'blablabla'); - -INSERT INTO string_types (__primary_key,t_enum) VALUES (23000,'x-large'); - -INSERT INTO string_types (__primary_key,t_set) VALUES (24000,'c'); - --- null case -INSERT INTO string_types (__primary_key) VALUES (801640048); diff --git a/tests/canon/mysql/dump/string_types_emoji.sql b/tests/canon/mysql/dump/string_types_emoji.sql deleted file mode 100644 index e5ab099f..00000000 --- a/tests/canon/mysql/dump/string_types_emoji.sql +++ /dev/null @@ -1,15 +0,0 @@ -create table if not exists string_types_emoji -( - __primary_key INT PRIMARY KEY, - - -- - - t_char CHAR -); - -ALTER TABLE string_types_emoji CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; - -INSERT INTO string_types_emoji (__primary_key,t_char) VALUES (10000,'❌'); - --- null case -INSERT INTO string_types_emoji (__primary_key) VALUES (801640048); diff --git a/tests/canon/parser/README.md b/tests/canon/parser/README.md deleted file mode 100644 index 0af05142..00000000 --- a/tests/canon/parser/README.md +++ /dev/null @@ -1,30 +0,0 @@ -### Canon for parsers - -For each *TEST_CASE* we will create a single message parse with followed config: -```golang - msg := persqueue.ReadMessage{ - Offset: 123, - SeqNo: 32, - SourceID: []byte("test_source_id"), - CreateTime: time.Now(), - WriteTime: time.Now(), - IP: "192.168.1.1", - Data: `TEST_CASE.data`, - ExtraFields: map[string]string{"some_field": "test"}, - } -``` - -#### Static definition - -Each parser sample defined in 2 files: - -```shell -{TEST_CASE}.config.json -{TEST_CASE}.data -``` - - -#### Dynamic definition - -Register your TestCase in `LoadDynamicTestCases` function in `canon_dynamic_test.go` function. -Handful in case when you need dynamically specify message while test runs (e.g. proto message tests). diff --git a/tests/canon/parser/canon_static_generic_test.go b/tests/canon/parser/canon_static_generic_test.go deleted file mode 100644 index f6f174a0..00000000 --- a/tests/canon/parser/canon_static_generic_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package parser - -import ( - "embed" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/internal/metrics" - "github.com/doublecloud/tross/pkg/abstract" - parsersfactory "github.com/doublecloud/tross/pkg/parsers" - "github.com/doublecloud/tross/pkg/stats" - "github.com/doublecloud/tross/tests/canon/parser/testcase" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/stretchr/testify/require" -) - -//go:embed samples/static/generic/* -var TestGenericSamples embed.FS - -func TestGenericParsers(t *testing.T) { - cases := testcase.LoadStaticTestCases(t, TestGenericSamples) - - for tc := range cases { - t.Run(tc, func(t *testing.T) { - currCase := cases[tc] - parser, err := parsersfactory.NewParserFromParserConfig(currCase.ParserConfig, false, logger.Log, stats.NewSourceStats(metrics.NewRegistry().WithTags(map[string]string{ - "id": "TestParser_Do", - }))) - require.NoError(t, err) - require.NotNil(t, parser) - res := parser.Do(currCase.Data, abstract.Partition{Topic: currCase.TopicName}) - require.NotNil(t, res) - sink := validator.New( - false, - validator.ValuesTypeChecker, - validator.Canonizator(t), - )() - require.NoError(t, sink.Push(res)) - require.NoError(t, sink.Close()) - }) - } -} diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted deleted file mode 100644 index 87fe8298..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted +++ /dev/null @@ -1,439 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "doubleField", - "floatField", - "int32Field", - "int64Field", - "uint32Field", - "uint64Field", - "sint32Field", - "sint64Field", - "fixed32Field", - "fixed64Field", - "sfixed32Field", - "sfixed64Field", - "boolField", - "stringField", - "bytesField", - "mapField", - "repeatedField", - "msgField" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "float64", - "value": 3.14 - }, - { - "type": "float32", - "value": 1.46 - }, - { - "type": "int32", - "value": -228 - }, - { - "type": "int64", - "value": -1337 - }, - { - "type": "uint32", - "value": 42 - }, - { - "type": "uint64", - "value": 300000000000 - }, - { - "type": "int32", - "value": -1 - }, - { - "type": "int64", - "value": -1488 - }, - { - "type": "uint32", - "value": 9001 - }, - { - "type": "uint64", - "value": 127 - }, - { - "type": "int32", - "value": -5 - }, - { - "type": "int64", - "value": -273 - }, - { - "type": "bool", - "value": true - }, - { - "type": "string", - "value": "Я знаю, что ты знаешь этот трек, готовься подпевать" - }, - { - "type": "[]uint8", - "value": "0KDQsNC3LCDQtNCy0LAsINGC0YDQuCwg0YfQtdGC0YvRgNC1" - }, - { - "type": "map[string]interface {}", - "value": { - "БУДУ С ТОБОЮ, САМОЙ ПРИМЕРНОЮ": { - "type": "int32", - "value": 2 - }, - "ДЕЛАЙ ЧТО ХОЧЕШЬ, Я ЗАБЫВАЮСЬ": { - "type": "int32", - "value": 1 - }, - "МУЗЫКА ГРОМЧЕ, ГЛАЗА ЗАКРЫТЫ": { - "type": "int32", - "value": 1 - }, - "МУЗЫКА ГРОМЧЕ, ГЛАЗА ЗАКРЫТЫ (хэй)": { - "type": "int32", - "value": 2 - }, - "УТРО В ОКНЕ И МЫ БУДЕМ ПЕРВЫМИ": { - "type": "int32", - "value": 2 - }, - "ЭТО НОНСТОООП, НОЧЬЮ ОТКРЫТИЙ": { - "type": "int32", - "value": 2 - }, - "ЭТО НОНСТОП, НЕ ПРЕКРАЩАЯСЬ": { - "type": "int32", - "value": 1 - }, - "ЭТО НОНСТОП, НОЧЬЮ ОТКРЫТИЙ": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "Этот трек делает тебя сильней" - }, - { - "type": "string", - "value": "Он прикольней, чем колёса, и роднее, чем портвейн" - }, - { - "type": "string", - "value": "Раза в три круче, чем самый первый секс-партнёр" - }, - { - "type": "string", - "value": "Все девчата в таком трипе, что аж «мама не горюй», йоп" - } - ] - }, - { - "type": "map[string]interface {}", - "value": { - "enumField": { - "type": "int32", - "value": 1 - }, - "int32Field": { - "type": "int32", - "value": 256 - }, - "stringField": { - "type": "string", - "value": "Прыгай в такт, прыгай в такт, будто ты совсем дурак" - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 1131, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "doubleField", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "floatField", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32Field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int64Field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "uint32Field", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "uint64Field", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "sint32Field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "sint64Field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "fixed32Field", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "fixed64Field", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "sfixed32Field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "sfixed64Field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "boolField", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "stringField", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bytesField", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "mapField", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "repeatedField", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "msgField", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_json_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_json_canon_0/extracted deleted file mode 100644 index 014b27da..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_json_canon_0/extracted +++ /dev/null @@ -1,1022 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "version", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25160 - }, - { - "type": "uint64", - "value": 89488198116272410 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 180, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "version", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "version", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25162 - }, - { - "type": "uint64", - "value": 89488198116272411 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 180, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "version", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "version", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25161 - }, - { - "type": "uint64", - "value": 89488198116272412 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 3 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 180, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "version", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "version", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25163 - }, - { - "type": "uint64", - "value": 89488198116272413 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 4 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 180, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "version", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_mdb_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_mdb_canon_0/extracted deleted file mode 100644 index d874d7d2..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_mdb_canon_0/extracted +++ /dev/null @@ -1,563 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "collect_time", - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "user", - "application_name", - "client_addr", - "client_hostname", - "client_port", - "backend_start", - "xact_start", - "query_start", - "state_change", - "wait_event_type", - "wait_event", - "state", - "backend_xid", - "backend_xmin", - "query", - "backend_type", - "blocking_pids", - "queryid", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "time.Time", - "value": "2020-08-10T09:46:22Z" - }, - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25167 - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "string", - "value": "" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "uint32", - "value": 0 - }, - { - "type": "time.Time", - "value": "2020-08-10T03:46:22.330498Z" - }, - { - "type": "nil", - "value": null - }, - { - "type": "time.Time", - "value": "2020-08-10T03:46:22.333591Z" - }, - { - "type": "time.Time", - "value": "2020-08-10T03:46:22.333623Z" - }, - { - "type": "string", - "value": "Client" - }, - { - "type": "string", - "value": "ClientRead" - }, - { - "type": "string", - "value": "idle" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "SELECT 42;" - }, - { - "type": "string", - "value": "client backend" - }, - { - "type": "string", - "value": "[]" - }, - { - "type": "string", - "value": "651361249" - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"mdb/porto/test/perf_diag/pg_stat_activity\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"mdb/porto/test/perf_diag/pg_stat_activity\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 682, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "mdb_porto_test_perf_diag_pg_stat_activity" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "collect_time", - "type": "datetime", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "user", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "application_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "client_addr", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "client_hostname", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "client_port", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "backend_start", - "type": "datetime", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "xact_start", - "type": "datetime", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "query_start", - "type": "datetime", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "state_change", - "type": "datetime", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "wait_event_type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "wait_event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "state", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "backend_xid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "backend_xmin", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "query", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "backend_type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "blocking_pids", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "queryid", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_canon_0/extracted deleted file mode 100644 index 58ccdb66..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_canon_0/extracted +++ /dev/null @@ -1,1430 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "AppVersionName", - "LogInfo", - "DeviceID", - "APIKey", - "AppPlatform", - "AppBuildNumber", - "ReceiveTimestamp", - "Latitude", - "Longitude", - "Age", - "Sex", - "EventID", - "EventName", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "397" - }, - { - "type": "string", - "value": "{\"advertiserId\": \"None\", \"campaignId\": \"10343\", \"product\": \"billboard\"}" - }, - { - "type": "string", - "value": "060D7218-49D2-4344-A42E-FDDB0C878251" - }, - { - "type": "uint32", - "value": 30488 - }, - { - "type": "string", - "value": "iOS" - }, - { - "type": "uint32", - "value": 402 - }, - { - "type": "uint64", - "value": 1566962865 - }, - { - "type": "float64", - "value": 56.823172635591256 - }, - { - "type": "float64", - "value": 60.4962324663226 - }, - { - "type": "uint8", - "value": 35 - }, - { - "type": "uint8", - "value": 1 - }, - { - "type": "uint64", - "value": 14009544124325608446 - }, - { - "type": "string", - "value": "geoadv.bb.pin.show" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 4091, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "maps-adv-statistics_raw_metrika_log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppVersionName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue.LogInfo", - "name": "LogInfo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "DeviceID", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "APIKey", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppPlatform", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppBuildNumber", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ReceiveTimestamp", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Latitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Longitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Age", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Sex", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventID", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "AppVersionName", - "LogInfo", - "DeviceID", - "APIKey", - "AppPlatform", - "AppBuildNumber", - "ReceiveTimestamp", - "Latitude", - "Longitude", - "Age", - "Sex", - "EventID", - "EventName", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "397" - }, - { - "type": "string", - "value": "{\"advertiserId\": \"None\", \"campaignId\": \"10343\", \"product\": \"billboard\"}" - }, - { - "type": "string", - "value": "060D7218-49D2-4344-A42E-FDDB0C878251" - }, - { - "type": "uint32", - "value": 30488 - }, - { - "type": "string", - "value": "iOS" - }, - { - "type": "uint32", - "value": 402 - }, - { - "type": "uint64", - "value": 1566962865 - }, - { - "type": "float64", - "value": 56.823172635591256 - }, - { - "type": "float64", - "value": 60.4962324663226 - }, - { - "type": "uint8", - "value": 35 - }, - { - "type": "uint8", - "value": 1 - }, - { - "type": "uint64", - "value": 14009544124325608446 - }, - { - "type": "string", - "value": "geoadv.bb.pin.show" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 4091, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "maps-adv-statistics_raw_metrika_log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppVersionName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue.LogInfo", - "name": "LogInfo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "DeviceID", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "APIKey", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppPlatform", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppBuildNumber", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ReceiveTimestamp", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Latitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Longitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Age", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Sex", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventID", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "AppVersionName", - "LogInfo", - "DeviceID", - "APIKey", - "AppPlatform", - "AppBuildNumber", - "ReceiveTimestamp", - "Latitude", - "Longitude", - "Age", - "Sex", - "EventID", - "EventName", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "397" - }, - { - "type": "string", - "value": "{\"advertiserId\": \"None\", \"campaignId\": \"10343\", \"product\": \"billboard\"}" - }, - { - "type": "string", - "value": "060D7218-49D2-4344-A42E-FDDB0C878251" - }, - { - "type": "uint32", - "value": 30488 - }, - { - "type": "string", - "value": "iOS" - }, - { - "type": "uint32", - "value": 402 - }, - { - "type": "uint64", - "value": 1566962865 - }, - { - "type": "float64", - "value": 56.823172635591256 - }, - { - "type": "float64", - "value": 60.4962324663226 - }, - { - "type": "uint8", - "value": 35 - }, - { - "type": "uint8", - "value": 1 - }, - { - "type": "uint64", - "value": 14009544124325608446 - }, - { - "type": "string", - "value": "geoadv.bb.pin.show" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 3 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 4091, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "maps-adv-statistics_raw_metrika_log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppVersionName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue.LogInfo", - "name": "LogInfo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "DeviceID", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "APIKey", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppPlatform", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppBuildNumber", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ReceiveTimestamp", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Latitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Longitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Age", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Sex", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventID", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "AppVersionName", - "LogInfo", - "DeviceID", - "APIKey", - "AppPlatform", - "AppBuildNumber", - "ReceiveTimestamp", - "Latitude", - "Longitude", - "Age", - "Sex", - "EventID", - "EventName", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "397" - }, - { - "type": "string", - "value": "{\"advertiserId\": \"None\", \"campaignId\": \"10343\", \"product\": \"billboard\"}" - }, - { - "type": "string", - "value": "060D7218-49D2-4344-A42E-FDDB0C878251" - }, - { - "type": "uint32", - "value": 30488 - }, - { - "type": "string", - "value": "iOS" - }, - { - "type": "uint32", - "value": 402 - }, - { - "type": "uint64", - "value": 1566962865 - }, - { - "type": "float64", - "value": 56.823172635591256 - }, - { - "type": "float64", - "value": 60.4962324663226 - }, - { - "type": "uint8", - "value": 35 - }, - { - "type": "uint8", - "value": 1 - }, - { - "type": "uint64", - "value": 14009544124325608446 - }, - { - "type": "string", - "value": "geoadv.bb.pin.show" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 4 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 4091, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "maps-adv-statistics_raw_metrika_log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppVersionName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue.LogInfo", - "name": "LogInfo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "DeviceID", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "APIKey", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppPlatform", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppBuildNumber", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ReceiveTimestamp", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Latitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Longitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Age", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Sex", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventID", - "type": "uint64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventName", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted deleted file mode 100644 index 480dc9a0..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted +++ /dev/null @@ -1,308 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "AppVersionName", - "CampaignID", - "EventGroupId", - "Latitude", - "Longitude", - "DeviceID", - "APIKey", - "AppPlatform", - "AppBuildNumber", - "EventName", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "371" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "4BCED9E0-E10B-4E72-83C7-42F80883CF3A" - }, - { - "type": "uint32", - "value": 30488 - }, - { - "type": "string", - "value": "iOS" - }, - { - "type": "uint32", - "value": 360 - }, - { - "type": "string", - "value": "geoadv.bb.pin.show" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"maps-adv-statistics/raw_metrika_log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 3779, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "maps-adv-statistics_raw_metrika_log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppVersionName", - "type": "string", - "key": false, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue.LogInfo.campaignId", - "name": "CampaignID", - "type": "UInt32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue.LogInfo.eventGroupId", - "name": "EventGroupId", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue/PlaceCoordinates/lat", - "name": "Latitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "EventValue/PlaceCoordinates/lon", - "name": "Longitude", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "DeviceID", - "type": "any", - "key": false, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "APIKey", - "type": "uint32", - "key": false, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppPlatform", - "type": "string", - "key": false, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "AppBuildNumber", - "type": "uint32", - "key": false, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EventName", - "type": "string", - "key": false, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_taxi_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_taxi_canon_0/extracted deleted file mode 100644 index 15ca2806..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_taxi_canon_0/extracted +++ /dev/null @@ -1,2266 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "timestamp", - "span_id", - "trace_id", - "parent_id", - "total_time", - "text", - "module", - "subkey", - "type", - "method", - "body", - "source_uri", - "host", - "link", - "parent_link", - "meta_user_id", - "iso_eventtime", - "acceptlang", - "level", - "uri", - "meta_type", - "meta_code", - "useragent", - "remote_ip", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "2019-08-14 23:14:49,517" - }, - { - "type": "string", - "value": "ff2d7cf70d34449a9f71a60de9b740e2" - }, - { - "type": "string", - "value": "7524cd862e5c4f1b975dffbeabab06f0" - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "0.03538799285888672" - }, - { - "type": "string", - "value": "Span finished" - }, - { - "type": "string", - "value": "taxi.opentracing.reporter" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "{\"operation_name\": \"/configs/values\", \"trace_id\": \"7524cd862e5c4f1b975dffbeabab06f0\", \"span_id\": \"ff2d7cf70d34449a9f71a60de9b740e2\", \"start_time\": 1565813689.481596, \"stop_time\": 1565813689.516984, \"tags\": {\"service\": \"chatterbox\", \"http.method\": \"POST\", \"http.url\": \"http://configs.taxi.yandex.net/configs/values\", \"span.kind\": \"client\", \"http.status_code\": 200, \"error\": false}, \"baggage\": \"{\\\\\"report\\\\\": \\\\\"true\\\\\"}\"}" - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "chatterbox-iva-01.taxi.yandex.net" - }, - { - "type": "string", - "value": "5e2746256dbf44e3a561c5b2c3c07500" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "INFO" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_type": { - "type": "string", - "value": "span" - } - } - }, - { - "type": "time.Time", - "value": "2019-08-14T23:14:49.517Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 730, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "taxi_taxi-chatterbox-yandex-taxi-chatterbox-log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "span_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "trace_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "total_time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "text", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "module", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "subkey", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "method", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "body", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "source_uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_user_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "iso_eventtime", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "acceptlang", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "level", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_code", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "useragent", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "remote_ip", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "timestamp", - "span_id", - "trace_id", - "parent_id", - "total_time", - "text", - "module", - "subkey", - "type", - "method", - "body", - "source_uri", - "host", - "link", - "parent_link", - "meta_user_id", - "iso_eventtime", - "acceptlang", - "level", - "uri", - "meta_type", - "meta_code", - "useragent", - "remote_ip", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "2019-08-14 23:14:49,569" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "config cache updated" - }, - { - "type": "string", - "value": "taxi.util.aiohttp_kit.context" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "chatterbox-iva-01.taxi.yandex.net" - }, - { - "type": "string", - "value": "None" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "INFO" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_type": { - "type": "string", - "value": "log" - } - } - }, - { - "type": "time.Time", - "value": "2019-08-14T23:14:49.569Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "taxi_taxi-chatterbox-yandex-taxi-chatterbox-log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "span_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "trace_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "total_time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "text", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "module", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "subkey", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "method", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "body", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "source_uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_user_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "iso_eventtime", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "acceptlang", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "level", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_code", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "useragent", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "remote_ip", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "timestamp", - "span_id", - "trace_id", - "parent_id", - "total_time", - "text", - "module", - "subkey", - "type", - "method", - "body", - "source_uri", - "host", - "link", - "parent_link", - "meta_user_id", - "iso_eventtime", - "acceptlang", - "level", - "uri", - "meta_type", - "meta_code", - "useragent", - "remote_ip", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "2019-08-14 23:14:49,577" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "config cache updated" - }, - { - "type": "string", - "value": "taxi.util.aiohttp_kit.context" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "chatterbox-iva-01.taxi.yandex.net" - }, - { - "type": "string", - "value": "None" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "INFO" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_type": { - "type": "string", - "value": "log" - } - } - }, - { - "type": "time.Time", - "value": "2019-08-14T23:14:49.577Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 3 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "taxi_taxi-chatterbox-yandex-taxi-chatterbox-log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "span_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "trace_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "total_time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "text", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "module", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "subkey", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "method", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "body", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "source_uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_user_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "iso_eventtime", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "acceptlang", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "level", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_code", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "useragent", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "remote_ip", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "timestamp", - "span_id", - "trace_id", - "parent_id", - "total_time", - "text", - "module", - "subkey", - "type", - "method", - "body", - "source_uri", - "host", - "link", - "parent_link", - "meta_user_id", - "iso_eventtime", - "acceptlang", - "level", - "uri", - "meta_type", - "meta_code", - "useragent", - "remote_ip", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "2019-08-14 23:14:50,240" - }, - { - "type": "string", - "value": "0d7618080583400fb485fc5ceaea5eb2" - }, - { - "type": "string", - "value": "3a5a49a299294f35b74ff8297b4a98fa" - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "0.02343916893005371" - }, - { - "type": "string", - "value": "Span finished" - }, - { - "type": "string", - "value": "taxi.opentracing.reporter" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "{\"operation_name\": \"/configs/values\", \"trace_id\": \"3a5a49a299294f35b74ff8297b4a98fa\", \"span_id\": \"0d7618080583400fb485fc5ceaea5eb2\", \"start_time\": 1565813690.21677, \"stop_time\": 1565813690.240209, \"tags\": {\"service\": \"chatterbox\", \"http.method\": \"POST\", \"http.url\": \"http://configs.taxi.yandex.net/configs/values\", \"span.kind\": \"client\", \"http.status_code\": 200, \"error\": false}, \"baggage\": \"{\\\\\"report\\\\\": \\\\\"true\\\\\"}\"}" - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "chatterbox-iva-01.taxi.yandex.net" - }, - { - "type": "string", - "value": "c6d771e23e7e4a9495c13f696033d8bf" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "string", - "value": "INFO" - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_type": { - "type": "string", - "value": "span" - } - } - }, - { - "type": "time.Time", - "value": "2019-08-14T23:14:50.24Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 4 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"taxi/taxi-chatterbox-yandex-taxi-chatterbox-log\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 729, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "taxi_taxi-chatterbox-yandex-taxi-chatterbox-log" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "span_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "trace_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "total_time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "text", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "module", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "subkey", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "method", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "body", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "source_uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parent_link", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_user_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "iso_eventtime", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "acceptlang", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "level", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "uri", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_type", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "meta_code", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "useragent", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "remote_ip", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted deleted file mode 100644 index b82dc5cd..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted +++ /dev/null @@ -1,308 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "message", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "Hello\\nworld" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"tm-5249\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"tm-5249\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 25, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "tm-5249" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "message", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "message", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "Hello world" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"tm-5249\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"tm-5249\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 24, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "tm-5249" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "message", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tskv_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tskv_canon_0/extracted deleted file mode 100644 index 9a86f1e1..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestGenericParsers_tskv_canon_0/extracted +++ /dev/null @@ -1,954 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25160 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25161 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25162 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 3 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "cluster_id", - "cluster_name", - "host", - "database", - "pid", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "mdbkbeut80vtiba04gid" - }, - { - "type": "string", - "value": "efimkins" - }, - { - "type": "string", - "value": "sas-au0cwt9wu98m6t18.db.yandex.net" - }, - { - "type": "string", - "value": "postgres" - }, - { - "type": "uint32", - "value": 25163 - }, - { - "type": "map[string]interface {}", - "value": {} - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 4 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_id", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cluster_name", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "host", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "database", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "uint32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted deleted file mode 100644 index 5da591dc..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted +++ /dev/null @@ -1,1190 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "hostname", - "pid", - "event", - "month", - "date", - "time", - "_topic", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "lbk-sas-1082" - }, - { - "type": "string", - "value": "463976" - }, - { - "type": "string", - "value": ":PQ_READ_PROXY INFO: session cookie 10960 client rtmr@rtmr-dev-sas-1 session rtmr@rtmr-dev-sas-1_555_10960_4548552247166245006 commit request from client for 7944668" - }, - { - "type": "string", - "value": "Mar" - }, - { - "type": "string", - "value": "10" - }, - { - "type": "string", - "value": "21:39:35" - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_lb_ctime": { - "type": "time.Time", - "value": "2020-02-02T10:02:21Z" - }, - "_lb_extra_some_field": { - "type": "string", - "value": "test" - }, - "_lb_wtime": { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - "_logfeller_timestamp": { - "type": "uint64", - "value": 1583854775 - }, - "iso_eventtime": { - "type": "string", - "value": "2020-03-10 18:39:35" - }, - "timestamp": { - "type": "string", - "value": "2020-03-10T18:39:35.194229Z" - } - } - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 238, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "hostname", - "type": "string", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "month", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_topic", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "hostname", - "pid", - "event", - "month", - "date", - "time", - "_topic", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "lbk-sas-1082" - }, - { - "type": "string", - "value": "463976" - }, - { - "type": "string", - "value": ":PQ_READ_PROXY INFO: session cookie 10960 client rtmr@rtmr-dev-sas-1 session rtmr@rtmr-dev-sas-1_555_10960_4548552247166245006 commit request from client for 7944675" - }, - { - "type": "string", - "value": "Mar" - }, - { - "type": "string", - "value": "10" - }, - { - "type": "string", - "value": "21:39:35" - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_lb_ctime": { - "type": "time.Time", - "value": "2020-02-02T10:02:21Z" - }, - "_lb_extra_some_field": { - "type": "string", - "value": "test" - }, - "_lb_wtime": { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - "_logfeller_timestamp": { - "type": "uint64", - "value": 1583854775 - }, - "iso_eventtime": { - "type": "string", - "value": "2020-03-10 18:39:35" - }, - "timestamp": { - "type": "string", - "value": "2020-03-10T18:39:35.196254Z" - } - } - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 238, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "hostname", - "type": "string", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "month", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_topic", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "hostname", - "pid", - "event", - "month", - "date", - "time", - "_topic", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "lbk-sas-1082" - }, - { - "type": "string", - "value": "463976" - }, - { - "type": "string", - "value": ":PQ_WRITE_PROXY INFO: CheckACL ticket 3:ser***CLAkg got result from TICKET_PARSER response: error: user: 2001309@tvm" - }, - { - "type": "string", - "value": "Mar" - }, - { - "type": "string", - "value": "10" - }, - { - "type": "string", - "value": "21:39:35" - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_lb_ctime": { - "type": "time.Time", - "value": "2020-02-02T10:02:21Z" - }, - "_lb_extra_some_field": { - "type": "string", - "value": "test" - }, - "_lb_wtime": { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - "_logfeller_timestamp": { - "type": "uint64", - "value": 1583854775 - }, - "iso_eventtime": { - "type": "string", - "value": "2020-03-10 18:39:35" - }, - "timestamp": { - "type": "string", - "value": "2020-03-10T18:39:35.228368Z" - } - } - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 3 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 190, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "hostname", - "type": "string", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "month", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_topic", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "hostname", - "pid", - "event", - "month", - "date", - "time", - "_topic", - "_rest", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "lbk-sas-1082" - }, - { - "type": "string", - "value": "463976" - }, - { - "type": "string", - "value": ":PQ_WRITE_PROXY INFO: session cookie: 1332024 sessionId: WRITE_rthub--favicon:14_1583842639258|26d60892-c7a0506b-781747db-d672bab_0 describe result for acl check" - }, - { - "type": "string", - "value": "Mar" - }, - { - "type": "string", - "value": "10" - }, - { - "type": "string", - "value": "21:39:35" - }, - { - "type": "nil", - "value": null - }, - { - "type": "map[string]interface {}", - "value": { - "_lb_ctime": { - "type": "time.Time", - "value": "2020-02-02T10:02:21Z" - }, - "_lb_extra_some_field": { - "type": "string", - "value": "test" - }, - "_lb_wtime": { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - "_logfeller_timestamp": { - "type": "uint64", - "value": 1583854775 - }, - "iso_eventtime": { - "type": "string", - "value": "2020-03-10 18:39:35" - }, - "timestamp": { - "type": "string", - "value": "2020-03-10T18:39:35.228416Z" - } - } - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 4 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 234, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "hostname", - "type": "string", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "pid", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "month", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "time", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_topic", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_rest", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted deleted file mode 100644 index 049dac1c..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted +++ /dev/null @@ -1,512 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "component", - "event", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "TENANT_SLOT_BROKER" - }, - { - "type": "string", - "value": "TTxUpdateNodeLocation Execute" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 159, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "ydb-ru-prestable" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "component", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "component", - "event", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "TENANT_SLOT_BROKER" - }, - { - "type": "string", - "value": "TTxUpdateNodeLocation Complete" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 2 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 160, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "ydb-ru-prestable" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "component", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "component", - "event", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "TENANT_SLOT_BROKER" - }, - { - "type": "string", - "value": "TTxUpdatePoolStatus execute for node 9601" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 3 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "ydb-ru-prestable" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "component", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "event", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted b/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted deleted file mode 100644 index ef70e358..00000000 --- a/tests/canon/parser/gotest/canondata/gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted +++ /dev/null @@ -1,223 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "timestamp", - "timezone", - "data1", - "data2", - "data3", - "_timestamp", - "_partition", - "_offset", - "_idx" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "2021-07-28 10:11:45" - }, - { - "type": "string", - "value": "+0000" - }, - { - "type": "string", - "value": "AQAD-sOme38sekReTS78sh0udnot38beknowUn9" - }, - { - "type": "string", - "value": "Lorem Ipsum Secret YOLO-l00kAtMyPrIvAtEda7aSIRthAnkSaALOTT" - }, - { - "type": "string", - "value": "compromized token AQAD-qJSKE2GAAAOymXXXXXXXXXXXXXXXXXXXXX" - }, - { - "type": "time.Time", - "value": "2020-02-02T10:02:20Z" - }, - { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - { - "type": "uint64", - "value": 123 - }, - { - "type": "uint32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"\"}" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 224, - "Values": 0 - } - }, - "Table": { - "type": "string", - "value": "" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timezone", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "data1", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "data2", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "data3", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_timestamp", - "type": "timestamp", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_partition", - "type": "string", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_offset", - "type": "uint64", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "_idx", - "type": "uint32", - "key": true, - "fake_key": false, - "required": true, - "expression": "", - "original_type": "" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/parser/gotest/canondata/result.json b/tests/canon/parser/gotest/canondata/result.json deleted file mode 100644 index c70c4768..00000000 --- a/tests/canon/parser/gotest/canondata/result.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "gotest.gotest.TestDynamicParsers/sample_parser/canon_0": { - "uri": "file://gotest.gotest.TestDynamicParsers_sample_parser_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/json/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_json_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/mdb/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_mdb_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/metrika/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_metrika_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/metrika_complex/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_metrika_complex_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/taxi/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_taxi_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/tm-5249/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_tm-5249_canon_0/extracted" - }, - "gotest.gotest.TestGenericParsers/tskv/canon_0": { - "uri": "file://gotest.gotest.TestGenericParsers_tskv_canon_0/extracted" - }, - "gotest.gotest.TestLogfellerParsers/kikimr/canon_0": { - "uri": "file://gotest.gotest.TestLogfellerParsers_kikimr_canon_0/extracted" - }, - "gotest.gotest.TestLogfellerParsers/kikimr_new/canon_0": { - "uri": "file://gotest.gotest.TestLogfellerParsers_kikimr_new_canon_0/extracted" - }, - "gotest.gotest.TestLogfellerParsers/sensitive/canon_0": { - "uri": "file://gotest.gotest.TestLogfellerParsers_sensitive_canon_0/extracted" - } -} diff --git a/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/README.MD b/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/README.MD deleted file mode 100644 index 109557e7..00000000 --- a/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/README.MD +++ /dev/null @@ -1,4 +0,0 @@ -### Generate desc file - -In order to generate desc file call: -``` ya make --add-result .desc .``` diff --git a/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/sample_proto.proto b/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/sample_proto.proto deleted file mode 100644 index 0f008849..00000000 --- a/tests/canon/parser/samples/dynamic/sample_proto/sample_proto/sample_proto.proto +++ /dev/null @@ -1,35 +0,0 @@ -syntax = "proto3"; - -option go_package = "github.com/doublecloud/tross/tests/canon/parser/samples/dynamic/sample_proto/sample_proto"; - -message SampleProto { - double doubleField = 1; - float floatField = 2; - int32 int32Field = 3; - int64 int64Field = 4; - uint32 uint32Field = 5; - uint64 uint64Field = 6; - sint32 sint32Field = 7; - sint64 sint64Field = 8; - fixed32 fixed32Field = 9; - fixed64 fixed64Field = 10; - sfixed32 sfixed32Field = 11; - sfixed64 sfixed64Field = 12; - bool boolField = 13; - string stringField = 14; - bytes bytesField = 15; - map mapField = 16; - repeated string repeatedField = 17; - SampleEmbeddedMsg msgField = 18; -} - -message SampleEmbeddedMsg { - string stringField = 1; - int32 int32Field = 2; - SampleEmbeddedEnum enumField = 3; -} - -enum SampleEmbeddedEnum { - ITEM_1 = 0; - ITEM_2 = 1; -} diff --git a/tests/canon/parser/samples/dynamic/sample_proto/test_case.go b/tests/canon/parser/samples/dynamic/sample_proto/test_case.go deleted file mode 100644 index ed336472..00000000 --- a/tests/canon/parser/samples/dynamic/sample_proto/test_case.go +++ /dev/null @@ -1,111 +0,0 @@ -package sampleproto - -import ( - "bytes" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/parsers/registry/protobuf" - "github.com/doublecloud/tross/tests/canon/parser/samples/dynamic/sample_proto/sample_proto" - "github.com/doublecloud/tross/tests/canon/parser/testcase" - "github.com/golang/protobuf/descriptor" - "github.com/golang/protobuf/proto" - "github.com/jhump/protoreflect/desc" -) - -func makeDesc() ([]byte, error) { - messageDescriptorProto, _ := descriptor.MessageDescriptorProto(new(sample_proto.SampleProto)) - fd, err := desc.CreateFileDescriptor(messageDescriptorProto) - if err != nil { - return nil, xerrors.Errorf("cannot create file descriptor: %w", err) - } - ffds := desc.ToFileDescriptorSet(fd) - data, err := proto.Marshal(ffds) - if err != nil { - return nil, xerrors.Errorf("cannot marshal file descriptor: %w", err) - } - return data, nil -} - -func buildMessages() []proto.Message { - return []proto.Message{ - &sample_proto.SampleProto{ - DoubleField: 3.14, - FloatField: 1.46, - Int32Field: -228, - Int64Field: -1337, - Uint32Field: 42, - Uint64Field: 300000000000, - Sint32Field: -1, - Sint64Field: -1488, - Fixed32Field: 9001, - Fixed64Field: 127, - Sfixed32Field: -5, - Sfixed64Field: -273, - BoolField: true, - StringField: "Я знаю, что ты знаешь этот трек, готовься подпевать", - BytesField: []byte("Раз, два, три, четыре"), - MapField: map[string]int32{ - "МУЗЫКА ГРОМЧЕ, ГЛАЗА ЗАКРЫТЫ": 1, - "ЭТО НОНСТОП, НОЧЬЮ ОТКРЫТИЙ": 1, - "ДЕЛАЙ ЧТО ХОЧЕШЬ, Я ЗАБЫВАЮСЬ": 1, - "ЭТО НОНСТОП, НЕ ПРЕКРАЩАЯСЬ": 1, - "МУЗЫКА ГРОМЧЕ, ГЛАЗА ЗАКРЫТЫ (хэй)": 2, - "ЭТО НОНСТОООП, НОЧЬЮ ОТКРЫТИЙ": 2, - "БУДУ С ТОБОЮ, САМОЙ ПРИМЕРНОЮ": 2, - "УТРО В ОКНЕ И МЫ БУДЕМ ПЕРВЫМИ": 2, - }, - RepeatedField: []string{ - "Этот трек делает тебя сильней", - "Он прикольней, чем колёса, и роднее, чем портвейн", - "Раза в три круче, чем самый первый секс-партнёр", - "Все девчата в таком трипе, что аж «мама не горюй», йоп", - }, - MsgField: &sample_proto.SampleEmbeddedMsg{ - StringField: "Прыгай в такт, прыгай в такт, будто ты совсем дурак", - Int32Field: 256, - EnumField: sample_proto.SampleEmbeddedEnum_ITEM_2, - }, - }, - } -} - -func makeData() []byte { - var out bytes.Buffer - for i, msg := range buildMessages() { - data, err := proto.Marshal(msg) - if err != nil { - logger.Log.Fatal("Cannot marshall message", log.Any("iteration", i), log.Error(err)) - } - out.Write(data) - } - return out.Bytes() -} - -func makeParser() *protobuf.ParserConfigProtoLb { - descFile, err := makeDesc() - if err != nil { - panic(err) - } - return &protobuf.ParserConfigProtoLb{ - DescFile: descFile, - DescResourceName: "", - MessageName: "SampleProto", - IncludeColumns: nil, - PrimaryKeys: nil, - PackageType: "PackageTypeSingleMsg", - NullKeysAllowed: false, - AddSystemColumns: false, - SkipDedupKeys: false, - AddSyntheticKeys: false, - } -} - -func MakeTestCaseSampleProto() testcase.TestCase { - return testcase.TestCase{ - TopicName: "/market-loyalty/testing/promos/market-perks", - ParserConfig: makeParser(), - Data: testcase.MakeDefaultPersqueueReadMessage(makeData()), - } -} diff --git a/tests/canon/parser/samples/static/generic/json.config.json b/tests/canon/parser/samples/static/generic/json.config.json deleted file mode 100644 index 49641377..00000000 --- a/tests/canon/parser/samples/static/generic/json.config.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "json.lb": { - "AddRest": true, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "cluster_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "cluster_name", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "host", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "database", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "pid", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "version", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint64" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": false, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": [""] -} diff --git a/tests/canon/parser/samples/static/generic/json.sample b/tests/canon/parser/samples/static/generic/json.sample deleted file mode 100644 index 6212fc17..00000000 --- a/tests/canon/parser/samples/static/generic/json.sample +++ /dev/null @@ -1,4 +0,0 @@ -{"cluster_id": "mdbkbeut80vtiba04gid", "cluster_name": "efimkins", "host": "sas-au0cwt9wu98m6t18.db.yandex.net", "database": "postgres", "pid": 25160, "version": 89488198116272410} -{"cluster_id": "mdbkbeut80vtiba04gid", "cluster_name": "efimkins", "host": "sas-au0cwt9wu98m6t18.db.yandex.net", "database": "postgres", "pid": 25162, "version": 89488198116272411} -{"cluster_id": "mdbkbeut80vtiba04gid", "cluster_name": "efimkins", "host": "sas-au0cwt9wu98m6t18.db.yandex.net", "database": "postgres", "pid": 25161, "version": 89488198116272412} -{"cluster_id": "mdbkbeut80vtiba04gid", "cluster_name": "efimkins", "host": "sas-au0cwt9wu98m6t18.db.yandex.net", "database": "postgres", "pid": 25163, "version": 89488198116272413} diff --git a/tests/canon/parser/samples/static/generic/mdb.config.json b/tests/canon/parser/samples/static/generic/mdb.config.json deleted file mode 100644 index 5444fcae..00000000 --- a/tests/canon/parser/samples/static/generic/mdb.config.json +++ /dev/null @@ -1,354 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "logbroker", - "Consumer": "mdb/porto/test/perf_diag/consumer", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": true, - "ParserConfig": { - "json.lb": { - "AddRest": true, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "collect_time", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "cluster_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "cluster_name", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "host", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "database", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "pid", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "user", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "application_name", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "client_addr", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "client_hostname", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "client_port", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "backend_start", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "xact_start", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "query_start", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "state_change", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "wait_event_type", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "wait_event", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "state", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "backend_xid", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "backend_xmin", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "query", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "backend_type", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "blocking_pids", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "queryid", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": false, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["mdb/porto/test/perf_diag/pg_stat_activity"] -} diff --git a/tests/canon/parser/samples/static/generic/mdb.sample b/tests/canon/parser/samples/static/generic/mdb.sample deleted file mode 100644 index c6567dc3..00000000 --- a/tests/canon/parser/samples/static/generic/mdb.sample +++ /dev/null @@ -1 +0,0 @@ -{"collect_time": "2020-08-10 09:46:22", "cluster_id": "mdbkbeut80vtiba04gid", "cluster_name": "efimkins", "host": "sas-au0cwt9wu98m6t18.db.yandex.net", "database": "postgres", "pid": 25167, "user": "postgres", "application_name": "", "client_addr": null, "client_hostname": null, "client_port": -1, "backend_start": "2020-08-10 09:46:22.330498+03", "xact_start": null, "query_start": "2020-08-10 09:46:22.333591+03", "state_change": "2020-08-10 09:46:22.333623+03", "wait_event_type": "Client", "wait_event": "ClientRead", "state": "idle", "backend_xid": null, "backend_xmin": null, "query": "SELECT 42;", "backend_type": "client backend", "blocking_pids": [], "queryid": 651361249} diff --git a/tests/canon/parser/samples/static/generic/metrika.config.json b/tests/canon/parser/samples/static/generic/metrika.config.json deleted file mode 100644 index 5e1518a9..00000000 --- a/tests/canon/parser/samples/static/generic/metrika.config.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "tskv.lb": { - "AddRest": false, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "AppVersionName", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "LogInfo", - "original_type": "", - "path": "EventValue.LogInfo", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DeviceID", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "APIKey", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "AppPlatform", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "AppBuildNumber", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "ReceiveTimestamp", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Latitude", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Longitude", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Age", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Sex", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "EventID", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "EventName", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": false, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["maps-adv-statistics/raw_metrika_log"] -} diff --git a/tests/canon/parser/samples/static/generic/metrika.sample b/tests/canon/parser/samples/static/generic/metrika.sample deleted file mode 100644 index ea2642ae..00000000 --- a/tests/canon/parser/samples/static/generic/metrika.sample +++ /dev/null @@ -1,4 +0,0 @@ -tskv tskv_format=metrika-mobile-log timestamp=2019-08-28 03:27:45 timezone=+0000 APIKey=30488 APIKey128=6f2668fb-be7c-4684-ac42-f07d87626435 Age=35 AppBuildNumber=402 AppDebuggable=false AppFramework=NATIVE AppID=ru.yandex.mobile.navigator AppPlatform=iOS AppVersionName=397 AttributionID=1 AttributionIDUUIDHash=14160930573867247291 ClickDate=1970-01-01 ClientIP=::ffff:31.173.103.187 ClientIPHash=4783886132929328837 ClientKitVersion=3006000 ClientPort=26499 ConnectionType=CONN_CELL CountryCode=250 CrashEncodeType=unknown DecodeStatus=unknown DeduplicationEnabled=1 DeviceID=060D7218-49D2-4344-A42E-FDDB0C878251 DeviceIDHash=15805882747688428900 DeviceIDSessionIDHash=898334369518194374 DeviceType=PHONE EventDate=2019-08-28 EventDateTime=2019-08-28 06:27:42 EventFirstOccurrence=false EventGlobalNumber=53301 EventID=14009544124325608446 EventName=geoadv.bb.pin.show EventNumber=180 EventNumberOfType=51869 EventSource=sdk EventTimeOffset=143 EventTimeZone=18000 EventTimestamp=1566962862 EventType=EVENT_CLIENT EventValue={"type":"route-ad","PlaceId":"459263","LogInfo":"{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","log_id":"351","CounterImpressionsPerDay":"1","oid":"459263","CounterImpressionsTotal":"1","PretendentsZoom":"14.3818102","PlaceCoordinates":"{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","UserCoordinates":"{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","id":"7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","PretendentsCount":"1"} EventValueJsonReference={0:"0",1:"1",2:"2",3:"3",4:"4",5:"1",6:"4",7:"7",8:"8",9:"9",10:"10",11:"4"} IFV=2003F812-1B82-48FB-98A2-DF783021E83D IsRevenueVerified=undefined KitBuildNumber=14209 KitBuildType=101 KitVersion=3006000 Latitude=56.823172635591256 LimitAdTracking=false Locale=ru_RU LocationAltitude=265 LocationDirection=304 LocationEnabled=true LocationPrecision=10 LocationSource=GPS LocationSpeed=31 LocationTimestamp=1566962858 Longitude=60.4962324663226 Manufacturer=Apple Model=iPhone 5s NetworkType=HSDPA OSApiLevel=12 OSVersion=12.4 OperatingSystem=ios OperatorID=2 OperatorName=MegaFon OriginalDeviceID=FA302D0F-D8CA-4E2F-8279-C7CA34EB8ADE OriginalManufacturer=Apple OriginalModel=iPhone6,2 ParsedParams_Key1=["type","PlaceId","LogInfo","log_id","CounterImpressionsPerDay","oid","CounterImpressionsTotal","PretendentsZoom","PlaceCoordinates","UserCoordinates","id","PretendentsCount"] ParsedParams_Key10=["","","","","","","","","","","",""] ParsedParams_Key2=["route-ad","459263","{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","351","1","459263","1","14.3818102","{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","1"] ParsedParams_Key3=["","","","","","","","","","","",""] ParsedParams_Key4=["","","","","","","","","","","",""] ParsedParams_Key5=["","","","","","","","","","","",""] ParsedParams_Key6=["","","","","","","","","","","",""] ParsedParams_Key7=["","","","","","","","","","","",""] ParsedParams_Key8=["","","","","","","","","","","",""] ParsedParams_Key9=["","","","","","","","","","","",""] ParsedParams_Quantity=[1,1,1,1,1,1,1,1,1,1,1,1] ParsedParams_ValueDouble=[0,459263,0,351,1,459263,1,14.3818102,0,0,0,1] ProfileAttributeVersion=26289272000610484 ReceiveDate=2019-08-28 ReceiveTimestamp=1566962865 RegionID=101976 RegionTimeZone=18000 RevenueOrderIdSource=autogenerated RevenueValidationEnvironment=undefined ScaleFactor=2 ScreenDPI=326 ScreenHeight=320 ScreenWidth=568 SendTimestamp=1566962864 SessionID=10000000660 SessionType=SESSION_FOREGROUND Sex=1 Sign=1 SimCards_AreRoaming=[0] SimCards_CountriesCodes=[250] SimCards_IccIDs=[""] SimCards_OperatorsIDs=[2] SimCards_OperatorsNames=["MegaFon"] StartDate=2019-08-28 StartTime=2019-08-28 06:25:19 StartTimeZone=18000 StartTimestamp=1566962719 UUID=4cf7162bd2c84de792073eb00fae2307 UUIDHash=15782877460591501196 WifiAccessPointState=UNKNOWN -tskv tskv_format=metrika-mobile-log timestamp=2019-08-28 03:27:45 timezone=+0000 APIKey=30488 APIKey128=6f2668fb-be7c-4684-ac42-f07d87626435 Age=35 AppBuildNumber=402 AppDebuggable=false AppFramework=NATIVE AppID=ru.yandex.mobile.navigator AppPlatform=iOS AppVersionName=397 AttributionID=1 AttributionIDUUIDHash=14160930573867247291 ClickDate=1970-01-01 ClientIP=::ffff:31.173.103.187 ClientIPHash=4783886132929328837 ClientKitVersion=3006000 ClientPort=26499 ConnectionType=CONN_CELL CountryCode=250 CrashEncodeType=unknown DecodeStatus=unknown DeduplicationEnabled=1 DeviceID=060D7218-49D2-4344-A42E-FDDB0C878251 DeviceIDHash=15805882747688428900 DeviceIDSessionIDHash=898334369518194374 DeviceType=PHONE EventDate=2019-08-28 EventDateTime=2019-08-28 06:27:42 EventFirstOccurrence=false EventGlobalNumber=53301 EventID=14009544124325608446 EventName=geoadv.bb.pin.show EventNumber=180 EventNumberOfType=51869 EventSource=sdk EventTimeOffset=143 EventTimeZone=18000 EventTimestamp=1566962862 EventType=EVENT_CLIENT EventValue={"type":"route-ad","PlaceId":"459263","LogInfo":"{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","log_id":"351","CounterImpressionsPerDay":"1","oid":"459263","CounterImpressionsTotal":"1","PretendentsZoom":"14.3818102","PlaceCoordinates":"{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","UserCoordinates":"{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","id":"7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","PretendentsCount":"1"} EventValueJsonReference={0:"0",1:"1",2:"2",3:"3",4:"4",5:"1",6:"4",7:"7",8:"8",9:"9",10:"10",11:"4"} IFV=2003F812-1B82-48FB-98A2-DF783021E83D IsRevenueVerified=undefined KitBuildNumber=14209 KitBuildType=101 KitVersion=3006000 Latitude=56.823172635591256 LimitAdTracking=false Locale=ru_RU LocationAltitude=265 LocationDirection=304 LocationEnabled=true LocationPrecision=10 LocationSource=GPS LocationSpeed=31 LocationTimestamp=1566962858 Longitude=60.4962324663226 Manufacturer=Apple Model=iPhone 5s NetworkType=HSDPA OSApiLevel=12 OSVersion=12.4 OperatingSystem=ios OperatorID=2 OperatorName=MegaFon OriginalDeviceID=FA302D0F-D8CA-4E2F-8279-C7CA34EB8ADE OriginalManufacturer=Apple OriginalModel=iPhone6,2 ParsedParams_Key1=["type","PlaceId","LogInfo","log_id","CounterImpressionsPerDay","oid","CounterImpressionsTotal","PretendentsZoom","PlaceCoordinates","UserCoordinates","id","PretendentsCount"] ParsedParams_Key10=["","","","","","","","","","","",""] ParsedParams_Key2=["route-ad","459263","{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","351","1","459263","1","14.3818102","{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","1"] ParsedParams_Key3=["","","","","","","","","","","",""] ParsedParams_Key4=["","","","","","","","","","","",""] ParsedParams_Key5=["","","","","","","","","","","",""] ParsedParams_Key6=["","","","","","","","","","","",""] ParsedParams_Key7=["","","","","","","","","","","",""] ParsedParams_Key8=["","","","","","","","","","","",""] ParsedParams_Key9=["","","","","","","","","","","",""] ParsedParams_Quantity=[1,1,1,1,1,1,1,1,1,1,1,1] ParsedParams_ValueDouble=[0,459263,0,351,1,459263,1,14.3818102,0,0,0,1] ProfileAttributeVersion=26289272000610484 ReceiveDate=2019-08-28 ReceiveTimestamp=1566962865 RegionID=101976 RegionTimeZone=18000 RevenueOrderIdSource=autogenerated RevenueValidationEnvironment=undefined ScaleFactor=2 ScreenDPI=326 ScreenHeight=320 ScreenWidth=568 SendTimestamp=1566962864 SessionID=10000000660 SessionType=SESSION_FOREGROUND Sex=1 Sign=1 SimCards_AreRoaming=[0] SimCards_CountriesCodes=[250] SimCards_IccIDs=[""] SimCards_OperatorsIDs=[2] SimCards_OperatorsNames=["MegaFon"] StartDate=2019-08-28 StartTime=2019-08-28 06:25:19 StartTimeZone=18000 StartTimestamp=1566962719 UUID=4cf7162bd2c84de792073eb00fae2307 UUIDHash=15782877460591501196 WifiAccessPointState=UNKNOWN -tskv tskv_format=metrika-mobile-log timestamp=2019-08-28 03:27:45 timezone=+0000 APIKey=30488 APIKey128=6f2668fb-be7c-4684-ac42-f07d87626435 Age=35 AppBuildNumber=402 AppDebuggable=false AppFramework=NATIVE AppID=ru.yandex.mobile.navigator AppPlatform=iOS AppVersionName=397 AttributionID=1 AttributionIDUUIDHash=14160930573867247291 ClickDate=1970-01-01 ClientIP=::ffff:31.173.103.187 ClientIPHash=4783886132929328837 ClientKitVersion=3006000 ClientPort=26499 ConnectionType=CONN_CELL CountryCode=250 CrashEncodeType=unknown DecodeStatus=unknown DeduplicationEnabled=1 DeviceID=060D7218-49D2-4344-A42E-FDDB0C878251 DeviceIDHash=15805882747688428900 DeviceIDSessionIDHash=898334369518194374 DeviceType=PHONE EventDate=2019-08-28 EventDateTime=2019-08-28 06:27:42 EventFirstOccurrence=false EventGlobalNumber=53301 EventID=14009544124325608446 EventName=geoadv.bb.pin.show EventNumber=180 EventNumberOfType=51869 EventSource=sdk EventTimeOffset=143 EventTimeZone=18000 EventTimestamp=1566962862 EventType=EVENT_CLIENT EventValue={"type":"route-ad","PlaceId":"459263","LogInfo":"{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","log_id":"351","CounterImpressionsPerDay":"1","oid":"459263","CounterImpressionsTotal":"1","PretendentsZoom":"14.3818102","PlaceCoordinates":"{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","UserCoordinates":"{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","id":"7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","PretendentsCount":"1"} EventValueJsonReference={0:"0",1:"1",2:"2",3:"3",4:"4",5:"1",6:"4",7:"7",8:"8",9:"9",10:"10",11:"4"} IFV=2003F812-1B82-48FB-98A2-DF783021E83D IsRevenueVerified=undefined KitBuildNumber=14209 KitBuildType=101 KitVersion=3006000 Latitude=56.823172635591256 LimitAdTracking=false Locale=ru_RU LocationAltitude=265 LocationDirection=304 LocationEnabled=true LocationPrecision=10 LocationSource=GPS LocationSpeed=31 LocationTimestamp=1566962858 Longitude=60.4962324663226 Manufacturer=Apple Model=iPhone 5s NetworkType=HSDPA OSApiLevel=12 OSVersion=12.4 OperatingSystem=ios OperatorID=2 OperatorName=MegaFon OriginalDeviceID=FA302D0F-D8CA-4E2F-8279-C7CA34EB8ADE OriginalManufacturer=Apple OriginalModel=iPhone6,2 ParsedParams_Key1=["type","PlaceId","LogInfo","log_id","CounterImpressionsPerDay","oid","CounterImpressionsTotal","PretendentsZoom","PlaceCoordinates","UserCoordinates","id","PretendentsCount"] ParsedParams_Key10=["","","","","","","","","","","",""] ParsedParams_Key2=["route-ad","459263","{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","351","1","459263","1","14.3818102","{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","1"] ParsedParams_Key3=["","","","","","","","","","","",""] ParsedParams_Key4=["","","","","","","","","","","",""] ParsedParams_Key5=["","","","","","","","","","","",""] ParsedParams_Key6=["","","","","","","","","","","",""] ParsedParams_Key7=["","","","","","","","","","","",""] ParsedParams_Key8=["","","","","","","","","","","",""] ParsedParams_Key9=["","","","","","","","","","","",""] ParsedParams_Quantity=[1,1,1,1,1,1,1,1,1,1,1,1] ParsedParams_ValueDouble=[0,459263,0,351,1,459263,1,14.3818102,0,0,0,1] ProfileAttributeVersion=26289272000610484 ReceiveDate=2019-08-28 ReceiveTimestamp=1566962865 RegionID=101976 RegionTimeZone=18000 RevenueOrderIdSource=autogenerated RevenueValidationEnvironment=undefined ScaleFactor=2 ScreenDPI=326 ScreenHeight=320 ScreenWidth=568 SendTimestamp=1566962864 SessionID=10000000660 SessionType=SESSION_FOREGROUND Sex=1 Sign=1 SimCards_AreRoaming=[0] SimCards_CountriesCodes=[250] SimCards_IccIDs=[""] SimCards_OperatorsIDs=[2] SimCards_OperatorsNames=["MegaFon"] StartDate=2019-08-28 StartTime=2019-08-28 06:25:19 StartTimeZone=18000 StartTimestamp=1566962719 UUID=4cf7162bd2c84de792073eb00fae2307 UUIDHash=15782877460591501196 WifiAccessPointState=UNKNOWN -tskv tskv_format=metrika-mobile-log timestamp=2019-08-28 03:27:45 timezone=+0000 APIKey=30488 APIKey128=6f2668fb-be7c-4684-ac42-f07d87626435 Age=35 AppBuildNumber=402 AppDebuggable=false AppFramework=NATIVE AppID=ru.yandex.mobile.navigator AppPlatform=iOS AppVersionName=397 AttributionID=1 AttributionIDUUIDHash=14160930573867247291 ClickDate=1970-01-01 ClientIP=::ffff:31.173.103.187 ClientIPHash=4783886132929328837 ClientKitVersion=3006000 ClientPort=26499 ConnectionType=CONN_CELL CountryCode=250 CrashEncodeType=unknown DecodeStatus=unknown DeduplicationEnabled=1 DeviceID=060D7218-49D2-4344-A42E-FDDB0C878251 DeviceIDHash=15805882747688428900 DeviceIDSessionIDHash=898334369518194374 DeviceType=PHONE EventDate=2019-08-28 EventDateTime=2019-08-28 06:27:42 EventFirstOccurrence=false EventGlobalNumber=53301 EventID=14009544124325608446 EventName=geoadv.bb.pin.show EventNumber=180 EventNumberOfType=51869 EventSource=sdk EventTimeOffset=143 EventTimeZone=18000 EventTimestamp=1566962862 EventType=EVENT_CLIENT EventValue={"type":"route-ad","PlaceId":"459263","LogInfo":"{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","log_id":"351","CounterImpressionsPerDay":"1","oid":"459263","CounterImpressionsTotal":"1","PretendentsZoom":"14.3818102","PlaceCoordinates":"{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","UserCoordinates":"{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","id":"7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","PretendentsCount":"1"} EventValueJsonReference={0:"0",1:"1",2:"2",3:"3",4:"4",5:"1",6:"4",7:"7",8:"8",9:"9",10:"10",11:"4"} IFV=2003F812-1B82-48FB-98A2-DF783021E83D IsRevenueVerified=undefined KitBuildNumber=14209 KitBuildType=101 KitVersion=3006000 Latitude=56.823172635591256 LimitAdTracking=false Locale=ru_RU LocationAltitude=265 LocationDirection=304 LocationEnabled=true LocationPrecision=10 LocationSource=GPS LocationSpeed=31 LocationTimestamp=1566962858 Longitude=60.4962324663226 Manufacturer=Apple Model=iPhone 5s NetworkType=HSDPA OSApiLevel=12 OSVersion=12.4 OperatingSystem=ios OperatorID=2 OperatorName=MegaFon OriginalDeviceID=FA302D0F-D8CA-4E2F-8279-C7CA34EB8ADE OriginalManufacturer=Apple OriginalModel=iPhone6,2 ParsedParams_Key1=["type","PlaceId","LogInfo","log_id","CounterImpressionsPerDay","oid","CounterImpressionsTotal","PretendentsZoom","PlaceCoordinates","UserCoordinates","id","PretendentsCount"] ParsedParams_Key10=["","","","","","","","","","","",""] ParsedParams_Key2=["route-ad","459263","{\\"advertiserId\\": \\"None\\", \\"campaignId\\": \\"10343\\", \\"product\\": \\"billboard\\"}","351","1","459263","1","14.3818102","{\\"lon\\": 60.491409301757812, \\"lat\\": 56.824722290039062}","{\\"lon\\": 60.495459868572915, \\"lat\\": 56.823475273034035}","7f07d659e8f0ac1b8e0814ea25f74db8004c188eea22de097116f9a72375a2cb","1"] ParsedParams_Key3=["","","","","","","","","","","",""] ParsedParams_Key4=["","","","","","","","","","","",""] ParsedParams_Key5=["","","","","","","","","","","",""] ParsedParams_Key6=["","","","","","","","","","","",""] ParsedParams_Key7=["","","","","","","","","","","",""] ParsedParams_Key8=["","","","","","","","","","","",""] ParsedParams_Key9=["","","","","","","","","","","",""] ParsedParams_Quantity=[1,1,1,1,1,1,1,1,1,1,1,1] ParsedParams_ValueDouble=[0,459263,0,351,1,459263,1,14.3818102,0,0,0,1] ProfileAttributeVersion=26289272000610484 ReceiveDate=2019-08-28 ReceiveTimestamp=1566962865 RegionID=101976 RegionTimeZone=18000 RevenueOrderIdSource=autogenerated RevenueValidationEnvironment=undefined ScaleFactor=2 ScreenDPI=326 ScreenHeight=320 ScreenWidth=568 SendTimestamp=1566962864 SessionID=10000000660 SessionType=SESSION_FOREGROUND Sex=1 Sign=1 SimCards_AreRoaming=[0] SimCards_CountriesCodes=[250] SimCards_IccIDs=[""] SimCards_OperatorsIDs=[2] SimCards_OperatorsNames=["MegaFon"] StartDate=2019-08-28 StartTime=2019-08-28 06:25:19 StartTimeZone=18000 StartTimestamp=1566962719 UUID=4cf7162bd2c84de792073eb00fae2307 UUIDHash=15782877460591501196 WifiAccessPointState=UNKNOWN diff --git a/tests/canon/parser/samples/static/generic/metrika_complex.config.json b/tests/canon/parser/samples/static/generic/metrika_complex.config.json deleted file mode 100644 index a2d0902a..00000000 --- a/tests/canon/parser/samples/static/generic/metrika_complex.config.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "tskv.lb": { - "AddRest": false, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "AppVersionName", - "original_type": "", - "path": "", - "required": true, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "CampaignID", - "original_type": "", - "path": "EventValue.LogInfo.campaignId", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "UInt32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "EventGroupId", - "original_type": "", - "path": "EventValue.LogInfo.eventGroupId", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Latitude", - "original_type": "", - "path": "EventValue/PlaceCoordinates/lat", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "Longitude", - "original_type": "", - "path": "EventValue/PlaceCoordinates/lon", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "DeviceID", - "original_type": "", - "path": "", - "required": true, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "APIKey", - "original_type": "", - "path": "", - "required": true, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "AppPlatform", - "original_type": "", - "path": "", - "required": true, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "AppBuildNumber", - "original_type": "", - "path": "", - "required": true, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "EventName", - "original_type": "", - "path": "", - "required": true, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": false, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["maps-adv-statistics/raw_metrika_log"] -} diff --git a/tests/canon/parser/samples/static/generic/metrika_complex.sample b/tests/canon/parser/samples/static/generic/metrika_complex.sample deleted file mode 100644 index 3587c466..00000000 --- a/tests/canon/parser/samples/static/generic/metrika_complex.sample +++ /dev/null @@ -1 +0,0 @@ -2019-10-09 14:43:08 rt3.man--maps-adv-statistics--raw_metrika_log@1 4363632 2 "tskv tskv_format=metrika-mobile-log timestamp=2019-10-09 14:25:18 timezone=+0000 APIKey=30488 APIKey128=6f2668fb-be7c-4684-ac42-f07d87626435 AccountID=858553454 AccountType=login AppBuildNumber=360 AppDebuggable=false AppFramework=NATIVE AppID=ru.yandex.mobile.navigator AppPlatform=iOS AppVersionName=371 AttributionID=1 AttributionIDUUIDHash=13642217933273614327 ClickDate=1970-01-01 ClientIP=::ffff:31.173.80.189 ClientIPHash=2974387875995586398 ClientKitVersion=3005000 ClientPort=7861 ConnectionType=CONN_CELL CountryCode=250 CrashEncodeType=unknown DecodeStatus=unknown DeduplicationEnabled=1 DeviceID=4BCED9E0-E10B-4E72-83C7-42F80883CF3A DeviceIDHash=10531297670044343457 DeviceIDSessionIDHash=15753579471505515614 DeviceType=PHONE EventDate=2019-10-09 EventDateTime=2019-10-09 17:25:16 EventFirstOccurrence=false EventID=6039214698637519789 EventName=geoadv.bb.pin.show EventNumber=119 EventSource=sdk EventTimeOffset=523 EventTimeZone=10800 EventTimestamp=1570631116 EventType=EVENT_CLIENT EventValue={""log_id"":""131"",""CounterImpressionsPerDay"":""1"",""LogInfo"":""{\\""advertiserId\\"":\\""3872\\"",\\""campaignId\\"":\\""12087\\"",\\""product\\"":\\""zero_speed_banner\\"",\\""eventGroupId\\"":\\""b52a4d7bffe1162b58108e62ff5c5c6b\\""}"",""UserCoordinates"":""{\\""lon\\"": 37.818678613035807, \\""lat\\"": 55.803707786523489}"",""PretendentsCount"":"""",""CounterImpressionsTotal"":""1""} EventValueJsonReference={0:""0"",1:""1"",2:""2"",3:""3"",4:""4"",5:""1""} IFV=4BCED9E0-E10B-4E72-83C7-42F80883CF3A IsInstallationTrackingEnabled=1 IsOwnedByYandex=1 IsRevenueVerified=undefined KitBuildNumber=13847 KitBuildType=101 KitVersion=3005000 Latitude=55.805187790998716 LimitAdTracking=true Locale=ru_RU LocationAltitude=166 LocationDirection=178 LocationEnabled=true LocationPrecision=10 LocationSource=GPS LocationSpeed=16 LocationTimestamp=1570631060 Longitude=37.81862852498979 Manufacturer=Apple MoblogLayerID=1 Model=iPhone 7 Plus NetworkType=LTE OSApiLevel=11 OSVersion=11.0 OperatingSystem=ios OperatorID=2 OperatorName=MegaFon OriginalManufacturer=Apple OriginalModel=iPhone9,2 ParsedParams_Key1=[""log_id"",""CounterImpressionsPerDay"",""LogInfo"",""UserCoordinates"",""PretendentsCount"",""CounterImpressionsTotal""] ParsedParams_Key10=["""","""","""","""","""",""""] ParsedParams_Key2=[""131"",""1"",""{\\""advertiserId\\"":\\""3872\\"",\\""campaignId\\"":\\""12087\\"",\\""product\\"":\\""zero_speed_banner\\"",\\""eventGroupId\\"":\\""b52a4d7bffe1162b58108e62ff5c5c6b\\""}"",""{\\""lon\\"": 37.818678613035807, \\""lat\\"": 55.803707786523489}"","""",""1""] ParsedParams_Key3=["""","""","""","""","""",""""] ParsedParams_Key4=["""","""","""","""","""",""""] ParsedParams_Key5=["""","""","""","""","""",""""] ParsedParams_Key6=["""","""","""","""","""",""""] ParsedParams_Key7=["""","""","""","""","""",""""] ParsedParams_Key8=["""","""","""","""","""",""""] ParsedParams_Key9=["""","""","""","""","""",""""] ParsedParams_Quantity=[1,1,1,1,1,1] ParsedParams_ValueDouble=[131,1,0,0,0,1] ProfileAttributeVersion=26350808714969207 ReceiveDate=2019-10-09 ReceiveTimestamp=1570631118 RegionID=116993 RegionTimeZone=10800 RevenueOrderIdSource=autogenerated RevenueValidationEnvironment=undefined ScaleFactor=3 ScreenDPI=326 ScreenHeight=414 ScreenWidth=736 SendTimestamp=1570631117 SessionID=10000002230 SessionType=SESSION_FOREGROUND Sign=1 SimCards_AreRoaming=[0] SimCards_CountriesCodes=[250] SimCards_IccIDs=[""""] SimCards_OperatorsIDs=[2] SimCards_OperatorsNames=[""MegaFon""] StartDate=2019-10-09 StartTime=2019-10-09 17:16:33 StartTimeZone=10800 StartTimestamp=1570630593 UUID=7be0d360c921feb067e972ae53732c31 UUIDHash=10919961828586788839 WifiAccessPointState=UNKNOWN" \ No newline at end of file diff --git a/tests/canon/parser/samples/static/generic/taxi.config.json b/tests/canon/parser/samples/static/generic/taxi.config.json deleted file mode 100644 index b45ef5d9..00000000 --- a/tests/canon/parser/samples/static/generic/taxi.config.json +++ /dev/null @@ -1,357 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "tskv.lb": { - "AddRest": true, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "span_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "trace_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "parent_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "total_time", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "text", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "module", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "subkey", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "type", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "method", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "body", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "source_uri", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "host", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "link", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "parent_link", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "meta_user_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "iso_eventtime", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "acceptlang", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "level", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "uri", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "meta_type", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "meta_code", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "useragent", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "remote_ip", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": true, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": { - "Col": "timestamp", - "Format": "" - } - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["taxi/taxi-chatterbox-yandex-taxi-chatterbox-log"] -} diff --git a/tests/canon/parser/samples/static/generic/taxi.sample b/tests/canon/parser/samples/static/generic/taxi.sample deleted file mode 100644 index ffd60a29..00000000 --- a/tests/canon/parser/samples/static/generic/taxi.sample +++ /dev/null @@ -1,4 +0,0 @@ -tskv timestamp=2019-08-14 23:14:49,517 module=taxi.opentracing.reporter level=INFO link=5e2746256dbf44e3a561c5b2c3c07500 _type=span host=chatterbox-iva-01.taxi.yandex.net body={"operation_name": "/configs/values", "trace_id": "7524cd862e5c4f1b975dffbeabab06f0", "span_id": "ff2d7cf70d34449a9f71a60de9b740e2", "start_time": 1565813689.481596, "stop_time": 1565813689.516984, "tags": {"service": "chatterbox", "http.method": "POST", "http.url": "http://configs.taxi.yandex.net/configs/values", "span.kind": "client", "http.status_code": 200, "error": false}, "baggage": "{\\"report\\": \\"true\\"}"} span_id=ff2d7cf70d34449a9f71a60de9b740e2 trace_id=7524cd862e5c4f1b975dffbeabab06f0 total_time=0.03538799285888672 text=Span finished -tskv timestamp=2019-08-14 23:14:49,569 module=taxi.util.aiohttp_kit.context level=INFO link=None _type=log host=chatterbox-iva-01.taxi.yandex.net text=config cache updated -tskv timestamp=2019-08-14 23:14:49,577 module=taxi.util.aiohttp_kit.context level=INFO link=None _type=log host=chatterbox-iva-01.taxi.yandex.net text=config cache updated -tskv timestamp=2019-08-14 23:14:50,240 module=taxi.opentracing.reporter level=INFO link=c6d771e23e7e4a9495c13f696033d8bf _type=span host=chatterbox-iva-01.taxi.yandex.net body={"operation_name": "/configs/values", "trace_id": "3a5a49a299294f35b74ff8297b4a98fa", "span_id": "0d7618080583400fb485fc5ceaea5eb2", "start_time": 1565813690.21677, "stop_time": 1565813690.240209, "tags": {"service": "chatterbox", "http.method": "POST", "http.url": "http://configs.taxi.yandex.net/configs/values", "span.kind": "client", "http.status_code": 200, "error": false}, "baggage": "{\\"report\\": \\"true\\"}"} span_id=0d7618080583400fb485fc5ceaea5eb2 trace_id=3a5a49a299294f35b74ff8297b4a98fa total_time=0.02343916893005371 text=Span finished diff --git a/tests/canon/parser/samples/static/generic/tm-5249.config.json b/tests/canon/parser/samples/static/generic/tm-5249.config.json deleted file mode 100644 index 1ba60c9f..00000000 --- a/tests/canon/parser/samples/static/generic/tm-5249.config.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "tskv.lb": { - "AddRest": false, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "message", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": false, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["tm-5249"] -} diff --git a/tests/canon/parser/samples/static/generic/tm-5249.sample b/tests/canon/parser/samples/static/generic/tm-5249.sample deleted file mode 100644 index fd98204a..00000000 --- a/tests/canon/parser/samples/static/generic/tm-5249.sample +++ /dev/null @@ -1,2 +0,0 @@ -tskv message=Hello\nworld -tskv message=Hello world diff --git a/tests/canon/parser/samples/static/generic/tskv.config.json b/tests/canon/parser/samples/static/generic/tskv.config.json deleted file mode 100644 index e63a5051..00000000 --- a/tests/canon/parser/samples/static/generic/tskv.config.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": false, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "tskv.lb": { - "AddRest": true, - "AddSystemCols": false, - "DropUnparsed": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "cluster_id", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "cluster_name", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "host", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "database", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "pid", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "uint32" - } - ], - "IgnoreColumnPaths": false, - "MaskSecrets": false, - "NullKeysAllowed": false, - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": [""] -} diff --git a/tests/canon/parser/samples/static/generic/tskv.sample b/tests/canon/parser/samples/static/generic/tskv.sample deleted file mode 100644 index c2375f9a..00000000 --- a/tests/canon/parser/samples/static/generic/tskv.sample +++ /dev/null @@ -1,4 +0,0 @@ -tskv cluster_id=mdbkbeut80vtiba04gid cluster_name=efimkins host=sas-au0cwt9wu98m6t18.db.yandex.net database=postgres pid=25160 -tskv cluster_id=mdbkbeut80vtiba04gid cluster_name=efimkins host=sas-au0cwt9wu98m6t18.db.yandex.net database=postgres pid=25161 -tskv cluster_id=mdbkbeut80vtiba04gid cluster_name=efimkins host=sas-au0cwt9wu98m6t18.db.yandex.net database=postgres pid=25162 -tskv cluster_id=mdbkbeut80vtiba04gid cluster_name=efimkins host=sas-au0cwt9wu98m6t18.db.yandex.net database=postgres pid=25163 diff --git a/tests/canon/parser/samples/static/logfeller/_type_check_rules.yaml b/tests/canon/parser/samples/static/logfeller/_type_check_rules.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/canon/parser/samples/static/logfeller/kikimr-log-2.yaml b/tests/canon/parser/samples/static/logfeller/kikimr-log-2.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/canon/parser/samples/static/logfeller/kikimr-log.yaml b/tests/canon/parser/samples/static/logfeller/kikimr-log.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/canon/parser/samples/static/logfeller/kikimr-new-log.yaml b/tests/canon/parser/samples/static/logfeller/kikimr-new-log.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/canon/parser/samples/static/logfeller/kikimr.config.json b/tests/canon/parser/samples/static/logfeller/kikimr.config.json deleted file mode 100644 index 9551eb83..00000000 --- a/tests/canon/parser/samples/static/logfeller/kikimr.config.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "logbroker", - "Consumer": "cdc/prod/logfeller_ydb", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": true, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "logfeller.lb": { - "AddRest": true, - "AddSystemCols": true, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "hostname", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "pid", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "event", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "month", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "date", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "time", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "LineSplitter": "\n", - "MaskSecrets": false, - "NullKeysAllowed": true, - "ParserName": "kikimr-log", - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["/kikimr/unified-agent/kikimr-debug-log"] -} diff --git a/tests/canon/parser/samples/static/logfeller/kikimr.sample b/tests/canon/parser/samples/static/logfeller/kikimr.sample deleted file mode 100644 index 69371f53..00000000 --- a/tests/canon/parser/samples/static/logfeller/kikimr.sample +++ /dev/null @@ -1,4 +0,0 @@ -Mar 10 21:39:35 lbk-sas-1082 KIKIMR[463976]: 2020-03-10T18:39:35.194229Z :PQ_READ_PROXY INFO: session cookie 10960 client rtmr@rtmr-dev-sas-1 session rtmr@rtmr-dev-sas-1_555_10960_4548552247166245006 commit request from client for 7944668 -Mar 10 21:39:35 lbk-sas-1082 KIKIMR[463976]: 2020-03-10T18:39:35.196254Z :PQ_READ_PROXY INFO: session cookie 10960 client rtmr@rtmr-dev-sas-1 session rtmr@rtmr-dev-sas-1_555_10960_4548552247166245006 commit request from client for 7944675 -Mar 10 21:39:35 lbk-sas-1082 KIKIMR[463976]: 2020-03-10T18:39:35.228368Z :PQ_WRITE_PROXY INFO: CheckACL ticket 3:ser***CLAkg got result from TICKET_PARSER response: error: user: 2001309@tvm -Mar 10 21:39:35 lbk-sas-1082 KIKIMR[463976]: 2020-03-10T18:39:35.228416Z :PQ_WRITE_PROXY INFO: session cookie: 1332024 sessionId: WRITE_rthub--favicon:14_1583842639258|26d60892-c7a0506b-781747db-d672bab_0 describe result for acl check diff --git a/tests/canon/parser/samples/static/logfeller/kikimr_new.config.json b/tests/canon/parser/samples/static/logfeller/kikimr_new.config.json deleted file mode 100644 index 36d19d78..00000000 --- a/tests/canon/parser/samples/static/logfeller/kikimr_new.config.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "logbroker", - "Consumer": "cdc/prod/logfeller_ydb", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": true, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 504857, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "logfeller.lb": { - "AddRest": false, - "AddSystemCols": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "component", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "event", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "LineSplitter": "\n", - "MaskSecrets": false, - "NullKeysAllowed": true, - "ParserName": "kikimr-new-log", - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": { - "Columns": [ - "cluster" - ] - }, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": [""] -} diff --git a/tests/canon/parser/samples/static/logfeller/kikimr_new.sample b/tests/canon/parser/samples/static/logfeller/kikimr_new.sample deleted file mode 100644 index ca12eb89..00000000 --- a/tests/canon/parser/samples/static/logfeller/kikimr_new.sample +++ /dev/null @@ -1,3 +0,0 @@ -Apr 01 18:01:58 ydb-ru-prestable ydb-ru-prestable-vla-0004 KIKIMR[823365]: 2020-04-01T15:01:58.433132Z :TENANT_SLOT_BROKER DEBUG: TTxUpdateNodeLocation Execute -Apr 01 18:01:58 ydb-ru-prestable ydb-ru-prestable-vla-0004 KIKIMR[823365]: 2020-04-01T15:01:58.433135Z :TENANT_SLOT_BROKER DEBUG: TTxUpdateNodeLocation Complete -Apr 01 18:01:58 ydb-ru-prestable ydb-ru-prestable-vla-0004 KIKIMR[823365]: 2020-04-01T15:01:58.470478Z :TENANT_SLOT_BROKER DEBUG: TTxUpdatePoolStatus execute for node 9601 \ No newline at end of file diff --git a/tests/canon/parser/samples/static/logfeller/sensitive.config.json b/tests/canon/parser/samples/static/logfeller/sensitive.config.json deleted file mode 100644 index ccbfc0bb..00000000 --- a/tests/canon/parser/samples/static/logfeller/sensitive.config.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "AllowTTLRewind": false, - "Cluster": "", - "Consumer": "", - "Credentials": null, - "Database": "", - "EnrichTopic": false, - "Extra": {}, - "HashColumn": "", - "InferSchema": false, - "InferTimezone": false, - "Instance": "", - "IsLbSink": false, - "LfParser": true, - "MaxConcurrentPartitions": 0, - "MaxIdleTime": 0, - "MaxMemory": 52428800, - "MaxReadMessagesCount": 0, - "MaxReadSize": 1048576, - "MaxTimeLag": 0, - "OnlyLocal": false, - "ParserConfig": { - "logfeller.lb": { - "AddRest": false, - "AddSystemCols": false, - "Fields": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timezone", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "data1", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "data2", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "data3", - "original_type": "", - "path": "", - "required": false, - "system_key": false, - "table_name": "", - "table_schema": "", - "type": "string" - } - ], - "LineSplitter": "\n", - "MaskSecrets": true, - "NullKeysAllowed": false, - "ParserName": "tskv-log-with-timestamp", - "SchemaResourceName": "", - "SkipSystemKeys": false, - "TableSplitter": null, - "TimeField": null - } - }, - "Port": 0, - "Sniff": false, - "Token": "", - "Topics": ["cdc/test-sensitive-data"] -} diff --git a/tests/canon/parser/samples/static/logfeller/sensitive.sample b/tests/canon/parser/samples/static/logfeller/sensitive.sample deleted file mode 100644 index f068b2db..00000000 --- a/tests/canon/parser/samples/static/logfeller/sensitive.sample +++ /dev/null @@ -1 +0,0 @@ -tskv timestamp=2021-07-28 10:11:45 timezone=+0000 data1=AQAD-sOme38sekReTS78sh0udnot38beknowUn9 data2=Lorem Ipsum Secret YOLO-l00kAtMyPrIvAtEda7aSIRthAnkSaALOTT data3=compromized token AQAD-qJSKE2GAAAOympMWTvi8EvUm5pGS8biVF8 diff --git a/tests/canon/parser/testcase/test_case.go b/tests/canon/parser/testcase/test_case.go deleted file mode 100644 index ca073e65..00000000 --- a/tests/canon/parser/testcase/test_case.go +++ /dev/null @@ -1,85 +0,0 @@ -package testcase - -import ( - "embed" - "encoding/json" - "fmt" - "io/fs" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/kikimr/public/sdk/go/persqueue" - "github.com/doublecloud/tross/pkg/parsers" - _ "github.com/doublecloud/tross/pkg/parsers/registry" - "github.com/doublecloud/tross/pkg/providers/logbroker" - "github.com/stretchr/testify/require" -) - -type TestCase struct { - TopicName string - ParserConfig parsers.AbstractParserConfig - Data persqueue.ReadMessage -} - -func LoadStaticTestCases(t *testing.T, samples embed.FS) map[string]TestCase { - cases := map[string]TestCase{} - require.NoError(t, fs.WalkDir(samples, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d == nil { - return nil - } - if d.IsDir() { - return nil - } - if !strings.HasSuffix(path, ".config.json") { - return nil - } - caseName := strings.ReplaceAll(filepath.Base(path), ".config.json", "") - // DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG - fmt.Printf("timmyb32rQQQ:caseName=%s\n", caseName) - // DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG - configData, err := fs.ReadFile(samples, path) - if err != nil { - return err - } - sampleData, err := fs.ReadFile(samples, filepath.Dir(path)+"/"+caseName+".sample") - if err != nil { - return err - } - var source logbroker.LfSource - if err := json.Unmarshal(configData, &source); err != nil { - logger.Log.Warn("unable to unmarshal", log.Error(err)) - return nil - } - source.WithDefaults() - parserConfig, err := parsers.ParserConfigMapToStruct(source.ParserConfig) - require.NoError(t, err) - - cases[caseName] = TestCase{ - TopicName: source.Topics[0], - ParserConfig: parserConfig, - Data: MakeDefaultPersqueueReadMessage(sampleData), - } - return nil - })) - return cases -} - -func MakeDefaultPersqueueReadMessage(data []byte) persqueue.ReadMessage { - return persqueue.ReadMessage{ - Offset: 123, - SeqNo: 32, - SourceID: []byte("test_source_id"), - CreateTime: time.Date(2020, 2, 2, 10, 2, 21, 0, time.UTC), - WriteTime: time.Date(2020, 2, 2, 10, 2, 20, 0, time.UTC), - IP: "192.168.1.1", - Data: data, - ExtraFields: map[string]string{"some_field": "test"}, - } -} diff --git a/tests/canon/postgres/canon_sql.go b/tests/canon/postgres/canon_sql.go deleted file mode 100644 index aec63193..00000000 --- a/tests/canon/postgres/canon_sql.go +++ /dev/null @@ -1,27 +0,0 @@ -package postgres - -import _ "embed" - -var ( - //go:embed dump/array_types.sql - array []byte - //go:embed dump/date_types.sql - date []byte - //go:embed dump/geom_types.sql - geom []byte - //go:embed dump/numeric_types.sql - numeric []byte - //go:embed dump/text_types.sql - text []byte - //go:embed dump/wtf_types.sql - wtf []byte -) - -var TableSQLs = map[string]string{ - "public.array_types": string(array), - "public.date_types": string(date), - "public.geom_types": string(geom), - "public.numeric_types": string(numeric), - "public.text_types": string(text), - "public.wtf_types": string(wtf), -} diff --git a/tests/canon/postgres/canon_test.go b/tests/canon/postgres/canon_test.go deleted file mode 100644 index a029591d..00000000 --- a/tests/canon/postgres/canon_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package postgres - -import ( - "context" - _ "embed" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/tcrecipes" - "github.com/stretchr/testify/require" -) - -func TestCanonSource(t *testing.T) { - if tcrecipes.Enabled() { - _ = pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump")) - } - _ = os.Setenv("YC", "1") // to not go to vanga - srcPort := helpers.GetIntFromEnv("PG_LOCAL_PORT") - Source := &postgres.PgSource{ - ClusterID: os.Getenv("PG_CLUSTER_ID"), - Hosts: []string{"localhost"}, - User: os.Getenv("PG_LOCAL_USER"), - Password: server.SecretString(os.Getenv("PG_LOCAL_PASSWORD")), - Database: os.Getenv("PG_LOCAL_DATABASE"), - Port: srcPort, - SlotID: "test_slot_id", - } - Source.WithDefaults() - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - }() - - tableCase := func(tableName string) func(t *testing.T) { - return func(t *testing.T) { - conn, err := postgres.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), fmt.Sprintf(`drop table if exists %s`, tableName)) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), TableSQLs[tableName]) - require.NoError(t, err) - - counterStorage, counterSinkFactory := validator.NewCounter() - transfer := helpers.MakeTransfer( - tableName, - Source, - &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(Source), - validator.InitDone(t), - validator.Canonizator(t), - validator.TypesystemChecker(postgres.ProviderType, func(colSchema abstract.ColSchema) string { - return postgres.ClearOriginalType(colSchema.OriginalType) - }), - counterSinkFactory, - ), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotAndIncrement, - ) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{tableName}} - worker := helpers.Activate(t, transfer) - - conn, err = postgres.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), TableSQLs[tableName]) - require.NoError(t, err) - srcStorage, err := postgres.NewStorage(Source.ToStorageParams(transfer)) - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCount(t, strings.Split(tableName, ".")[0], strings.Split(tableName, ".")[1], srcStorage, counterStorage, time.Second*60)) - defer worker.Close(t) - } - } - t.Run("array_types", tableCase("public.array_types")) - t.Run("date_types", tableCase("public.date_types")) - t.Run("geom_types", tableCase("public.geom_types")) - t.Run("numeric_types", tableCase("public.numeric_types")) - t.Run("text_types", tableCase("public.text_types")) - t.Run("wtf_types", tableCase("public.wtf_types")) -} diff --git a/tests/canon/postgres/dump/array_types.sql b/tests/canon/postgres/dump/array_types.sql deleted file mode 100644 index 02da86b8..00000000 --- a/tests/canon/postgres/dump/array_types.sql +++ /dev/null @@ -1,183 +0,0 @@ -DO -$$ -BEGIN - IF NOT EXISTS (SELECT * FROM pg_type typ - INNER JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace - WHERE nsp.nspname = current_schema() AND typ.typname = 'my_enum_type') - THEN - CREATE TYPE my_enum_type AS ENUM - ( - 'VALUE_ONE', - 'VALUE_TWO', - 'VALUE_THREE' - ); - END IF; -END; -$$; - -create table if not exists public.array_types -( - __primary_key serial primary key, - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME3 TIME(3)[], - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ3 TIME(3) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP3 TIMESTAMP(3)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[], - - --ENUM - ARR_ENUM my_enum_type[] -); - -insert into array_types values -( - default, - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123", "04:05:06.123"}', -- TIME3 TIME(3), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575-04", "2020-05-26 13:30:25.575-04"}', -- TIMETZ3 TIME(3) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987", "2004-10-19 10:23:54.987"}', -- TIMESTAMP6 TIMESTAMP(3), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}', -- DECIMAL_5_2 DECIMAL(5,2), - - -- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, - -- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, - -- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, - -- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, - -- '{"Tom","Tom"}' -- CITEXT_ CITEXT - - ARRAY ['VALUE_ONE', 'VALUE_THREE']::my_enum_type[] --ENUM -); diff --git a/tests/canon/postgres/dump/date_types.sql b/tests/canon/postgres/dump/date_types.sql deleted file mode 100644 index 13a2bcd6..00000000 --- a/tests/canon/postgres/dump/date_types.sql +++ /dev/null @@ -1,52 +0,0 @@ -create table if not exists public.date_types -( - __primary_key serial primary key, - - t_timestamptz timestamptz, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a postgresql extension - t_tst timestamp with time zone, - t_timetz timetz, - t_time_with_time_zone_ time with time zone, - t_interval interval, - - t_date date, - t_time time, - t_time_1 time(1), -- precision: this is a fractional digits number placed in the seconds’ field. this can be up to six digits. hh:mm:ss.pppppp - t_time_3 time(3), - t_time_6 time(6), - - t_timetz_1 time(1) with time zone, - t_timetz_3 time(3) with time zone, - t_timetz_6 time(6) with time zone, - - t_timestamp_1 timestamp(1), - t_timestamp_3 timestamp(3), - t_timestamp_6 timestamp(6), - t_timestamp timestamp -); - -insert into public.date_types values -( - default, - '2004-10-19 10:23:54+02', -- TIMESTAMPTZ - - '2004-10-19 11:23:54+02', -- TIMESTAMP WITH TIME ZONE - '00:51:02.746572-08', -- TIMETZ - '00:51:02.746572-08', -- TIME WITH TIME ZONE - interval '1 day 01:00:00', -- interval - - 'January 8, 1999', -- date - - '04:05:06', -- time - '04:05:06.1', -- time(1) - '04:05:06.123', -- time(3) - '04:05:06.123456', -- time(6) - - '2020-05-26 13:30:25.5-04', -- time(1) with time zone - '2020-05-26 13:30:25.575-04', -- time(3) with time zone - '2020-05-26 13:30:25.575401-04', -- time(6) with time zone - - '2004-10-19 10:23:54.9', -- timestamp(1) - '2004-10-19 10:23:54.987', -- timestamp(3) - '2004-10-19 10:23:54.987654', -- timestamp(6) - '2004-10-19 10:23:54' -- timestamp -); diff --git a/tests/canon/postgres/dump/geom_types.sql b/tests/canon/postgres/dump/geom_types.sql deleted file mode 100644 index d2611170..00000000 --- a/tests/canon/postgres/dump/geom_types.sql +++ /dev/null @@ -1,28 +0,0 @@ -create table if not exists public.geom_types -( - __primary_key serial primary key, - - t_point point, - t_line line, - t_lseg lseg, - t_box box, - t_path path, - t_polygon polygon, - t_circle circle -); - - -INSERT INTO public.geom_types VALUES -( - default, - '( 1 , 2 )', - '[ ( 1 , 2 ) , ( 2 , 3 ) ]', - '( ( 1 , 2 ) , ( 2 , 3 ) )', - '( ( 1 , 1 ) , ( 3 , 3 ) )', - '[ ( 1 , 1 ) , ( 2 , 2 ) , ( 2 , 3 ) ]', - '( ( 1 , 1 ) , ( 2 , 2 ) , ( 2 , 3 ), (1 , 1 ) )', - '( ( 1 , 1 ) , 10 )' -); - --- null case -INSERT INTO public.geom_types (__primary_key) VALUES (default); diff --git a/tests/canon/postgres/dump/numeric_types.sql b/tests/canon/postgres/dump/numeric_types.sql deleted file mode 100644 index edb52cf0..00000000 --- a/tests/canon/postgres/dump/numeric_types.sql +++ /dev/null @@ -1,150 +0,0 @@ -create table if not exists public.numeric_types -( - __primary_key serial primary key, - - t_boolean boolean, - - t_smallint smallint, - t_integer integer, - t_bigint bigint, - t_oid oid, - - t_decimal decimal, - t_decimal_5 decimal(5), - t_decimal_5_2 decimal(5, 2), - - t_numeric numeric, - - t_numeric_5 numeric(5), - t_numeric_5_2 numeric(5, 2), - - t_real real, - t_float_4 float4, - t_float_8 float8, - t_float_11 float(11), - - t_double_precision double precision, - t_serial serial, - t_bigserial bigserial, - t_money money -); - -INSERT INTO public.numeric_types VALUES -( - default, - true, -- boolean - - -32768, -- smallint, - -8388605, -- integer, - 1, -- bigint, - 2, -- oid, - 123456, -- decimal - 12345, -- decimal(5) - 123.23, -- decimal(5,2) - - 1267650600228229401496703205376, -- numeric - - 12345, -- numeric(5) - 123.67, -- numeric(5,2) - - 1.45e-45, -- real, MAX PRECISION - 1.45e-45, -- float4, - 3.14e-324, -- float8, - 3.14e-45, -- float11, - 3.14e-324, -- double precision, MAX PRECISION - 0, -- serial, - 3372036854775807, -- bigserial, - 99.23 --money -); - - -INSERT INTO public.numeric_types VALUES -( - default, - true, -- boolean - - 32767, -- smallint, - 8388605, -- integer, - 9223372036854775807, -- bigint, - 2, -- oid, - 123456.123456789012345678901234567890123456789012345678901234567890, -- decimal - 12345, -- decimal(5) - 123.23, -- decimal(5,2) - - 123456.123456789012345678901234567890123456789012345678901234567890, -- numeric - - 12345, -- numeric(5) - 123.67, -- numeric(5,2) - - 123456.123456789012345, -- real, - 123456.123456789012345, -- float4, - 123456.123456789012345, -- float8, - 123456.123456789012345, -- float11, - 123456.123456789012345, -- double precision, - 2147483647, -- serial, - 9223372036854775807, -- bigserial, - 99.23123456 --money -); - -INSERT INTO public.numeric_types -( - t_numeric, - t_numeric_5, - t_numeric_5_2 -) -VALUES -( - 0.0, -- numeric - 0, -- numeric(5) - 0.0 -- numeric(5,2) -); - -INSERT INTO public.numeric_types -( - t_numeric, - t_numeric_5, - t_numeric_5_2 -) -VALUES -( - -123456.123456789012345678901234567890123456789012345678901234567890, -- numeric - -12345, -- numeric(5) - -123.67 -- numeric(5,2) -); - - --- INSERT INTO public.numeric_types --- ( --- t_real, --- t_double_precision --- ) --- VALUES --- ( --- 'Infinity', -- real, --- 'Infinity' -- double precision --- ); - --- INSERT INTO public.numeric_types --- ( --- t_real, --- t_double_precision --- ) --- VALUES --- ( --- '-Infinity', -- real, --- '-Infinity' -- double precision --- ); - --- INSERT INTO public.numeric_types --- ( --- t_real, --- t_double_precision --- ) --- VALUES --- ( --- 'NaN', -- real, --- 'NaN' -- double precision --- ); - --- null case -INSERT INTO public.numeric_types (__primary_key) VALUES (default); diff --git a/tests/canon/postgres/dump/text_types.sql b/tests/canon/postgres/dump/text_types.sql deleted file mode 100644 index c1f663d3..00000000 --- a/tests/canon/postgres/dump/text_types.sql +++ /dev/null @@ -1,36 +0,0 @@ -create table if not exists public.text_types -( - __primary_key serial primary key, - t_text text, - - t_char char, - t_varchar_256 varchar(256), - - t_character_ character(4), - t_character_varying_ character varying(5), - - t_bit_1 bit(1), - t_bit_8 bit(8), - t_varbit_8 varbit(8), - - t_bytea bytea -); - -INSERT INTO public.text_types VALUES -( - default, - 'text_example', -- text - '1', -- char - 'varchar_example', -- varchar(256) - 'abcd', -- CHARACTER(4) - 'varc', -- CHARACTER VARYING(5) - - - b'1', -- bit(1) - b'10101111', -- bit(8), - b'10101110', -- varbit(8) - decode('CAFEBABE', 'hex') -- bytea -); - --- null case -INSERT INTO public.text_types (__primary_key) VALUES (default); diff --git a/tests/canon/postgres/dump/wtf_types.sql b/tests/canon/postgres/dump/wtf_types.sql deleted file mode 100644 index 26a953e9..00000000 --- a/tests/canon/postgres/dump/wtf_types.sql +++ /dev/null @@ -1,36 +0,0 @@ -create extension if not exists hstore; -create extension if not exists ltree; -create extension if not exists citext; - -create table if not exists public.wtf_types -( - __primary_key serial primary key, - - t_hstore hstore, - t_iner inet, - t_cidr cidr, - t_macaddr macaddr, - -- macaddr8 not supported by postgresql 9.6 (which is in our recipes) - -- ltree - should be in special table, i suppose - t_citext citext, - - j json, - jb jsonb -); - -INSERT INTO public.wtf_types VALUES -( - default, - - 'a=>1,b=>2', -- t_hstore - '192.168.1.5', -- t_iner - '10.1/16', -- t_cidr - '08:00:2b:01:02:03', -- t_macaddr - 'Tom', -- t_citext - - '{"k": "v", "ki": 42, "kf": 1.2, "kn": null, "ks": "Ho Ho Ho my name''s \"SANTA CLAWS\""}', -- j json - '"\"String in quotes\""' -- jb jsonb -); - --- null case -INSERT INTO public.wtf_types (__primary_key) VALUES (default); diff --git a/tests/canon/postgres/gotest/canondata/result.json b/tests/canon/postgres/gotest/canondata/result.json deleted file mode 100644 index bdac9c57..00000000 --- a/tests/canon/postgres/gotest/canondata/result.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "gotest.gotest.TestCanonSource/array_types/canon_0#01": { - "uri": "file://gotest.gotest.TestCanonSource_array_types_canon_0#01/extracted" - }, - "gotest.gotest.TestCanonSource/array_types/canon_0#03": { - "uri": "file://gotest.gotest.TestCanonSource_array_types_canon_0#03/extracted" - }, - "gotest.gotest.TestCanonSource/date_types/canon_0#01": { - "uri": "file://gotest.gotest.TestCanonSource_date_types_canon_0#01/extracted" - }, - "gotest.gotest.TestCanonSource/date_types/canon_0#03": { - "uri": "file://gotest.gotest.TestCanonSource_date_types_canon_0#03/extracted" - }, - "gotest.gotest.TestCanonSource/geom_types/canon_0#01": { - "uri": "file://gotest.gotest.TestCanonSource_geom_types_canon_0#01/extracted" - }, - "gotest.gotest.TestCanonSource/geom_types/canon_0#03": { - "uri": "file://gotest.gotest.TestCanonSource_geom_types_canon_0#03/extracted" - }, - "gotest.gotest.TestCanonSource/numeric_types/canon_0#01": { - "uri": "file://gotest.gotest.TestCanonSource_numeric_types_canon_0#01/extracted" - }, - "gotest.gotest.TestCanonSource/numeric_types/canon_0#03": { - "uri": "file://gotest.gotest.TestCanonSource_numeric_types_canon_0#03/extracted" - }, - "gotest.gotest.TestCanonSource/text_types/canon_0#01": { - "uri": "file://gotest.gotest.TestCanonSource_text_types_canon_0#01/extracted" - }, - "gotest.gotest.TestCanonSource/text_types/canon_0#03": { - "uri": "file://gotest.gotest.TestCanonSource_text_types_canon_0#03/extracted" - }, - "gotest.gotest.TestCanonSource/wtf_types/canon_0#01": { - "uri": "file://gotest.gotest.TestCanonSource_wtf_types_canon_0#01/extracted" - }, - "gotest.gotest.TestCanonSource/wtf_types/canon_0#03": { - "uri": "file://gotest.gotest.TestCanonSource_wtf_types_canon_0#03/extracted" - } -} diff --git a/tests/canon/s3/csv/canon_test.go b/tests/canon/s3/csv/canon_test.go deleted file mode 100644 index 14168fa6..00000000 --- a/tests/canon/s3/csv/canon_test.go +++ /dev/null @@ -1,338 +0,0 @@ -package csv - -import ( - _ "embed" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/s3" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - - testCasePath := "test_csv_all_types" - src := s3.PrepareCfg(t, "", "") - src.PathPrefix = testCasePath - if os.Getenv("S3MDS_PORT") != "" { // for local recipe we need to upload test case to internet - src.Bucket = "data4" - s3.CreateBucket(t, src) - s3.PrepareTestCase(t, src, src.PathPrefix) - logger.Log.Info("dir uploaded") - } - src.TableNamespace = "test" - src.TableName = "types" - src.Format.CSVSetting = new(s3.CSVSetting) - src.Format.CSVSetting.BlockSize = 1 * 1024 * 1024 - src.Format.CSVSetting.QuoteChar = "\"" - src.InputFormat = server.ParsingFormatCSV - src.WithDefaults() - src.HideSystemCols = true - src.OutputSchema = []abstract.ColSchema{ - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "0", - DataType: schema.TypeBoolean.String(), - ColumnName: "boolean", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "1", - DataType: schema.TypeUint8.String(), - ColumnName: "uint8", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "2", - DataType: schema.TypeUint16.String(), - ColumnName: "uint16", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "3", - DataType: schema.TypeUint32.String(), - ColumnName: "uint32", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "4", - DataType: schema.TypeUint64.String(), - ColumnName: "uint64", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "5", - DataType: schema.TypeInt8.String(), - ColumnName: "int8", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "6", - DataType: schema.TypeInt16.String(), - ColumnName: "int16", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "7", - DataType: schema.TypeInt32.String(), - ColumnName: "int32", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "8", - DataType: schema.TypeInt64.String(), - ColumnName: "int64", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "9", - DataType: schema.TypeFloat32.String(), - ColumnName: "float32", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "10", - DataType: schema.TypeFloat64.String(), - ColumnName: "float64", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "11", - DataType: schema.TypeBytes.String(), - ColumnName: "bytes", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "12", - DataType: schema.TypeString.String(), - ColumnName: "string", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "13", - DataType: schema.TypeDate.String(), - ColumnName: "date", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "14", - DataType: schema.TypeDatetime.String(), - ColumnName: "dateTime", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "15", - DataType: schema.TypeTimestamp.String(), - ColumnName: "timestamp", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "16", - DataType: schema.TypeInterval.String(), - ColumnName: "interval", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "17", - DataType: schema.TypeAny.String(), - ColumnName: "any", - }, - } - transfer := helpers.MakeTransfer( - helpers.TransferID, - src, - &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(src), - validator.InitDone(t), - validator.Referencer(t), - validator.TypesystemChecker(s3.ProviderType, func(colSchema abstract.ColSchema) string { - return colSchema.OriginalType - }), - ), - Cleanup: server.Drop, - }, - abstract.TransferTypeSnapshotOnly, - ) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - time.Sleep(1 * time.Second) -} - -var processed []abstract.ChangeItem - -func TestNativeS3WithProvidedSchemaAndSystemCols(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - - processed = make([]abstract.ChangeItem, 0) - testCasePath := "test_csv_all_types" - src := s3.PrepareCfg(t, "", "") - src.PathPrefix = testCasePath - if os.Getenv("S3MDS_PORT") != "" { // for local recipe we need to upload test case to internet - src.Bucket = "data4" - s3.CreateBucket(t, src) - s3.PrepareTestCase(t, src, src.PathPrefix) - logger.Log.Info("dir uploaded") - } - src.TableNamespace = "test" - src.TableName = "types" - src.Format.CSVSetting = new(s3.CSVSetting) - src.Format.CSVSetting.QuoteChar = "\"" - src.InputFormat = server.ParsingFormatCSV - src.WithDefaults() - src.Format.CSVSetting.BlockSize = 1 * 1024 * 1024 - - src.HideSystemCols = false - src.OutputSchema = []abstract.ColSchema{ - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "0", - DataType: schema.TypeBoolean.String(), - ColumnName: "boolean", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "1", - DataType: schema.TypeUint8.String(), - ColumnName: "uint8", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "2", - DataType: schema.TypeUint16.String(), - ColumnName: "uint16", - }, - } - - transfer := helpers.MakeTransfer(helpers.TransferID, src, &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(src), - validator.Canonizator(t, storeItems), - ), - Cleanup: server.DisabledCleanup, - }, abstract.TransferTypeSnapshotOnly) - - helpers.Activate(t, transfer) - - require.Len(t, processed, 3) - - sampleColumns := processed[0].ColumnNames - require.Len(t, sampleColumns, 5) // contains system columns appended at the end - require.Equal(t, "__file_name", sampleColumns[0]) - require.Equal(t, "__row_index", sampleColumns[1]) -} - -func storeItems(item []abstract.ChangeItem) []abstract.ChangeItem { - processed = append(processed, item...) - return item -} - -func TestNativeS3MissingColumnsAreFilled(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - - processed = make([]abstract.ChangeItem, 0) - testCasePath := "test_csv_all_types" - src := s3.PrepareCfg(t, "", "") - src.PathPrefix = testCasePath - if os.Getenv("S3MDS_PORT") != "" { // for local recipe we need to upload test case to internet - src.Bucket = "data4" - s3.CreateBucket(t, src) - s3.PrepareTestCase(t, src, src.PathPrefix) - logger.Log.Info("dir uploaded") - } - src.TableNamespace = "test" - src.TableName = "types" - src.Format.CSVSetting = new(s3.CSVSetting) - - src.InputFormat = server.ParsingFormatCSV - src.WithDefaults() - src.Format.CSVSetting.BlockSize = 1 * 1024 * 1024 - src.Format.CSVSetting.QuoteChar = "\"" - src.Format.CSVSetting.AdditionalReaderOptions.IncludeMissingColumns = true - src.HideSystemCols = true - src.OutputSchema = []abstract.ColSchema{ - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "0", - DataType: schema.TypeBoolean.String(), - ColumnName: "boolean", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "1", - DataType: schema.TypeUint8.String(), - ColumnName: "uint8", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "20", - DataType: schema.TypeString.String(), - ColumnName: "test_missing_column_string", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "21", - DataType: schema.TypeInt8.String(), - ColumnName: "test_missing_column_int", - }, - { - TableSchema: src.TableNamespace, - TableName: src.TableName, - Path: "22", - DataType: schema.TypeBoolean.String(), - ColumnName: "test_missing_column_bool", - }, - } - - transfer := helpers.MakeTransfer(helpers.TransferID, src, &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(src), - validator.Canonizator(t, storeItems), - ), - Cleanup: server.DisabledCleanup, - }, abstract.TransferTypeSnapshotOnly) - - helpers.Activate(t, transfer) - - require.Len(t, processed, 3) - - sampleColumnValues := processed[0].ColumnValues - require.Len(t, sampleColumnValues, 5) // contains system columns appended at the end - require.Equal(t, "", sampleColumnValues[2]) - require.Equal(t, int8(0), sampleColumnValues[3]) - require.Equal(t, false, sampleColumnValues[4]) -} diff --git a/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted b/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted deleted file mode 100644 index 3314a92e..00000000 --- a/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted +++ /dev/null @@ -1,461 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "boolean", - "uint8", - "test_missing_column_string", - "test_missing_column_int", - "test_missing_column_bool" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "bool", - "value": false - }, - { - "type": "uint8", - "value": 0 - }, - { - "type": "string", - "value": "" - }, - { - "type": "int8", - "value": 0 - }, - { - "type": "bool", - "value": false - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "test" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 140, - "Values": 124 - } - }, - "Table": { - "type": "string", - "value": "types" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "test", - "table_name": "types", - "path": "0", - "name": "boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "1", - "name": "uint8", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "20", - "name": "test_missing_column_string", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:utf8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "21", - "name": "test_missing_column_int", - "type": "int8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:int8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "22", - "name": "test_missing_column_bool", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "boolean", - "uint8", - "test_missing_column_string", - "test_missing_column_int", - "test_missing_column_bool" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "bool", - "value": true - }, - { - "type": "uint8", - "value": 0 - }, - { - "type": "string", - "value": "" - }, - { - "type": "int8", - "value": 0 - }, - { - "type": "bool", - "value": false - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "test" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 140, - "Values": 124 - } - }, - "Table": { - "type": "string", - "value": "types" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "test", - "table_name": "types", - "path": "0", - "name": "boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "1", - "name": "uint8", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "20", - "name": "test_missing_column_string", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:utf8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "21", - "name": "test_missing_column_int", - "type": "int8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:int8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "22", - "name": "test_missing_column_bool", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "boolean", - "uint8", - "test_missing_column_string", - "test_missing_column_int", - "test_missing_column_bool" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "bool", - "value": false - }, - { - "type": "uint8", - "value": 0 - }, - { - "type": "string", - "value": "" - }, - { - "type": "int8", - "value": 0 - }, - { - "type": "bool", - "value": false - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "test" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 140, - "Values": 124 - } - }, - "Table": { - "type": "string", - "value": "types" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "test", - "table_name": "types", - "path": "0", - "name": "boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "1", - "name": "uint8", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "20", - "name": "test_missing_column_string", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:utf8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "21", - "name": "test_missing_column_int", - "type": "int8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:int8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "22", - "name": "test_missing_column_bool", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted b/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted deleted file mode 100644 index 6b97a85a..00000000 --- a/tests/canon/s3/csv/canondata/csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted +++ /dev/null @@ -1,461 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "boolean", - "uint8", - "uint16" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "bool", - "value": false - }, - { - "type": "uint8", - "value": 0 - }, - { - "type": "uint16", - "value": 0 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "test" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 195, - "Values": 164 - } - }, - "Table": { - "type": "string", - "value": "types" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "0", - "name": "boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "1", - "name": "uint8", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "2", - "name": "uint16", - "type": "uint16", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint16" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "boolean", - "uint8", - "uint16" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "bool", - "value": true - }, - { - "type": "uint8", - "value": 0 - }, - { - "type": "uint16", - "value": 0 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "test" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 195, - "Values": 164 - } - }, - "Table": { - "type": "string", - "value": "types" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "0", - "name": "boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "1", - "name": "uint8", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "2", - "name": "uint16", - "type": "uint16", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint16" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "boolean", - "uint8", - "uint16" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "bool", - "value": false - }, - { - "type": "uint8", - "value": 0 - }, - { - "type": "uint16", - "value": 0 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 0 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "test_csv_all_types/all_types.csv" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "test" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 195, - "Values": 164 - } - }, - "Table": { - "type": "string", - "value": "types" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "0", - "name": "boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:boolean" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "1", - "name": "uint8", - "type": "uint8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint8" - }, - { - "table_schema": "test", - "table_name": "types", - "path": "2", - "name": "uint16", - "type": "uint16", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "csv:uint16" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/csv/canondata/result.json b/tests/canon/s3/csv/canondata/result.json deleted file mode 100644 index 25bd940f..00000000 --- a/tests/canon/s3/csv/canondata/result.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "csv.csv.TestCanonSource/test.types": { - "Rows": [ - { - "Data": { - "any": { - "GoType": "string", - "Val": "\"{\\\"A2\\\": {\\\"a\\\": \\\"321\\\"}, \\\"B2\\\": {\\\"b1\\\": \\\"654\\\", \\\"b2\\\": \\\"987\\\"}}\"" - }, - "boolean": { - "GoType": "bool", - "Val": false - }, - "bytes": { - "GoType": "[]uint8", - "Val": "WzcyIDEwMSAxMDggMTA4IDExMV0=" - }, - "date": { - "GoType": "time.Time", - "Val": "2016-02-01T00:00:00Z" - }, - "dateTime": { - "GoType": "time.Time", - "Val": "2001-01-01T05:30:00Z" - }, - "float32": { - "GoType": "float32", - "Val": 1.1920929e-07 - }, - "float64": { - "GoType": "json.Number", - "Val": 0.3333333333333333 - }, - "int16": { - "GoType": "int16", - "Val": -2 - }, - "int32": { - "GoType": "int32", - "Val": -3 - }, - "int64": { - "GoType": "int64", - "Val": -4 - }, - "int8": { - "GoType": "int8", - "Val": -1 - }, - "interval": { - "GoType": "time.Duration", - "Val": 5000000000 - }, - "string": { - "GoType": "string", - "Val": "\"This is a test\"" - }, - "timestamp": { - "GoType": "time.Time", - "Val": "2023-07-04T09:30:40Z" - }, - "uint16": { - "GoType": "uint16", - "Val": 0 - }, - "uint32": { - "GoType": "uint32", - "Val": 0 - }, - "uint64": { - "GoType": "uint64", - "Val": 0 - }, - "uint8": { - "GoType": "uint8", - "Val": 0 - } - } - }, - { - "Data": { - "any": { - "GoType": "string", - "Val": "\"{\\\"A2\\\": {\\\"a\\\": \\\"321\\\"}, \\\"B2\\\": {\\\"b1\\\": \\\"654\\\", \\\"b2\\\": \\\"987\\\"}}\"" - }, - "boolean": { - "GoType": "bool", - "Val": true - }, - "bytes": { - "GoType": "[]uint8", - "Val": "WzcyIDEwMSAxMDggMTA4IDExMV0=" - }, - "date": { - "GoType": "time.Time", - "Val": "2015-02-01T00:00:00Z" - }, - "dateTime": { - "GoType": "time.Time", - "Val": "2005-01-01T15:30:00Z" - }, - "float32": { - "GoType": "float32", - "Val": 1.1920929e-07 - }, - "float64": { - "GoType": "json.Number", - "Val": 0.3333333333333333 - }, - "int16": { - "GoType": "int16", - "Val": -2 - }, - "int32": { - "GoType": "int32", - "Val": -3 - }, - "int64": { - "GoType": "int64", - "Val": -4 - }, - "int8": { - "GoType": "int8", - "Val": -1 - }, - "interval": { - "GoType": "time.Duration", - "Val": 18000000000000 - }, - "string": { - "GoType": "string", - "Val": "\"This is a test 2\"" - }, - "timestamp": { - "GoType": "time.Time", - "Val": "2023-07-04T09:30:40Z" - }, - "uint16": { - "GoType": "uint16", - "Val": 0 - }, - "uint32": { - "GoType": "uint32", - "Val": 0 - }, - "uint64": { - "GoType": "uint64", - "Val": 0 - }, - "uint8": { - "GoType": "uint8", - "Val": 0 - } - } - }, - { - "Data": { - "any": { - "GoType": "string", - "Val": "\"[\\\"a\\\", \\\"b\\\"]\"" - }, - "boolean": { - "GoType": "bool", - "Val": false - }, - "bytes": { - "GoType": "[]uint8", - "Val": "WzcyIDEwMSAxMDggMTA4IDExMV0=" - }, - "date": { - "GoType": "time.Time", - "Val": "2017-02-01T00:00:00Z" - }, - "dateTime": { - "GoType": "time.Time", - "Val": "2000-01-01T23:30:00Z" - }, - "float32": { - "GoType": "float32", - "Val": 1.1920929e-07 - }, - "float64": { - "GoType": "json.Number", - "Val": 0.3333333333333333 - }, - "int16": { - "GoType": "int16", - "Val": -2 - }, - "int32": { - "GoType": "int32", - "Val": -3 - }, - "int64": { - "GoType": "int64", - "Val": -4 - }, - "int8": { - "GoType": "int8", - "Val": -1 - }, - "interval": { - "GoType": "time.Duration", - "Val": 300000000000 - }, - "string": { - "GoType": "string", - "Val": "\"This is a test 3\"" - }, - "timestamp": { - "GoType": "time.Time", - "Val": "2023-07-04T09:30:40Z" - }, - "uint16": { - "GoType": "uint16", - "Val": 0 - }, - "uint32": { - "GoType": "uint32", - "Val": 0 - }, - "uint64": { - "GoType": "uint64", - "Val": 0 - }, - "uint8": { - "GoType": "uint8", - "Val": 0 - } - } - } - ], - "TableID": { - "Name": "types", - "Namespace": "test" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": false, - "name": "boolean", - "original_type": "csv:boolean", - "path": "0", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "uint8", - "original_type": "csv:uint8", - "path": "1", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "uint8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "uint16", - "original_type": "csv:uint16", - "path": "2", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "uint16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "uint32", - "original_type": "csv:uint32", - "path": "3", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "uint32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "uint64", - "original_type": "csv:uint64", - "path": "4", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int8", - "original_type": "csv:int8", - "path": "5", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "int8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int16", - "original_type": "csv:int16", - "path": "6", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "int16" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int32", - "original_type": "csv:int32", - "path": "7", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "int32" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "int64", - "original_type": "csv:int64", - "path": "8", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "int64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "float32", - "original_type": "csv:float", - "path": "9", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "float" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "float64", - "original_type": "csv:double", - "path": "10", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "bytes", - "original_type": "csv:string", - "path": "11", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "string" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "string", - "original_type": "csv:utf8", - "path": "12", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "date", - "original_type": "csv:date", - "path": "13", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "date" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "dateTime", - "original_type": "csv:datetime", - "path": "14", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "datetime" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "timestamp", - "original_type": "csv:timestamp", - "path": "15", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "timestamp" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "interval", - "original_type": "csv:interval", - "path": "16", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "interval" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "any", - "original_type": "csv:any", - "path": "17", - "required": false, - "table_name": "types", - "table_schema": "test", - "type": "any" - } - ] - }, - "csv.csv.TestNativeS3MissingColumnsAreFilled/canon_0#01": { - "uri": "file://csv.csv.TestNativeS3MissingColumnsAreFilled_canon_0#01/extracted" - }, - "csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols/canon_0#01": { - "uri": "file://csv.csv.TestNativeS3WithProvidedSchemaAndSystemCols_canon_0#01/extracted" - } -} diff --git a/tests/canon/s3/jsonline/canon_test.go b/tests/canon/s3/jsonline/canon_test.go deleted file mode 100644 index 939835a6..00000000 --- a/tests/canon/s3/jsonline/canon_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package jsonline - -import ( - _ "embed" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/s3" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" -) - -func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - - testCasePath := "test_jsonline_all_types" - src := s3.PrepareCfg(t, "", "") - src.PathPrefix = testCasePath - if os.Getenv("S3MDS_PORT") != "" { // for local recipe we need to upload test case to internet - src.Bucket = "data4" - s3.CreateBucket(t, src) - s3.PrepareTestCase(t, src, src.PathPrefix) - logger.Log.Info("dir uploaded") - } - src.TableNamespace = "test" - src.TableName = "types" - src.InputFormat = server.ParsingFormatJSONLine - src.WithDefaults() - src.Format.JSONLSetting.BlockSize = 1 * 1024 * 1024 - src.HideSystemCols = false - - src.OutputSchema = []abstract.ColSchema{ - { - ColumnName: "array", - OriginalType: "jsonl:array", - DataType: "any", - }, - { - ColumnName: "boolean", - OriginalType: "jsonl:boolean", - DataType: "boolean", - }, - { - ColumnName: "date", - OriginalType: "jsonl:string", - DataType: "utf8", - }, - { - ColumnName: "id", - OriginalType: "jsonl:number", - DataType: "double", - }, - { - ColumnName: "name", - OriginalType: "jsonl:string", - DataType: "utf8", - }, - { - ColumnName: "object", - OriginalType: "jsonl:object", - DataType: "any", - }, - { - ColumnName: "rest", - OriginalType: "jsonl:object", - DataType: "any", - }, - } - - transfer := helpers.MakeTransfer( - helpers.TransferID, - src, - &server.MockDestination{ - SinkerFactory: validator.New( - server.IsStrictSource(src), - validator.InitDone(t), - validator.Referencer(t), - validator.TypesystemChecker(s3.ProviderType, func(colSchema abstract.ColSchema) string { - return colSchema.OriginalType - }), - ), - Cleanup: server.Drop, - }, - abstract.TransferTypeSnapshotOnly, - ) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - time.Sleep(1 * time.Second) -} diff --git a/tests/canon/s3/jsonline/canondata/result.json b/tests/canon/s3/jsonline/canondata/result.json deleted file mode 100644 index c51a4f69..00000000 --- a/tests/canon/s3/jsonline/canondata/result.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "jsonline.jsonline.TestCanonSource/test.types": { - "Rows": [ - { - "Data": { - "__file_name": { - "GoType": "string", - "Val": "test_jsonline_all_types/test_jsonline_all_types.jsonl" - }, - "__row_index": { - "GoType": "uint64", - "Val": 1 - }, - "array": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - }, - "boolean": { - "GoType": "bool", - "Val": true - }, - "date": { - "GoType": "string", - "Val": "2023-07-06 10:27:42.023151056 +0200 CEST m=+0.001546549" - }, - "id": { - "GoType": "json.Number", - "Val": 0 - }, - "name": { - "GoType": "string", - "Val": "test0" - }, - "object": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - }, - "rest": { - "GoType": "map[string]interface {}", - "Val": { - "unknown": "unknown_0" - } - } - } - }, - { - "Data": { - "__file_name": { - "GoType": "string", - "Val": "test_jsonline_all_types/test_jsonline_all_types.jsonl" - }, - "__row_index": { - "GoType": "uint64", - "Val": 2 - }, - "array": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - }, - "boolean": { - "GoType": "bool", - "Val": true - }, - "date": { - "GoType": "string", - "Val": "2023-07-06 10:27:42.023151056 +0200 CEST m=+0.001546549" - }, - "id": { - "GoType": "json.Number", - "Val": 1 - }, - "name": { - "GoType": "string", - "Val": "test1" - }, - "object": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - }, - "rest": { - "GoType": "map[string]interface {}", - "Val": { - "unknown": "unknown_1" - } - } - } - }, - { - "Data": { - "__file_name": { - "GoType": "string", - "Val": "test_jsonline_all_types/test_jsonline_all_types.jsonl" - }, - "__row_index": { - "GoType": "uint64", - "Val": 3 - }, - "array": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - }, - "boolean": { - "GoType": "bool", - "Val": true - }, - "date": { - "GoType": "string", - "Val": "2023-07-06 10:27:42.023151056 +0200 CEST m=+0.001546549" - }, - "id": { - "GoType": "json.Number", - "Val": 2 - }, - "name": { - "GoType": "string", - "Val": "test2" - }, - "object": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - }, - "rest": { - "GoType": "map[string]interface {}", - "Val": { - "unknown": "unknown_2" - } - } - } - }, - { - "Data": { - "__file_name": { - "GoType": "string", - "Val": "test_jsonline_all_types/test_jsonline_all_types.jsonl" - }, - "__row_index": { - "GoType": "uint64", - "Val": 4 - }, - "array": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - }, - "boolean": { - "GoType": "bool", - "Val": true - }, - "date": { - "GoType": "string", - "Val": "2023-07-06 10:27:42.023151056 +0200 CEST m=+0.001546549" - }, - "id": { - "GoType": "json.Number", - "Val": 3 - }, - "name": { - "GoType": "string", - "Val": "test3" - }, - "object": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - }, - "rest": { - "GoType": "map[string]interface {}", - "Val": { - "unknown": "unknown_3" - } - } - } - }, - { - "Data": { - "__file_name": { - "GoType": "string", - "Val": "test_jsonline_all_types/test_jsonline_all_types.jsonl" - }, - "__row_index": { - "GoType": "uint64", - "Val": 5 - }, - "array": { - "GoType": "[]interface {}", - "Val": [ - "a", - "b" - ] - }, - "boolean": { - "GoType": "bool", - "Val": true - }, - "date": { - "GoType": "string", - "Val": "2023-07-06 10:27:42.023151056 +0200 CEST m=+0.001546549" - }, - "id": { - "GoType": "json.Number", - "Val": 4 - }, - "name": { - "GoType": "string", - "Val": "test4" - }, - "object": { - "GoType": "map[string]interface {}", - "Val": { - "a": "b" - } - }, - "rest": { - "GoType": "map[string]interface {}", - "Val": { - "unknown": "unknown_4" - } - } - } - } - ], - "TableID": { - "Name": "types", - "Namespace": "test" - }, - "TableSchema": [ - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__file_name", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": true, - "name": "__row_index", - "original_type": "", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "uint64" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "array", - "original_type": "jsonl:array", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "boolean", - "original_type": "jsonl:boolean", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "boolean" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "date", - "original_type": "jsonl:string", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "id", - "original_type": "jsonl:number", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "double" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "name", - "original_type": "jsonl:string", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "utf8" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "object", - "original_type": "jsonl:object", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - }, - { - "expression": "", - "fake_key": false, - "key": false, - "name": "rest", - "original_type": "jsonl:object", - "path": "", - "required": false, - "table_name": "", - "table_schema": "", - "type": "any" - } - ] - } -} diff --git a/tests/canon/s3/parquet/canon_test.go b/tests/canon/s3/parquet/canon_test.go deleted file mode 100644 index dde25622..00000000 --- a/tests/canon/s3/parquet/canon_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package parquet - -import ( - _ "embed" - "os" - "path/filepath" - "regexp" - "strings" - "testing" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/s3" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestUnsopportedData(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - absPath, err := filepath.Abs("unsupported_data") - require.NoError(t, err) - files, err := os.ReadDir(absPath) - require.NoError(t, err) - src := s3.PrepareCfg(t, "canon-parquet-bad", "") - testCasePath := "data" - src.PathPrefix = testCasePath - s3.CreateBucket(t, src) - s3.PrepareTestCase(t, src, "data") - for _, file := range files { - t.Run(file.Name(), func(t *testing.T) { - src.TableNamespace = "s3_source_parquet" - src.TableName = file.Name() - src.InputFormat = server.ParsingFormatPARQUET - src.PathPattern = "data/" + file.Name() - src.WithDefaults() - - transfer := helpers.MakeTransfer( - helpers.TransferID, - src, - &server.MockDestination{ - SinkerFactory: validator.New(server.IsStrictSource(src)), - Cleanup: server.Drop, - }, - abstract.TransferTypeSnapshotOnly, - ) - _, err = helpers.ActivateErr(transfer) - require.Error(t, err) - }) - } -} - -// rowsCutter will limit number of rows pushed to child sink -type rowsCutter struct { - sink abstract.Sinker - pushed bool -} - -func (r *rowsCutter) Close() error { - if !r.pushed { - return xerrors.New("where is my data Lebovsky?") - } - return r.sink.Close() -} - -func (r *rowsCutter) Push(items []abstract.ChangeItem) error { - var filteredRows []abstract.ChangeItem - for _, row := range items { - if row.IsRowEvent() { - filteredRows = append(filteredRows, row) - } - } - if len(filteredRows) == 0 { - return nil - } - r.pushed = true - if len(filteredRows) > 3 { - return r.sink.Push(filteredRows[:3]) // funny cat face :3 - } - return r.sink.Push(filteredRows) -} - -func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - absPath, err := filepath.Abs("data") - require.NoError(t, err) - files, err := os.ReadDir(absPath) - require.NoError(t, err) - src := s3.PrepareCfg(t, "canon-parquet", "") - testCasePath := "data" - src.PathPrefix = testCasePath - s3.CreateBucket(t, src) - s3.PrepareTestCase(t, src, "data") - - for _, file := range files { - t.Run(file.Name(), func(t *testing.T) { - src.TableNamespace = "s3_source_parquet" - src.TableName = file.Name() - src.InputFormat = server.ParsingFormatPARQUET - src.PathPattern = "data/" + file.Name() - src.WithDefaults() - - transfer := helpers.MakeTransfer( - helpers.TransferID, - src, - &server.MockDestination{ - SinkerFactory: func() abstract.Sinker { - return &rowsCutter{ - sink: validator.New( - server.IsStrictSource(src), - validator.InitDone(t), - validator.ValuesTypeChecker, - validator.Canonizator(t), - validator.TypesystemChecker(s3.ProviderType, func(colSchema abstract.ColSchema) string { - clearType := strings.ReplaceAll(colSchema.OriginalType, "optional", "") - re := regexp.MustCompile(`\(.*\)$`) // Matches the last parenthesis and its contents - return re.ReplaceAllString(clearType, "") - }), - )(), - } - }, - Cleanup: server.Drop, - }, - abstract.TransferTypeSnapshotOnly, - ) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - }) - } -} diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted deleted file mode 100644 index 4b3738a9..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted +++ /dev/null @@ -1,580 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_dictionary.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "bool", - "value": true - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "float32", - "value": 0 - }, - { - "type": "float64", - "value": 0 - }, - { - "type": "string", - "value": "01/01/09" - }, - { - "type": "string", - "value": "0" - }, - { - "type": "string", - "value": "45283676094696639722160128" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_dictionary.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 408, - "Values": 408 - } - }, - "Table": { - "type": "string", - "value": "alltypes_dictionary.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_dictionary.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "bool", - "value": false - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int64", - "value": 10 - }, - { - "type": "float32", - "value": 1.1 - }, - { - "type": "float64", - "value": 10.1 - }, - { - "type": "string", - "value": "01/01/09" - }, - { - "type": "string", - "value": "1" - }, - { - "type": "string", - "value": "45283676094696699722160128" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_dictionary.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 408, - "Values": 408 - } - }, - "Table": { - "type": "string", - "value": "alltypes_dictionary.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted deleted file mode 100644 index f0bef20d..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted +++ /dev/null @@ -1,869 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_plain.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 4 - }, - { - "type": "bool", - "value": true - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "float32", - "value": 0 - }, - { - "type": "float64", - "value": 0 - }, - { - "type": "string", - "value": "03/01/09" - }, - { - "type": "string", - "value": "0" - }, - { - "type": "string", - "value": "45284764452596988585705472" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_plain.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 403, - "Values": 403 - } - }, - "Table": { - "type": "string", - "value": "alltypes_plain.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_plain.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 5 - }, - { - "type": "bool", - "value": false - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int64", - "value": 10 - }, - { - "type": "float32", - "value": 1.1 - }, - { - "type": "float64", - "value": 10.1 - }, - { - "type": "string", - "value": "03/01/09" - }, - { - "type": "string", - "value": "1" - }, - { - "type": "string", - "value": "45284764452597048585705472" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_plain.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 403, - "Values": 403 - } - }, - "Table": { - "type": "string", - "value": "alltypes_plain.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_plain.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int32", - "value": 6 - }, - { - "type": "bool", - "value": true - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "float32", - "value": 0 - }, - { - "type": "float64", - "value": 0 - }, - { - "type": "string", - "value": "04/01/09" - }, - { - "type": "string", - "value": "0" - }, - { - "type": "string", - "value": "45285336301663273581805568" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_plain.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 403, - "Values": 403 - } - }, - "Table": { - "type": "string", - "value": "alltypes_plain.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted deleted file mode 100644 index a241d13b..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted +++ /dev/null @@ -1,580 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_plain.snappy.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 6 - }, - { - "type": "bool", - "value": true - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int32", - "value": 0 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "float32", - "value": 0 - }, - { - "type": "float64", - "value": 0 - }, - { - "type": "string", - "value": "04/01/09" - }, - { - "type": "string", - "value": "0" - }, - { - "type": "string", - "value": "45285336301663273581805568" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_plain.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 410, - "Values": 410 - } - }, - "Table": { - "type": "string", - "value": "alltypes_plain.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "bool_col", - "tinyint_col", - "smallint_col", - "int_col", - "bigint_col", - "float_col", - "double_col", - "date_string_col", - "string_col", - "timestamp_col" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/alltypes_plain.snappy.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 7 - }, - { - "type": "bool", - "value": false - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int64", - "value": 10 - }, - { - "type": "float32", - "value": 1.1 - }, - { - "type": "float64", - "value": 10.1 - }, - { - "type": "string", - "value": "04/01/09" - }, - { - "type": "string", - "value": "1" - }, - { - "type": "string", - "value": "45285336301663333581805568" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/alltypes_plain.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 410, - "Values": 410 - } - }, - "Table": { - "type": "string", - "value": "alltypes_plain.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bool_col", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "tinyint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "smallint_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_col", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bigint_col", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "float_col", - "type": "float", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FLOAT" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "double_col", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "date_string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "string_col", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "timestamp_col", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT96" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted deleted file mode 100644 index 0a3a1ef2..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "foo" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/binary.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "\u0000" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/binary.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 132, - "Values": 132 - } - }, - "Table": { - "type": "string", - "value": "binary.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "foo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "foo" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/binary.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "\u0001" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/binary.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 132, - "Values": 132 - } - }, - "Table": { - "type": "string", - "value": "binary.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "foo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "foo" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/binary.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "\u0002" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/binary.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 132, - "Values": 132 - } - }, - "Table": { - "type": "string", - "value": "binary.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "foo", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted deleted file mode 100644 index 8de5c54e..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/byte_array_decimal.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "d" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/byte_array_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 144, - "Values": 144 - } - }, - "Table": { - "type": "string", - "value": "byte_array_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/byte_array_decimal.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "\u0000\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/byte_array_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 145, - "Values": 145 - } - }, - "Table": { - "type": "string", - "value": "byte_array_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/byte_array_decimal.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "\u0001," - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/byte_array_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 145, - "Values": 145 - } - }, - "Table": { - "type": "string", - "value": "byte_array_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted deleted file mode 100644 index e4443d1d..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "String" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/data_index_bloom_encoding_stats.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "Hello" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/data_index_bloom_encoding_stats.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 161, - "Values": 161 - } - }, - "Table": { - "type": "string", - "value": "data_index_bloom_encoding_stats.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "String", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "String" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/data_index_bloom_encoding_stats.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "This is" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/data_index_bloom_encoding_stats.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 163, - "Values": 163 - } - }, - "Table": { - "type": "string", - "value": "data_index_bloom_encoding_stats.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "String", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "String" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/data_index_bloom_encoding_stats.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "a" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/data_index_bloom_encoding_stats.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 157, - "Values": 157 - } - }, - "Table": { - "type": "string", - "value": "data_index_bloom_encoding_stats.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "String", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted deleted file mode 100644 index 0d3dd192..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted +++ /dev/null @@ -1,596 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b", - "c", - "d", - "e" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/datapage_v2.snappy.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "abc" - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "float64", - "value": 2 - }, - { - "type": "bool", - "value": true - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 3 - } - } - } - ] - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/datapage_v2.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 492, - "Values": 492 - } - }, - "Table": { - "type": "string", - "value": "datapage_v2.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "d", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "e", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b", - "c", - "d", - "e" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/datapage_v2.snappy.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "abc" - }, - { - "type": "int32", - "value": 2 - }, - { - "type": "float64", - "value": 3 - }, - { - "type": "bool", - "value": true - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/datapage_v2.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 223, - "Values": 223 - } - }, - "Table": { - "type": "string", - "value": "datapage_v2.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "d", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "e", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b", - "c", - "d", - "e" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/datapage_v2.snappy.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "abc" - }, - { - "type": "int32", - "value": 3 - }, - { - "type": "float64", - "value": 4 - }, - { - "type": "bool", - "value": true - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/datapage_v2.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 223, - "Values": 223 - } - }, - "Table": { - "type": "string", - "value": "datapage_v2.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "d", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "e", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted deleted file mode 100644 index e7323578..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted +++ /dev/null @@ -1,1175 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c_customer_sk", - "c_current_cdemo_sk", - "c_current_hdemo_sk", - "c_current_addr_sk", - "c_first_shipto_date_sk", - "c_first_sales_date_sk", - "c_birth_day", - "c_birth_month", - "c_birth_year", - "c_customer_id", - "c_salutation", - "c_first_name", - "c_last_name", - "c_preferred_cust_flag", - "c_birth_country", - "c_email_address", - "c_last_review_date" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_encoding_optional_column.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int64", - "value": 100 - }, - { - "type": "int64", - "value": 1254468 - }, - { - "type": "int64", - "value": 6370 - }, - { - "type": "int64", - "value": 6672 - }, - { - "type": "int64", - "value": 2449148 - }, - { - "type": "int64", - "value": 2449118 - }, - { - "type": "int64", - "value": 13 - }, - { - "type": "int64", - "value": 7 - }, - { - "type": "int64", - "value": 1958 - }, - { - "type": "string", - "value": "AAAAAAAAEGAAAAAA" - }, - { - "type": "string", - "value": "Ms." - }, - { - "type": "string", - "value": "Jeannette" - }, - { - "type": "string", - "value": "Johnson" - }, - { - "type": "string", - "value": "Y" - }, - { - "type": "string", - "value": "BANGLADESH" - }, - { - "type": "string", - "value": "Jeannette.Johnson@8BvSqgp.com" - }, - { - "type": "string", - "value": "2452635" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_encoding_optional_column.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 677, - "Values": 677 - } - }, - "Table": { - "type": "string", - "value": "delta_encoding_optional_column.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_cdemo_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_hdemo_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_addr_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_shipto_date_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_sales_date_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_day", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_month", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_year", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_id", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_salutation", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_name", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_name", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_preferred_cust_flag", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_country", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_email_address", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_review_date", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c_customer_sk", - "c_current_cdemo_sk", - "c_current_hdemo_sk", - "c_current_addr_sk", - "c_first_shipto_date_sk", - "c_first_sales_date_sk", - "c_birth_day", - "c_birth_month", - "c_birth_year", - "c_customer_id", - "c_salutation", - "c_first_name", - "c_last_name", - "c_preferred_cust_flag", - "c_birth_country", - "c_email_address", - "c_last_review_date" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_encoding_optional_column.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int64", - "value": 99 - }, - { - "type": "int64", - "value": 622676 - }, - { - "type": "int64", - "value": 2152 - }, - { - "type": "int64", - "value": 17228 - }, - { - "type": "int64", - "value": 2451687 - }, - { - "type": "int64", - "value": 2451657 - }, - { - "type": "int64", - "value": 9 - }, - { - "type": "int64", - "value": 12 - }, - { - "type": "int64", - "value": 1961 - }, - { - "type": "string", - "value": "AAAAAAAADGAAAAAA" - }, - { - "type": "string", - "value": "Sir" - }, - { - "type": "string", - "value": "Austin" - }, - { - "type": "string", - "value": "Tran" - }, - { - "type": "string", - "value": "Y" - }, - { - "type": "string", - "value": "NAMIBIA" - }, - { - "type": "string", - "value": "Austin.Tran@ect7cnjLsucbd.edu" - }, - { - "type": "string", - "value": "2452437" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_encoding_optional_column.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 668, - "Values": 668 - } - }, - "Table": { - "type": "string", - "value": "delta_encoding_optional_column.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_cdemo_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_hdemo_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_addr_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_shipto_date_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_sales_date_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_day", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_month", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_year", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_id", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_salutation", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_name", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_name", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_preferred_cust_flag", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_country", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_email_address", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_review_date", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c_customer_sk", - "c_current_cdemo_sk", - "c_current_hdemo_sk", - "c_current_addr_sk", - "c_first_shipto_date_sk", - "c_first_sales_date_sk", - "c_birth_day", - "c_birth_month", - "c_birth_year", - "c_customer_id", - "c_salutation", - "c_first_name", - "c_last_name", - "c_preferred_cust_flag", - "c_birth_country", - "c_email_address", - "c_last_review_date" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_encoding_optional_column.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int64", - "value": 98 - }, - { - "type": "int64", - "value": 574977 - }, - { - "type": "int64", - "value": 1615 - }, - { - "type": "int64", - "value": 43853 - }, - { - "type": "int64", - "value": 2450894 - }, - { - "type": "int64", - "value": 2450864 - }, - { - "type": "int64", - "value": 23 - }, - { - "type": "int64", - "value": 6 - }, - { - "type": "int64", - "value": 1965 - }, - { - "type": "string", - "value": "AAAAAAAACGAAAAAA" - }, - { - "type": "string", - "value": "Dr." - }, - { - "type": "string", - "value": "David" - }, - { - "type": "string", - "value": "Lewis" - }, - { - "type": "string", - "value": "N" - }, - { - "type": "string", - "value": "KIRIBATI" - }, - { - "type": "string", - "value": "David.Lewis@5mhvq.org" - }, - { - "type": "string", - "value": "2452558" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_encoding_optional_column.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 661, - "Values": 661 - } - }, - "Table": { - "type": "string", - "value": "delta_encoding_optional_column.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_cdemo_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_hdemo_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_addr_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_shipto_date_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_sales_date_sk", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_day", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_month", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_year", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_id", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_salutation", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_name", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_name", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_preferred_cust_flag", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_country", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_email_address", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_review_date", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted deleted file mode 100644 index 0281f9f7..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted +++ /dev/null @@ -1,1175 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c_customer_sk:", - "c_current_cdemo_sk:", - "c_current_hdemo_sk:", - "c_current_addr_sk:", - "c_first_shipto_date_sk:", - "c_first_sales_date_sk:", - "c_birth_day:", - "c_birth_month:", - "c_birth_year:", - "c_customer_id:", - "c_salutation:", - "c_first_name:", - "c_last_name:", - "c_preferred_cust_flag:", - "c_birth_country:", - "c_email_address:", - "c_last_review_date:" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_encoding_required_column.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 105 - }, - { - "type": "int32", - "value": 949850 - }, - { - "type": "int32", - "value": 383 - }, - { - "type": "int32", - "value": 46916 - }, - { - "type": "int32", - "value": 2452463 - }, - { - "type": "int32", - "value": 2452433 - }, - { - "type": "int32", - "value": 14 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "int32", - "value": 1945 - }, - { - "type": "string", - "value": "AAAAAAAAJGAAAAAA" - }, - { - "type": "string", - "value": "Dr." - }, - { - "type": "string", - "value": "Frank" - }, - { - "type": "string", - "value": "Strain" - }, - { - "type": "string", - "value": "Y" - }, - { - "type": "string", - "value": "VIRGIN ISLANDS, U.S." - }, - { - "type": "string", - "value": "Frank.Strain@MbOHByB.edu" - }, - { - "type": "string", - "value": "2452378" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_encoding_required_column.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 641, - "Values": 641 - } - }, - "Table": { - "type": "string", - "value": "delta_encoding_required_column.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_cdemo_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_hdemo_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_addr_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_shipto_date_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_sales_date_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_day:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_month:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_year:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_id:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_salutation:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_name:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_name:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_preferred_cust_flag:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_country:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_email_address:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_review_date:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c_customer_sk:", - "c_current_cdemo_sk:", - "c_current_hdemo_sk:", - "c_current_addr_sk:", - "c_first_shipto_date_sk:", - "c_first_sales_date_sk:", - "c_birth_day:", - "c_birth_month:", - "c_birth_year:", - "c_customer_id:", - "c_salutation:", - "c_first_name:", - "c_last_name:", - "c_preferred_cust_flag:", - "c_birth_country:", - "c_email_address:", - "c_last_review_date:" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_encoding_required_column.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 104 - }, - { - "type": "int32", - "value": 1090695 - }, - { - "type": "int32", - "value": 3116 - }, - { - "type": "int32", - "value": 25490 - }, - { - "type": "int32", - "value": 2450355 - }, - { - "type": "int32", - "value": 2450325 - }, - { - "type": "int32", - "value": 29 - }, - { - "type": "int32", - "value": 11 - }, - { - "type": "int32", - "value": 1936 - }, - { - "type": "string", - "value": "AAAAAAAAIGAAAAAA" - }, - { - "type": "string", - "value": "Dr." - }, - { - "type": "string", - "value": "Benjamin" - }, - { - "type": "string", - "value": "Johnson" - }, - { - "type": "string", - "value": "Y" - }, - { - "type": "string", - "value": "BAHRAIN" - }, - { - "type": "string", - "value": "Benjamin.Johnson@HL2ugJBTO.com" - }, - { - "type": "string", - "value": "2452499" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_encoding_required_column.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 638, - "Values": 638 - } - }, - "Table": { - "type": "string", - "value": "delta_encoding_required_column.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_cdemo_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_hdemo_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_addr_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_shipto_date_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_sales_date_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_day:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_month:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_year:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_id:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_salutation:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_name:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_name:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_preferred_cust_flag:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_country:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_email_address:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_review_date:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c_customer_sk:", - "c_current_cdemo_sk:", - "c_current_hdemo_sk:", - "c_current_addr_sk:", - "c_first_shipto_date_sk:", - "c_first_sales_date_sk:", - "c_birth_day:", - "c_birth_month:", - "c_birth_year:", - "c_customer_id:", - "c_salutation:", - "c_first_name:", - "c_last_name:", - "c_preferred_cust_flag:", - "c_birth_country:", - "c_email_address:", - "c_last_review_date:" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_encoding_required_column.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int32", - "value": 103 - }, - { - "type": "int32", - "value": 1659630 - }, - { - "type": "int32", - "value": 5909 - }, - { - "type": "int32", - "value": 33035 - }, - { - "type": "int32", - "value": 2451586 - }, - { - "type": "int32", - "value": 2451556 - }, - { - "type": "int32", - "value": 3 - }, - { - "type": "int32", - "value": 5 - }, - { - "type": "int32", - "value": 1947 - }, - { - "type": "string", - "value": "AAAAAAAAHGAAAAAA" - }, - { - "type": "string", - "value": "Dr." - }, - { - "type": "string", - "value": "James" - }, - { - "type": "string", - "value": "Porter" - }, - { - "type": "string", - "value": "N" - }, - { - "type": "string", - "value": "AFGHANISTAN" - }, - { - "type": "string", - "value": "James.Porter@3C1oBhj.com" - }, - { - "type": "string", - "value": "2452359" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_encoding_required_column.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 632, - "Values": 632 - } - }, - "Table": { - "type": "string", - "value": "delta_encoding_required_column.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_cdemo_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_hdemo_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_current_addr_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_shipto_date_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_sales_date_sk:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_day:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_month:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_year:", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_customer_id:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_salutation:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_first_name:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_name:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_preferred_cust_flag:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_birth_country:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_email_address:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c_last_review_date:", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted deleted file mode 100644 index 58cc7421..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted +++ /dev/null @@ -1,2858 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "apple_banana_mango0" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 167, - "Values": 167 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "apple_banana_mango1" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 167, - "Values": 167 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "apple_banana_mango4" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 167, - "Values": 167 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 130 - }, - { - "type": "string", - "value": "apple_banana_mango16641" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 130 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 171 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 131 - }, - { - "type": "string", - "value": "apple_banana_mango16900" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 131 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 171 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 132 - }, - { - "type": "string", - "value": "apple_banana_mango17161" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 132 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 171 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 259 - }, - { - "type": "string", - "value": "apple_banana_mango66564" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 259 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 171 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 260 - }, - { - "type": "string", - "value": "apple_banana_mango67081" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 260 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 171 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 261 - }, - { - "type": "string", - "value": "apple_banana_mango67600" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 261 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 171, - "Values": 171 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 388 - }, - { - "type": "string", - "value": "apple_banana_mango149769" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 388 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 389 - }, - { - "type": "string", - "value": "apple_banana_mango150544" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 389 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 390 - }, - { - "type": "string", - "value": "apple_banana_mango151321" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 390 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 517 - }, - { - "type": "string", - "value": "apple_banana_mango266256" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 517 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 518 - }, - { - "type": "string", - "value": "apple_banana_mango267289" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 518 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 519 - }, - { - "type": "string", - "value": "apple_banana_mango268324" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 519 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 646 - }, - { - "type": "string", - "value": "apple_banana_mango416025" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 646 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 647 - }, - { - "type": "string", - "value": "apple_banana_mango417316" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 647 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 648 - }, - { - "type": "string", - "value": "apple_banana_mango418609" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 648 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 775 - }, - { - "type": "string", - "value": "apple_banana_mango599076" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 775 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 776 - }, - { - "type": "string", - "value": "apple_banana_mango600625" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 776 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 777 - }, - { - "type": "string", - "value": "apple_banana_mango602176" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 777 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 904 - }, - { - "type": "string", - "value": "apple_banana_mango815409" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 904 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 905 - }, - { - "type": "string", - "value": "apple_banana_mango817216" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 905 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "FRUIT" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 906 - }, - { - "type": "string", - "value": "apple_banana_mango819025" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 906 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/delta_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 172, - "Values": 172 - } - }, - "Table": { - "type": "string", - "value": "delta_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "FRUIT", - "type": "utf8", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:STRING" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted deleted file mode 100644 index c8dc0ab7..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "l_partkey" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/dict-page-offset-zero.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 1552 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/dict-page-offset-zero.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "dict-page-offset-zero.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "l_partkey", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "l_partkey" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/dict-page-offset-zero.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 1552 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/dict-page-offset-zero.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "dict-page-offset-zero.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "l_partkey", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "l_partkey" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/dict-page-offset-zero.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int32", - "value": 1552 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/dict-page-offset-zero.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "dict-page-offset-zero.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "l_partkey", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted deleted file mode 100644 index 44d78e3f..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted +++ /dev/null @@ -1,2858 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "\u0000\u0000\u0003\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "\u0000\u0000\u0003\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "\u0000\u0000\u0003\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 130 - }, - { - "type": "string", - "value": "\u0000\u0000\u0003f" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 130 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 131 - }, - { - "type": "string", - "value": "\u0000\u0000\u0003e" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 131 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 132 - }, - { - "type": "string", - "value": "\u0000\u0000\u0003d" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 132 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 259 - }, - { - "type": "string", - "value": "\u0000\u0000\u0002\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 259 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 260 - }, - { - "type": "string", - "value": "\u0000\u0000\u0002\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 260 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 261 - }, - { - "type": "string", - "value": "\u0000\u0000\u0002\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 261 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 388 - }, - { - "type": "string", - "value": "\u0000\u0000\u0002\\" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 388 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 389 - }, - { - "type": "string", - "value": "\u0000\u0000\u0002[" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 389 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 390 - }, - { - "type": "string", - "value": "\u0000\u0000\u0002Y" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 390 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 517 - }, - { - "type": "string", - "value": "\u0000\u0000\u0001\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 517 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 518 - }, - { - "type": "string", - "value": "\u0000\u0000\u0001\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 518 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 519 - }, - { - "type": "string", - "value": "\u0000\u0000\u0001\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 519 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 646 - }, - { - "type": "string", - "value": "\u0000\u0000\u0001_" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 646 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 647 - }, - { - "type": "string", - "value": "\u0000\u0000\u0001^" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 647 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 648 - }, - { - "type": "string", - "value": "\u0000\u0000\u0001[" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 648 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 775 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 775 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 776 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 776 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 777 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 777 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 904 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000`" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 904 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 905 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000_" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 905 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "flba_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - { - "type": "uint64", - "value": 906 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000^" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 906 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_byte_array.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 152, - "Values": 152 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_byte_array.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "flba_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(4)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted deleted file mode 100644 index 4c12f6a5..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_decimal.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000d" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 156, - "Values": 156 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(11)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_decimal.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 156, - "Values": 156 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(11)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_decimal.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001," - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 156, - "Values": 156 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(11)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted deleted file mode 100644 index b5fbb20c..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_decimal_legacy.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\u0000\u0000d" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_decimal_legacy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 158, - "Values": 158 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_decimal_legacy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(6)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_decimal_legacy.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\u0000\u0000\ufffd" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_decimal_legacy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 158, - "Values": 158 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_decimal_legacy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(6)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/fixed_length_decimal_legacy.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "string", - "value": "\u0000\u0000\u0000\u0000\u0001," - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/fixed_length_decimal_legacy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 158, - "Values": 158 - } - }, - "Table": { - "type": "string", - "value": "fixed_length_decimal_legacy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:FIXED_LEN_BYTE_ARRAY(6)" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted deleted file mode 100644 index 26134acd..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_decimal.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 100 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 126 - } - }, - "Table": { - "type": "string", - "value": "int32_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_decimal.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 200 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 126 - } - }, - "Table": { - "type": "string", - "value": "int32_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_decimal.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int32", - "value": 300 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 126, - "Values": 126 - } - }, - "Table": { - "type": "string", - "value": "int32_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted deleted file mode 100644 index 43de86cf..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted +++ /dev/null @@ -1,2858 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": -654807448 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": -465559769 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int32", - "value": -34563097 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 130 - }, - { - "type": "int32", - "value": -1689290271 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 130 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 131 - }, - { - "type": "int32", - "value": 1745329571 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 131 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 132 - }, - { - "type": "int32", - "value": -1717925870 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 132 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 259 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 259 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 260 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 260 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 261 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 261 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 388 - }, - { - "type": "int32", - "value": -187025414 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 388 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 389 - }, - { - "type": "int32", - "value": -1241385771 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 389 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 390 - }, - { - "type": "int32", - "value": 177814932 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 390 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 517 - }, - { - "type": "int32", - "value": -1268231836 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 517 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 518 - }, - { - "type": "int32", - "value": 1075344848 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 518 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 519 - }, - { - "type": "int32", - "value": -1499712974 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 519 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 646 - }, - { - "type": "int32", - "value": 216351686 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 646 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 647 - }, - { - "type": "int32", - "value": -1093266191 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 647 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 648 - }, - { - "type": "int32", - "value": 2125689411 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 648 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 775 - }, - { - "type": "int32", - "value": -1391713441 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 775 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 776 - }, - { - "type": "int32", - "value": 168600889 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 776 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 777 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 777 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 904 - }, - { - "type": "int32", - "value": 1586125964 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 904 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 905 - }, - { - "type": "int32", - "value": 624223890 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 905 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int32_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - { - "type": "uint64", - "value": 906 - }, - { - "type": "int32", - "value": 1298187183 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 906 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int32_with_null_pages.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 134, - "Values": 134 - } - }, - "Table": { - "type": "string", - "value": "int32_with_null_pages.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int32_field", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted deleted file mode 100644 index 06fb66d9..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int64_decimal.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int64", - "value": 100 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int64_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int64_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int64_decimal.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int64", - "value": 200 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int64_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int64_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "value" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/int64_decimal.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int64", - "value": 300 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/int64_decimal.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "int64_decimal.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "value", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted deleted file mode 100644 index 6ff2c641..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted +++ /dev/null @@ -1,557 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int64_list", - "utf8_list" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/list_columns.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "int64", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "int64", - "value": 2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "int64", - "value": 3 - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "string", - "value": "abc" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "string", - "value": "efg" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "string", - "value": "hij" - } - } - } - ] - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/list_columns.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 714, - "Values": 714 - } - }, - "Table": { - "type": "string", - "value": "list_columns.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int64_list", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "utf8_list", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int64_list", - "utf8_list" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/list_columns.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "int64", - "value": 1 - } - } - } - ] - } - } - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/list_columns.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 333, - "Values": 333 - } - }, - "Table": { - "type": "string", - "value": "list_columns.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int64_list", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "utf8_list", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "int64_list", - "utf8_list" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/list_columns.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "int64", - "value": 4 - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "string", - "value": "efg" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "string", - "value": "hij" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "item": { - "type": "string", - "value": "xyz" - } - } - } - ] - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/list_columns.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 638, - "Values": 638 - } - }, - "Table": { - "type": "string", - "value": "list_columns.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int64_list", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "utf8_list", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted deleted file mode 100644 index d544292c..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted +++ /dev/null @@ -1,461 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c0", - "c1", - "v11" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/lz4_raw_compressed.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int64", - "value": 1593604800 - }, - { - "type": "string", - "value": "abc" - }, - { - "type": "float64", - "value": 42 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/lz4_raw_compressed.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 194, - "Values": 194 - } - }, - "Table": { - "type": "string", - "value": "lz4_raw_compressed.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c0", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c1", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "v11", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c0", - "c1", - "v11" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/lz4_raw_compressed.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int64", - "value": 1593604800 - }, - { - "type": "string", - "value": "def" - }, - { - "type": "float64", - "value": 7.7 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/lz4_raw_compressed.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 194, - "Values": 194 - } - }, - "Table": { - "type": "string", - "value": "lz4_raw_compressed.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c0", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c1", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "v11", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "c0", - "c1", - "v11" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/lz4_raw_compressed.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int64", - "value": 1593604801 - }, - { - "type": "string", - "value": "abc" - }, - { - "type": "float64", - "value": 42.125 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/lz4_raw_compressed.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 194, - "Values": 194 - } - }, - "Table": { - "type": "string", - "value": "lz4_raw_compressed.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c0", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c1", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "v11", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted deleted file mode 100644 index 11564346..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted +++ /dev/null @@ -1,830 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_lists.snappy.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "a" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "b" - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "c" - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "d" - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "int32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_lists.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 1254, - "Values": 1254 - } - }, - "Table": { - "type": "string", - "value": "nested_lists.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_lists.snappy.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "a" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "b" - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "c" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "d" - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "e" - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "int32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_lists.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 1334, - "Values": 1334 - } - }, - "Table": { - "type": "string", - "value": "nested_lists.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_lists.snappy.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "a" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "b" - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "c" - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "d" - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "e" - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "string", - "value": "f" - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "int32", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_lists.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 1545, - "Values": 1545 - } - }, - "Table": { - "type": "string", - "value": "nested_lists.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted deleted file mode 100644 index 7f99a7c3..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted +++ /dev/null @@ -1,569 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b", - "c" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_maps.snappy.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "key_value": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "a" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "key_value": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "int32", - "value": 1 - }, - "value": { - "type": "bool", - "value": true - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "int32", - "value": 2 - }, - "value": { - "type": "bool", - "value": false - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "float64", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_maps.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 632, - "Values": 632 - } - }, - "Table": { - "type": "string", - "value": "nested_maps.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b", - "c" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_maps.snappy.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "map[string]interface {}", - "value": { - "key_value": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "b" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "key_value": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "int32", - "value": 1 - }, - "value": { - "type": "bool", - "value": true - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "float64", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_maps.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 531, - "Values": 531 - } - }, - "Table": { - "type": "string", - "value": "nested_maps.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "a", - "b", - "c" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_maps.snappy.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "map[string]interface {}", - "value": { - "key_value": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "c" - }, - "value": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "float64", - "value": 1 - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_maps.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 357, - "Values": 357 - } - }, - "Table": { - "type": "string", - "value": "nested_maps.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "a", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "c", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted deleted file mode 100644 index c4fe1181..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted +++ /dev/null @@ -1,1616 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "roll_num", - "PC_CUR", - "CVA_2012", - "CVA_2016", - "BIA_3", - "BIA_4", - "ACTUAL_FRONTAGE", - "ACTUAL_DEPTH", - "ACTUAL_LOT_SIZE", - "GLA", - "SOURCE_GLA", - "IPS_GLA", - "GLA_ALL", - "bia", - "EFFECTIVE_LOT_SIZE", - "effective_lot_area", - "EFFECTIVE_FRONTAGE", - "EFFECTIVE_DEPTH", - "rw_area_tot", - "effective_lot_sqft", - "dup", - "nonCTXT", - "vacantland", - "parkingbillboard", - "cvalte10", - "condootherhotel", - "calculated_lot_size", - "calculated_efflot_size", - "missingsite", - "missinggla", - "missingsitegla", - "actual_lot_size_sqft", - "lotsize_sqft", - "count", - "ul_observation_date", - "ul_tz_offset_minutes_ul_observation_date" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nested_structs.rust.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 190407175004000 - }, - "mean": { - "type": "int64", - "value": 190406671229999 - }, - "min": { - "type": "int64", - "value": 190406409000602 - }, - "sum": { - "type": "int64", - "value": 94251302258849568 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 742 - }, - "mean": { - "type": "int64", - "value": 416 - }, - "min": { - "type": "int64", - "value": 115 - }, - "sum": { - "type": "int64", - "value": 206195 - }, - "variance": { - "type": "int64", - "value": 10374 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 32150509 - }, - "mean": { - "type": "int64", - "value": 2401239 - }, - "min": { - "type": "int64", - "value": 737 - }, - "sum": { - "type": "int64", - "value": 1188613496 - }, - "variance": { - "type": "int64", - "value": 12977533288261 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 35195000 - }, - "mean": { - "type": "int64", - "value": 3519838 - }, - "min": { - "type": "int64", - "value": 1000 - }, - "sum": { - "type": "int64", - "value": 1742320297 - }, - "variance": { - "type": "int64", - "value": 24581100553044 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 0 - }, - "max": { - "type": "float64", - "value": 0 - }, - "mean": { - "type": "float64", - "value": 0 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 0 - }, - "variance": { - "type": "float64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 0 - }, - "max": { - "type": "float64", - "value": 0 - }, - "mean": { - "type": "float64", - "value": 0 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 0 - }, - "variance": { - "type": "float64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 351 - }, - "max": { - "type": "float64", - "value": 658.63 - }, - "mean": { - "type": "float64", - "value": 57.76452991452993 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 20275.350000000006 - }, - "variance": { - "type": "float64", - "value": 6310.500499135526 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 336 - }, - "max": { - "type": "float64", - "value": 312.16 - }, - "mean": { - "type": "float64", - "value": 49.40901785714286 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 16601.43 - }, - "variance": { - "type": "float64", - "value": 3214.842695450431 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 366 - }, - "max": { - "type": "float64", - "value": 74136 - }, - "mean": { - "type": "float64", - "value": 6162.133196721318 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 2255340.7500000023 - }, - "variance": { - "type": "float64", - "value": 104255249.59826614 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 334 - }, - "max": { - "type": "float64", - "value": 523800 - }, - "mean": { - "type": "float64", - "value": 19484.146706586827 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 6507705 - }, - "variance": { - "type": "float64", - "value": 3563198650.906335 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 10 - }, - "max": { - "type": "float64", - "value": 16085 - }, - "mean": { - "type": "float64", - "value": 6698.8 - }, - "min": { - "type": "float64", - "value": 2628 - }, - "sum": { - "type": "float64", - "value": 66988 - }, - "variance": { - "type": "float64", - "value": 28540252.400000002 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 4 - }, - "max": { - "type": "float64", - "value": 1985 - }, - "mean": { - "type": "float64", - "value": 1285 - }, - "min": { - "type": "float64", - "value": 288 - }, - "sum": { - "type": "float64", - "value": 5140 - }, - "variance": { - "type": "float64", - "value": 509875.3333333333 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 348 - }, - "max": { - "type": "float64", - "value": 523800 - }, - "mean": { - "type": "float64", - "value": 18907.566091954024 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 6579833 - }, - "variance": { - "type": "float64", - "value": 3428378496.7881336 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 1 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 452 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 352 - }, - "max": { - "type": "float64", - "value": 64749.63000000001 - }, - "mean": { - "type": "float64", - "value": 4951.024888352274 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 1742760.7607000005 - }, - "variance": { - "type": "float64", - "value": 81195383.98823886 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 352 - }, - "max": { - "type": "float64", - "value": 2.8000000000000003 - }, - "mean": { - "type": "float64", - "value": 0.14237550619122732 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 50.11617817931202 - }, - "variance": { - "type": "float64", - "value": 0.07516922114035923 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 339 - }, - "max": { - "type": "float64", - "value": 658063 - }, - "mean": { - "type": "float64", - "value": 1991.3067846607655 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 675052.9999999995 - }, - "variance": { - "type": "float64", - "value": 1277234044.0126908 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 337 - }, - "max": { - "type": "float64", - "value": 300.7 - }, - "mean": { - "type": "float64", - "value": 65.32364985163204 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 22014.069999999996 - }, - "variance": { - "type": "float64", - "value": 3904.805190507992 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 441 - }, - "max": { - "type": "float64", - "value": 18169 - }, - "mean": { - "type": "float64", - "value": 1528.077097505669 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 673882 - }, - "variance": { - "type": "float64", - "value": 6122348.621315204 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 352 - }, - "max": { - "type": "float64", - "value": 121968 - }, - "mean": { - "type": "float64", - "value": 6201.877049689864 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 2183060.721490832 - }, - "variance": { - "type": "float64", - "value": 142631612.6463931 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 81 - }, - "max": { - "type": "float64", - "value": 100 - }, - "mean": { - "type": "float64", - "value": 1.2345679012345678 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 100 - }, - "variance": { - "type": "float64", - "value": 123.45679012345684 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 101 - }, - "max": { - "type": "float64", - "value": 4172.084000000002 - }, - "mean": { - "type": "float64", - "value": 42.29786138613863 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 4272.084000000002 - }, - "variance": { - "type": "float64", - "value": 172355.84886194076 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 1 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 208 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 1 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 44 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 366 - }, - "max": { - "type": "float64", - "value": 121968 - }, - "mean": { - "type": "float64", - "value": 8685.222814207653 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 3178791.550000001 - }, - "variance": { - "type": "float64", - "value": 243347757.98270744 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 364 - }, - "max": { - "type": "float64", - "value": 121968 - }, - "mean": { - "type": "float64", - "value": 8841.174394454862 - }, - "min": { - "type": "float64", - "value": 0 - }, - "sum": { - "type": "float64", - "value": 3218187.4795815693 - }, - "variance": { - "type": "float64", - "value": 244563632.41811454 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 1 - }, - "mean": { - "type": "int64", - "value": 1 - }, - "min": { - "type": "int64", - "value": 1 - }, - "sum": { - "type": "int64", - "value": 495 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 1608822900000000000 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 1608822900000000000 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "count": { - "type": "int64", - "value": 495 - }, - "max": { - "type": "int64", - "value": 0 - }, - "mean": { - "type": "int64", - "value": 0 - }, - "min": { - "type": "int64", - "value": 0 - }, - "sum": { - "type": "int64", - "value": 0 - }, - "variance": { - "type": "int64", - "value": 0 - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nested_structs.rust.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 10552, - "Values": 10552 - } - }, - "Table": { - "type": "string", - "value": "nested_structs.rust.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "roll_num", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "PC_CUR", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "CVA_2012", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "CVA_2016", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "BIA_3", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "BIA_4", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ACTUAL_FRONTAGE", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ACTUAL_DEPTH", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ACTUAL_LOT_SIZE", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "GLA", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "SOURCE_GLA", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "IPS_GLA", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "GLA_ALL", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "bia", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EFFECTIVE_LOT_SIZE", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "effective_lot_area", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EFFECTIVE_FRONTAGE", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "EFFECTIVE_DEPTH", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "rw_area_tot", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "effective_lot_sqft", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "dup", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "nonCTXT", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "vacantland", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "parkingbillboard", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "cvalte10", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "condootherhotel", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "calculated_lot_size", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "calculated_efflot_size", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "missingsite", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "missinggla", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "missingsitegla", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "actual_lot_size_sqft", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "lotsize_sqft", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "count", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ul_observation_date", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ul_tz_offset_minutes_ul_observation_date", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted deleted file mode 100644 index 2a29dec5..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted +++ /dev/null @@ -1,450 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "ID", - "Int_Array", - "int_array_array", - "Int_Map", - "int_map_array", - "nested_Struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nonnullable.impala.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int64", - "value": 8 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": -1 - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": -1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": -2 - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k1" - }, - "value": { - "type": "int32", - "value": -1 - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k1" - }, - "value": { - "type": "int32", - "value": 1 - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "B": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": -1 - } - } - } - ] - } - } - }, - "G": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - }, - "a": { - "type": "int32", - "value": -1 - }, - "c": { - "type": "map[string]interface {}", - "value": { - "D": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "e": { - "type": "int32", - "value": -1 - }, - "f": { - "type": "string", - "value": "nonnullable" - } - } - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nonnullable.impala.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 2459, - "Values": 2459 - } - }, - "Table": { - "type": "string", - "value": "nonnullable.impala.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "ID", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Int_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array_array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "Int_Map", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_map_array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "nested_Struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted deleted file mode 100644 index 4a3caaac..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted +++ /dev/null @@ -1,126 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "emptylist" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/null_list.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/null_list.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 186, - "Values": 186 - } - }, - "Table": { - "type": "string", - "value": "null_list.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "emptylist", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted deleted file mode 100644 index 4ec75cff..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted +++ /dev/null @@ -1,1669 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "int_array", - "int_array_Array", - "int_map", - "int_Map_Array", - "nested_struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nullable.impala.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 3 - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 2 - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 3 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 4 - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k1" - }, - "value": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k2" - }, - "value": { - "type": "int32", - "value": 100 - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k1" - }, - "value": { - "type": "int32", - "value": 1 - } - } - } - ] - } - } - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "A": { - "type": "int32", - "value": 1 - }, - "C": { - "type": "map[string]interface {}", - "value": { - "d": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "int32", - "value": 10 - }, - "F": { - "type": "string", - "value": "aaa" - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "int32", - "value": -10 - }, - "F": { - "type": "string", - "value": "bbb" - } - } - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "int32", - "value": 11 - }, - "F": { - "type": "string", - "value": "c" - } - } - } - } - } - ] - } - } - } - } - } - ] - } - } - } - } - }, - "b": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 1 - } - } - } - ] - } - } - }, - "g": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "foo" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "H": { - "type": "map[string]interface {}", - "value": { - "i": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "float64", - "value": 1.1 - } - } - } - ] - } - } - } - } - } - } - } - } - } - ] - } - } - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nullable.impala.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 3229, - "Values": 3229 - } - }, - "Table": { - "type": "string", - "value": "nullable.impala.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_map", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_Map_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "nested_struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "int_array", - "int_array_Array", - "int_map", - "int_Map_Array", - "nested_struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nullable.impala.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int64", - "value": 2 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 3 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 1 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 3 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "int32", - "value": 4 - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k1" - }, - "value": { - "type": "int32", - "value": 2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k2" - }, - "value": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k3" - }, - "value": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "k1" - }, - "value": { - "type": "int32", - "value": 1 - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "A": { - "type": "nil", - "value": null - }, - "C": { - "type": "map[string]interface {}", - "value": { - "d": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "nil", - "value": null - }, - "F": { - "type": "nil", - "value": null - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "int32", - "value": 10 - }, - "F": { - "type": "string", - "value": "aaa" - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "nil", - "value": null - }, - "F": { - "type": "nil", - "value": null - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "int32", - "value": -10 - }, - "F": { - "type": "string", - "value": "bbb" - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "nil", - "value": null - }, - "F": { - "type": "nil", - "value": null - } - } - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "E": { - "type": "int32", - "value": 11 - }, - "F": { - "type": "string", - "value": "c" - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - } - } - }, - "b": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - "g": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "g1" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "H": { - "type": "map[string]interface {}", - "value": { - "i": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "float64", - "value": 2.2 - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - } - } - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "g2" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "H": { - "type": "map[string]interface {}", - "value": { - "i": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - } - } - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "g3" - }, - "value": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "g4" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "H": { - "type": "map[string]interface {}", - "value": { - "i": { - "type": "nil", - "value": null - } - } - } - } - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "key": { - "type": "string", - "value": "g5" - }, - "value": { - "type": "map[string]interface {}", - "value": { - "H": { - "type": "nil", - "value": null - } - } - } - } - } - ] - } - } - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nullable.impala.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 5555, - "Values": 5555 - } - }, - "Table": { - "type": "string", - "value": "nullable.impala.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_map", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_Map_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "nested_struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "int_array", - "int_array_Array", - "int_map", - "int_Map_Array", - "nested_struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nullable.impala.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int64", - "value": 3 - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": [ - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "element": { - "type": "nil", - "value": null - } - } - } - ] - } - } - }, - { - "type": "map[string]interface {}", - "value": { - "A": { - "type": "nil", - "value": null - }, - "C": { - "type": "map[string]interface {}", - "value": { - "d": { - "type": "map[string]interface {}", - "value": { - "list": { - "type": "[]interface {}", - "value": null - } - } - } - } - }, - "b": { - "type": "nil", - "value": null - }, - "g": { - "type": "map[string]interface {}", - "value": { - "map": { - "type": "[]interface {}", - "value": null - } - } - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nullable.impala.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 988, - "Values": 988 - } - }, - "Table": { - "type": "string", - "value": "nullable.impala.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_array_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_map", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "int_Map_Array", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "nested_struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted deleted file mode 100644 index 34e4c830..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted +++ /dev/null @@ -1,374 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "b_struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nulls.snappy.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "map[string]interface {}", - "value": { - "b_c_int": { - "type": "nil", - "value": null - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nulls.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 168, - "Values": 168 - } - }, - "Table": { - "type": "string", - "value": "nulls.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b_struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "b_struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nulls.snappy.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "map[string]interface {}", - "value": { - "b_c_int": { - "type": "nil", - "value": null - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nulls.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 168, - "Values": 168 - } - }, - "Table": { - "type": "string", - "value": "nulls.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b_struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "b_struct" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/nulls.snappy.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "map[string]interface {}", - "value": { - "b_c_int": { - "type": "nil", - "value": null - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/nulls.snappy.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 168, - "Values": 168 - } - }, - "Table": { - "type": "string", - "value": "nulls.snappy.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "b_struct", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted deleted file mode 100644 index 2a591e5a..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted +++ /dev/null @@ -1,3266 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 130 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 130 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 131 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 131 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 132 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 132 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 259 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 259 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 260 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 260 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 261 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 261 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 388 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 388 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 389 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 389 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 390 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 390 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 517 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 517 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 518 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 518 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 519 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 519 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 646 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 646 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 647 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 647 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 648 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 648 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 775 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 775 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 776 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 776 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 777 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 777 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 904 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 904 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 905 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 905 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "long_field", - "binary_field" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - { - "type": "uint64", - "value": 906 - }, - { - "type": "int64", - "value": 0 - }, - { - "type": "string", - "value": "a655fd0e-9949-4059-bcae-fd6a002a4652" - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 906 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/plain-dict-uncompressed-checksum.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 217, - "Values": 217 - } - }, - "Table": { - "type": "string", - "value": "plain-dict-uncompressed-checksum.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "long_field", - "type": "int64", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT64" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "binary_field", - "type": "string", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BYTE_ARRAY" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted deleted file mode 100644 index b630217d..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted +++ /dev/null @@ -1,415 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "phoneNumbers" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/repeated_no_annotation.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "int32", - "value": 1 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/repeated_no_annotation.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 151, - "Values": 151 - } - }, - "Table": { - "type": "string", - "value": "repeated_no_annotation.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "phoneNumbers", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "phoneNumbers" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/repeated_no_annotation.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "int32", - "value": 2 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/repeated_no_annotation.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 151, - "Values": 151 - } - }, - "Table": { - "type": "string", - "value": "repeated_no_annotation.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "phoneNumbers", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "id", - "phoneNumbers" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/repeated_no_annotation.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "int32", - "value": 3 - }, - { - "type": "map[string]interface {}", - "value": { - "phone": { - "type": "[]interface {}", - "value": null - } - } - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/repeated_no_annotation.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 220, - "Values": 220 - } - }, - "Table": { - "type": "string", - "value": "repeated_no_annotation.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "id", - "type": "int32", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:INT32" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "phoneNumbers", - "type": "any", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:group" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted deleted file mode 100644 index 7dcd0e3c..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "datatype_boolean" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/rle_boolean_encoding.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "bool", - "value": true - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/rle_boolean_encoding.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "rle_boolean_encoding.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "datatype_boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "datatype_boolean" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/rle_boolean_encoding.parquet" - }, - { - "type": "uint64", - "value": 2 - }, - { - "type": "bool", - "value": false - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 2 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/rle_boolean_encoding.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 130, - "Values": 130 - } - }, - "Table": { - "type": "string", - "value": "rle_boolean_encoding.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "datatype_boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - }, - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "datatype_boolean" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/rle_boolean_encoding.parquet" - }, - { - "type": "uint64", - "value": 3 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 3 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/rle_boolean_encoding.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 129, - "Values": 129 - } - }, - "Table": { - "type": "string", - "value": "rle_boolean_encoding.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "datatype_boolean", - "type": "boolean", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:BOOLEAN" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted b/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted deleted file mode 100644 index af6edd63..00000000 --- a/tests/canon/s3/parquet/canondata/parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted +++ /dev/null @@ -1,121 +0,0 @@ -[ - { - "ColumnNames": { - "type": "[]string", - "value": [ - "__file_name", - "__row_index", - "mycol" - ] - }, - "ColumnValues": { - "type": "[]interface {}", - "value": [ - { - "type": "string", - "value": "data/single_nan.parquet" - }, - { - "type": "uint64", - "value": 1 - }, - { - "type": "nil", - "value": null - } - ] - }, - "CommitTime": { - "type": "uint64", - "value": 0 - }, - "Counter": { - "type": "int", - "value": 1 - }, - "ID": { - "type": "uint32", - "value": 0 - }, - "Kind": { - "type": "changeitem.Kind", - "value": "insert" - }, - "LSN": { - "type": "uint64", - "value": 0 - }, - "OldKeys": { - "type": "changeitem.OldKeysType", - "value": {} - }, - "PartID": { - "type": "string", - "value": "data/single_nan.parquet" - }, - "Query": { - "type": "string", - "value": "" - }, - "Schema": { - "type": "string", - "value": "s3_source_parquet" - }, - "Size": { - "type": "changeitem.EventSize", - "value": { - "Read": 119, - "Values": 119 - } - }, - "Table": { - "type": "string", - "value": "single_nan.parquet" - }, - "TableSchema": { - "type": "[]changeitem.ColSchema", - "value": [ - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__file_name", - "type": "utf8", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "__row_index", - "type": "uint64", - "key": true, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "" - }, - { - "table_schema": "", - "table_name": "", - "path": "", - "name": "mycol", - "type": "double", - "key": false, - "fake_key": false, - "required": false, - "expression": "", - "original_type": "parquet:DOUBLE" - } - ] - }, - "TxID": { - "type": "string", - "value": "" - } - } -] \ No newline at end of file diff --git a/tests/canon/s3/parquet/canondata/result.json b/tests/canon/s3/parquet/canondata/result.json deleted file mode 100644 index 2fb5f9f8..00000000 --- a/tests/canon/s3/parquet/canondata/result.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "parquet.parquet.TestCanonSource/alltypes_dictionary.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_alltypes_dictionary.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/alltypes_plain.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_alltypes_plain.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/alltypes_plain.snappy.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_alltypes_plain.snappy.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/binary.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_binary.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/byte_array_decimal.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_byte_array_decimal.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/data_index_bloom_encoding_stats.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_data_index_bloom_encoding_stats.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/datapage_v1-snappy-compressed-checksum.parquet/canon_0": { - "checksum": "c216612ff65f2f9d2def605d7e3884be", - "size": 462391, - "uri": "https://storage.yandex-team.ru/get-devtools/1814674/808a21807fab5f4b33c6fbdc9b8ccffc617a7470/resource.tar.gz#parquet.parquet.TestCanonSource_datapage_v1-snappy-compressed-checksum.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/datapage_v1-uncompressed-checksum.parquet/canon_0": { - "checksum": "9ef34eaca71eee6c58a01078616eeb40", - "size": 460591, - "uri": "https://storage.yandex-team.ru/get-devtools/1814674/808a21807fab5f4b33c6fbdc9b8ccffc617a7470/resource.tar.gz#parquet.parquet.TestCanonSource_datapage_v1-uncompressed-checksum.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/datapage_v2.snappy.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_datapage_v2.snappy.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/delta_binary_packed.parquet/canon_0": { - "checksum": "0784938776e8d84a1f839799b2ff2436", - "size": 242681, - "uri": "https://storage.yandex-team.ru/get-devtools/1814674/808a21807fab5f4b33c6fbdc9b8ccffc617a7470/resource.tar.gz#parquet.parquet.TestCanonSource_delta_binary_packed.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/delta_byte_array.parquet/canon_0": { - "checksum": "9ea6e34f163b60af967a4ff868f9eac6", - "size": 189869, - "uri": "https://storage.yandex-team.ru/get-devtools/1814674/808a21807fab5f4b33c6fbdc9b8ccffc617a7470/resource.tar.gz#parquet.parquet.TestCanonSource_delta_byte_array.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/delta_encoding_optional_column.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_delta_encoding_optional_column.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/delta_encoding_required_column.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_delta_encoding_required_column.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/delta_length_byte_array.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_delta_length_byte_array.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/dict-page-offset-zero.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_dict-page-offset-zero.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/fixed_length_byte_array.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_fixed_length_byte_array.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/fixed_length_decimal.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_fixed_length_decimal.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/fixed_length_decimal_legacy.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_fixed_length_decimal_legacy.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/int32_decimal.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_int32_decimal.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/int32_with_null_pages.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_int32_with_null_pages.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/int64_decimal.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_int64_decimal.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/list_columns.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_list_columns.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/lz4_raw_compressed.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_lz4_raw_compressed.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/lz4_raw_compressed_larger.parquet/canon_0": { - "checksum": "683c4e161e08eb6fab784ca383d40560", - "size": 769568, - "uri": "https://storage.yandex-team.ru/get-devtools/1814674/808a21807fab5f4b33c6fbdc9b8ccffc617a7470/resource.tar.gz#parquet.parquet.TestCanonSource_lz4_raw_compressed_larger.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/nested_lists.snappy.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_nested_lists.snappy.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/nested_maps.snappy.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_nested_maps.snappy.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/nested_structs.rust.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_nested_structs.rust.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/nonnullable.impala.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_nonnullable.impala.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/null_list.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_null_list.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/nullable.impala.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_nullable.impala.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/nulls.snappy.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_nulls.snappy.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/plain-dict-uncompressed-checksum.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_plain-dict-uncompressed-checksum.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/repeated_no_annotation.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_repeated_no_annotation.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/rle_boolean_encoding.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_rle_boolean_encoding.parquet_canon_0/extracted" - }, - "parquet.parquet.TestCanonSource/single_nan.parquet/canon_0": { - "uri": "file://parquet.parquet.TestCanonSource_single_nan.parquet_canon_0/extracted" - } -} diff --git a/tests/canon/sequences/README.md b/tests/canon/sequences/README.md deleted file mode 100644 index e5dad17d..00000000 --- a/tests/canon/sequences/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# `sequences`: canonized sequences of ChangeItems in replication - -This directory contains code to **canonize** sequences of ChangeItems in replication for usage in sink correctness tests. This is not an actual test. - -## Usage - -The output of canonization is stored in the [**sub**directories of `canondata`](./canondata/). Each `extracted` file in those directories contains a canonized sequence of ChangeItems obtained by execution of a `SNAPSHOT_AND_INCREMENT` transfer with `PostgreSQL` source and `Mock` target. It is possible to set a particular SQL for initialization (which is not transformed into canonized items) and for replication parts of each test. SQLs are in the [`dump` subdirectory](./dump/). - -### Add a new canonized sequence - -In order to add a new canonized sequence, do the following: - -1. Add a SQL file(s) for initialization (optional) and for replication (required) in the [`dump` subdirectory](./dump/). -2. Import the new file(s) in the [`sequences_test.go` file](./sequences_test.go) using the `//go:embed` directive, as is done with other files. Add the `GO_TEST_EMBED_PATTERN` to the [`ya.make` file](./ya.make). -3. Add a `t.Run()` call and a corresponding test name to the list of tests/ -4. Recanonize the sequences using `ya make -AZ .` run from the current directory. diff --git a/tests/canon/sequences/canondata/result.json b/tests/canon/sequences/canondata/result.json deleted file mode 100644 index a7f126c6..00000000 --- a/tests/canon/sequences/canondata/result.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "sequences.sequences.TestCanonizeSequences/insert_update_delete/canon_0": { - "uri": "file://sequences.sequences.TestCanonizeSequences_insert_update_delete_canon_0/extracted" - }, - "sequences.sequences.TestCanonizeSequences/insert_update_insert/canon_0": { - "uri": "file://sequences.sequences.TestCanonizeSequences_insert_update_insert_canon_0/extracted" - }, - "sequences.sequences.TestCanonizeSequences/updatepk/canon_0": { - "uri": "file://sequences.sequences.TestCanonizeSequences_updatepk_canon_0/extracted" - } -} diff --git a/tests/canon/sequences/dump/00_insert_update_delete.sql b/tests/canon/sequences/dump/00_insert_update_delete.sql deleted file mode 100644 index 7d5ed73b..00000000 --- a/tests/canon/sequences/dump/00_insert_update_delete.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO insert_update_delete(i1, i2, t) VALUES (2, 2, 'b'); -UPDATE insert_update_delete SET t = 'c' WHERE i1 = 2 AND i2 = 2; -DELETE FROM insert_update_delete WHERE i1 = 2 AND i2 = 2; diff --git a/tests/canon/sequences/dump/01_updatepk.sql b/tests/canon/sequences/dump/01_updatepk.sql deleted file mode 100644 index 5ec44ffb..00000000 --- a/tests/canon/sequences/dump/01_updatepk.sql +++ /dev/null @@ -1,4 +0,0 @@ -INSERT INTO insert_update_delete(i1, i2, t) VALUES (1, 1, 'a'); -UPDATE insert_update_delete SET t = 'b' WHERE i1 = 1 AND i2 = 1; -UPDATE insert_update_delete SET i1 = 2, i2 = 2 WHERE i1 = 1 AND i2 = 1; -UPDATE insert_update_delete SET t = 'c' WHERE i1 = 2 AND i2 = 2; diff --git a/tests/canon/sequences/dump/02_insert_update_insert.sql b/tests/canon/sequences/dump/02_insert_update_insert.sql deleted file mode 100644 index cb205433..00000000 --- a/tests/canon/sequences/dump/02_insert_update_insert.sql +++ /dev/null @@ -1,12 +0,0 @@ -INSERT INTO insert_update_delete(i1, i2, t) VALUES (1, 1, '1a'); -UPDATE insert_update_delete SET t = '1b' WHERE i1 = 1 AND i2 = 1; -UPDATE insert_update_delete SET i1 = 2, i2 = 2 WHERE i1 = 1 AND i2 = 1; - -INSERT INTO insert_update_delete(i1, i2, t) VALUES (1, 1, '2a'); -UPDATE insert_update_delete SET t = '2b' WHERE i1 = 1 AND i2 = 1; - -INSERT INTO insert_update_delete(i1, i2, t) VALUES (3, 3, '3a'); -UPDATE insert_update_delete SET t = '3b' WHERE i1 = 3 AND i2 = 3; -DELETE FROM insert_update_delete WHERE i1 = 3 AND i2 = 3; -INSERT INTO insert_update_delete(i1, i2, t) VALUES (3, 3, '4a'); -UPDATE insert_update_delete SET t = '4b' WHERE i1 = 3 AND i2 = 3; diff --git a/tests/canon/sequences/dump/init.insert_update_delete.sql b/tests/canon/sequences/dump/init.insert_update_delete.sql deleted file mode 100644 index 02973008..00000000 --- a/tests/canon/sequences/dump/init.insert_update_delete.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE IF EXISTS insert_update_delete; -CREATE TABLE insert_update_delete(i1 INT, i2 INT, t TEXT, PRIMARY KEY(i1, i2)); -ALTER TABLE insert_update_delete REPLICA IDENTITY FULL; diff --git a/tests/canon/sequences/sequences_test.go b/tests/canon/sequences/sequences_test.go deleted file mode 100644 index abe9e54f..00000000 --- a/tests/canon/sequences/sequences_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package sequences - -import ( - "context" - _ "embed" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/tests/canon" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - //go:embed dump/00_insert_update_delete.sql - insertUpdateDelete []byte - //go:embed dump/01_updatepk.sql - updatePK []byte - //go:embed dump/02_insert_update_insert.sql - insertUpdateInsert []byte - //go:embed dump/init.insert_update_delete.sql - initInsertUpdateDelete []byte -) - -func TestCanonizeSequences(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - Source := &pgcommon.PgSource{ - ClusterID: os.Getenv("PG_CLUSTER_ID"), - Hosts: []string{"localhost"}, - User: os.Getenv("PG_LOCAL_USER"), - Password: server.SecretString(os.Getenv("PG_LOCAL_PASSWORD")), - Database: os.Getenv("PG_LOCAL_DATABASE"), - Port: helpers.GetIntFromEnv("PG_LOCAL_PORT"), - SlotID: "test_slot_id", - } - Source.WithDefaults() - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - }() - - conn, err := pgcommon.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - - canonizationCase := func(initSQL string, replicationSQL string) func(t *testing.T) { - return func(t *testing.T) { - _, err = conn.Exec(context.Background(), initSQL) - require.NoError(t, err) - - sequencer, produceSequenceDump := validator.Sequencer(t, DropNonRowKindsExceptRowMiddleware(), validator.RemoveVariableFieldsRowMiddleware, SynthesizeCommitTimeRowMiddleware()) - - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New(server.IsStrictSource(Source), sequencer), - Cleanup: server.Drop, - }, - abstract.TransferTypeSnapshotAndIncrement, - ) - - defer produceSequenceDump() - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - _, err = conn.Exec(context.Background(), replicationSQL) - require.NoError(t, err) - time.Sleep(time.Second * 10) - } - } - - t.Run(canon.SequenceTestCases[0], canonizationCase(string(initInsertUpdateDelete), string(insertUpdateDelete))) - t.Run(canon.SequenceTestCases[1], canonizationCase(string(initInsertUpdateDelete), string(updatePK))) - t.Run(canon.SequenceTestCases[2], canonizationCase(string(initInsertUpdateDelete), string(insertUpdateInsert))) - // new cases should be added here. The name of the cases MUST be placed in canon.SequenceTestCases so that the change in the set of cases is automatically propagated to all sinks under test -} - -// DropNonRowKindsExceptRowMiddleware drops all non-row items except the ones whose kind is among the given kinds -func DropNonRowKindsExceptRowMiddleware(preserveKinds ...abstract.Kind) func([]abstract.ChangeItem) []abstract.ChangeItem { - preserveKindsMap := make(map[abstract.Kind]bool) - for _, k := range preserveKinds { - preserveKindsMap[k] = true - } - return func(items []abstract.ChangeItem) []abstract.ChangeItem { - result := make([]abstract.ChangeItem, 0) - for i := range items { - if !items[i].IsRowEvent() && !preserveKindsMap[items[i].Kind] { - continue - } - result = append(result, items[i]) - } - return result - } -} - -// SynthesizeCommitTimeRowMiddleware sets synthetic sequential CommitTime for all items that it processes -func SynthesizeCommitTimeRowMiddleware() func([]abstract.ChangeItem) []abstract.ChangeItem { - return func(items []abstract.ChangeItem) []abstract.ChangeItem { - for i := range items { - items[i].CommitTime = uint64(i + 1) - } - return items - } -} diff --git a/tests/canon/ydb/canon_test.go b/tests/canon/ydb/canon_test.go deleted file mode 100644 index 9360ec25..00000000 --- a/tests/canon/ydb/canon_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package ydb - -import ( - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/ydb" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Common = &dpconfig.CommonConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestCanonSource(t *testing.T) { - Source := &ydb.YdbSource{ - Token: server.SecretString(os.Getenv("YDB_TOKEN")), - Database: helpers.GetEnvOfFail(t, "YDB_DATABASE"), - Instance: helpers.GetEnvOfFail(t, "YDB_ENDPOINT"), - Tables: []string{"canon_table"}, - TableColumnsFilter: nil, - SubNetworkID: "", - Underlay: false, - ServiceAccountID: "", - ChangeFeedMode: ydb.ChangeFeedModeNewImage, - UseFullPaths: false, - } - Source.WithDefaults() - runCanon( - t, - Source, - "canon_table", - validator.InitDone(t), - validator.ValuesTypeChecker, - validator.Canonizator(t), - validator.TypesystemChecker(ydb.ProviderType, func(colSchema abstract.ColSchema) string { - return strings.TrimPrefix(colSchema.OriginalType, "ydb:") - }), - ) -} - -func TestCanonLongPathSource(t *testing.T) { - Source := &ydb.YdbSource{ - Token: server.SecretString(os.Getenv("YDB_TOKEN")), - Database: helpers.GetEnvOfFail(t, "YDB_DATABASE"), - Instance: helpers.GetEnvOfFail(t, "YDB_ENDPOINT"), - Tables: nil, - TableColumnsFilter: nil, - SubNetworkID: "", - Underlay: false, - ServiceAccountID: "", - ChangeFeedMode: ydb.ChangeFeedModeNewImage, - UseFullPaths: false, - } - Source.WithDefaults() - t.Run("enable_full_path", func(t *testing.T) { - Source.Tables = []string{"foo/enable_full_path"} - Source.UseFullPaths = true - runCanon(t, Source, "foo/enable_full_path", validator.InitDone(t)) - }) - t.Run("disable_full_path", func(t *testing.T) { - Source.Tables = []string{"foo/disable_full_path"} - Source.UseFullPaths = false - runCanon(t, Source, "foo/disable_full_path", validator.InitDone(t)) - }) -} - -func runCanon(t *testing.T, Source *ydb.YdbSource, tablePath string, validators ...func() abstract.Sinker) { - Target := &ydb.YdbDestination{ - Database: Source.Database, - Token: Source.Token, - Instance: Source.Instance, - } - Target.WithDefaults() - sinker, err := ydb.NewSinker(logger.Log, Target, solomon.NewRegistry(solomon.NewRegistryOpts())) - require.NoError(t, err) - - currChangeItem := helpers.YDBInitChangeItem(tablePath) - require.NoError(t, sinker.Push([]abstract.ChangeItem{*currChangeItem})) - // null case - nullChangeItem := helpers.YDBInitChangeItem(tablePath) - require.Greater(t, len(nullChangeItem.ColumnNames), 0) - require.Equal(t, "id", nullChangeItem.ColumnNames[0]) - nullChangeItem.ColumnValues[0] = 801640048 - for i := 1; i < len(nullChangeItem.ColumnValues); i++ { - if nullChangeItem.TableSchema.Columns()[i].DataType == string(schema.TypeDate) || - nullChangeItem.TableSchema.Columns()[i].DataType == string(schema.TypeDatetime) || - nullChangeItem.TableSchema.Columns()[i].DataType == string(schema.TypeTimestamp) { - continue - } - nullChangeItem.ColumnValues[i] = nil - } - require.NoError(t, sinker.Push([]abstract.ChangeItem{*nullChangeItem})) - - counter, waiterSink := validator.NewCounter() - - validators = append(validators, waiterSink) - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New(server.IsStrictSource(Source), validators...), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotAndIncrement, - ) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - replicationChangeItem := helpers.YDBStmtInsert(t, tablePath, 2) - require.NoError(t, sinker.Push([]abstract.ChangeItem{*replicationChangeItem})) - - require.NoError(t, helpers.WaitCond(time.Second*60, - func() bool { - if counter.GetSum() != 2 { - logger.Log.Warnf(" counter rows sum (%v) is not equal to %v", counter.GetSum(), 2) - return false - } - return true - })) -} diff --git a/tests/canon/ydb/canondata/result.json b/tests/canon/ydb/canondata/result.json deleted file mode 100644 index 37b97514..00000000 --- a/tests/canon/ydb/canondata/result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "ydb.ydb.TestCanonSource/canon_0#01": { - "uri": "file://ydb.ydb.TestCanonSource_canon_0#01/extracted" - } -} diff --git a/tests/canon/yt/canon_test.go b/tests/canon/yt/canon_test.go deleted file mode 100644 index a5dd7885..00000000 --- a/tests/canon/yt/canon_test.go +++ /dev/null @@ -1,262 +0,0 @@ -package yt - -import ( - "context" - "math" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - yt2 "github.com/doublecloud/tross/pkg/providers/yt" - ytclient "github.com/doublecloud/tross/pkg/providers/yt/client" - "github.com/doublecloud/tross/tests/canon/validator" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/yt/go/schema" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yt" - "github.com/spf13/cast" - "github.com/stretchr/testify/require" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Common = &dpconfig.CommonConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -var TestData = []map[string]any{ - { - "t_int8": math.MinInt8, - "t_int16": math.MinInt16, - "t_int32": math.MinInt32, - "t_int64": math.MinInt64, - "t_uint8": 0, - "t_uint16": 0, - "t_uint32": 0, - "t_uint64": 0, - "t_float": float32(0.0), - "t_double": 0.0, - "t_bool": false, - "t_string": "", - "t_utf8": "", - "t_date": 0, // Min allowed by YT Date. - "t_datetime": 0, // Min allowed by YT Datetime. - "t_timestamp": 0, // Min allowed by YT Timestamp. - "t_interval": ytInterval(-49673*24*time.Hour + 1000), // Min allowed by YT Duration. - // "t_yson": It is optional field and not enabled here. - // "t_opt_int64": It is optional field and not enabled here. - "t_list": []float64{}, - "t_struct": map[string]any{"fieldInt16": 100, "fieldFloat32": 100.01, "fieldString": "abc"}, - "t_tuple": []any{-5, 300.03, "my data"}, - "t_variant_named": []any{"fieldInt16", 100}, - "t_variant_unnamed": []any{0, 100}, - "t_dict": [][]any{}, - "t_tagged": []any{"fieldInt16", 100}, - }, - { - "t_int8": 10, - "t_int16": -2000, - "t_int32": -200000, - "t_int64": -20000000000, - "t_uint8": 20, - "t_uint16": 2000, - "t_uint32": 2000000, - "t_uint64": 20000000000, - "t_float": float32(2.2), - "t_double": 2.2, - "t_bool": true, - "t_string": "Test byte string 2", - "t_utf8": "Test utf8 string 2", - "t_date": 1640604030 / secondsPerDay, - "t_datetime": 1640604030, - "t_timestamp": 1640604030502383, - "t_interval": ytInterval(time.Minute), - "t_yson": []uint64{100, 200, 300}, - "t_opt_int64": math.MaxInt64, - "t_list": []float64{-1.01}, - "t_struct": map[string]any{"fieldInt16": 100, "fieldFloat32": 100.01, "fieldString": "abc"}, - "t_tuple": []any{-5, 300.03, "my data"}, - "t_variant_named": []any{"fieldFloat32", 100.01}, - "t_variant_unnamed": []any{1, 100.01}, - "t_dict": [][]any{{"my_key", 100}}, - "t_tagged": []any{"fieldFloat32", 100.01}, - }, - { - "t_int8": math.MaxInt8, - "t_int16": math.MaxInt16, - "t_int32": math.MaxInt32, - "t_int64": math.MaxInt64, - "t_uint8": math.MaxUint8, - "t_uint16": math.MaxInt16, // TODO: Replace to math.MaxUint16 while fixing TM-7588. - "t_uint32": math.MaxInt32, // TODO: Replace to math.MaxUint32 while fixing TM-7588. - "t_uint64": math.MaxInt64, // TODO: Replace to math.MaxUint32 while fixing TM-7588. - "t_float": float32(42), - "t_double": 42.0, - "t_bool": false, - "t_string": "Test byte string 3", - "t_utf8": "Test utf8 string 3", - "t_date": cast.ToTime("2105-12-31T23:59:59").Unix() / secondsPerDay, // Max allowed by YT Date. - "t_datetime": cast.ToTime("2105-12-31T23:59:59").Unix(), // Max allowed by YT Datetime. - "t_timestamp": cast.ToTime("2105-12-31 23:59:59").UnixMicro(), // TODO: Max allowed by CH-target Timestamp. - "t_interval": ytInterval(49673*24*time.Hour - 1000), // Max allowed by YT Duration. - "t_yson": nil, - "t_opt_int64": nil, - "t_list": []float64{-1.01, 2.0, 1294.21}, - "t_struct": map[string]any{"fieldInt16": 100, "fieldFloat32": 100.01, "fieldString": "abc"}, - "t_tuple": []any{-5, 300.03, "my data"}, - "t_variant_named": []any{"fieldString", "magotan"}, - "t_variant_unnamed": []any{2, "magotan"}, - "t_dict": [][]any{{"key1", 1}, {"key2", 20}, {"key3", 300}}, - "t_tagged": []any{"fieldString", "100"}, - }, -} - -func ytInterval(duration time.Duration) schema.Interval { - res, err := schema.NewInterval(duration) - if err != nil { - panic(err) - } - return res -} - -var ( - members = []schema.StructMember{ - {Name: "fieldInt16", Type: schema.TypeInt16}, - {Name: "fieldFloat32", Type: schema.TypeFloat32}, - {Name: "fieldString", Type: schema.TypeString}, - } - elements = []schema.TupleElement{ - {Type: schema.TypeInt16}, - {Type: schema.TypeFloat32}, - {Type: schema.TypeString}, - } - secondsPerDay = int64(24 * 60 * 60) -) - -var YtColumns = []schema.Column{ - // Primitives - {Name: "t_int8", ComplexType: schema.TypeInt8, SortOrder: schema.SortAscending}, - {Name: "t_int16", ComplexType: schema.TypeInt16}, - {Name: "t_int32", ComplexType: schema.TypeInt32}, - {Name: "t_int64", ComplexType: schema.TypeInt64}, - {Name: "t_uint8", ComplexType: schema.TypeUint8}, - {Name: "t_uint16", ComplexType: schema.TypeUint16}, - {Name: "t_uint32", ComplexType: schema.TypeUint32}, - {Name: "t_uint64", ComplexType: schema.TypeUint64}, - {Name: "t_float", ComplexType: schema.TypeFloat32}, - {Name: "t_double", ComplexType: schema.TypeFloat64}, - {Name: "t_bool", ComplexType: schema.TypeBoolean}, - {Name: "t_string", ComplexType: schema.TypeBytes}, - {Name: "t_utf8", ComplexType: schema.TypeString}, - {Name: "t_date", ComplexType: schema.TypeDate}, - {Name: "t_datetime", ComplexType: schema.TypeDatetime}, - {Name: "t_timestamp", ComplexType: schema.TypeTimestamp}, - {Name: "t_interval", ComplexType: schema.TypeInterval}, // FIXME: support in CH - {Name: "t_yson", ComplexType: schema.Optional{Item: schema.TypeAny}}, - {Name: "t_opt_int64", ComplexType: schema.Optional{Item: schema.TypeInt64}}, - {Name: "t_list", ComplexType: schema.List{Item: schema.TypeFloat64}}, - {Name: "t_struct", ComplexType: schema.Struct{Members: members}}, - {Name: "t_tuple", ComplexType: schema.Tuple{Elements: elements}}, - {Name: "t_variant_named", ComplexType: schema.Variant{Members: members}}, - {Name: "t_variant_unnamed", ComplexType: schema.Variant{Elements: elements}}, - {Name: "t_dict", ComplexType: schema.Dict{Key: schema.TypeString, Value: schema.TypeInt64}}, - {Name: "t_tagged", ComplexType: schema.Tagged{Tag: "mytag", Item: schema.Variant{Members: members}}}, -} - -func TestCanonSource(t *testing.T) { - Source := &yt2.YtSource{ - Cluster: os.Getenv("YT_PROXY"), - Proxy: os.Getenv("YT_PROXY"), - Paths: []string{"//home/cdc/junk/test_table"}, - YtToken: "", - RowIdxColumnName: "row_idx", - } - Source.WithDefaults() - - createTestData(t, Source, Source.Paths[0]) - - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New(server.IsStrictSource(Source), validator.Canonizator(t)), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotOnly, - ) - _ = helpers.Activate(t, transfer) -} - -func TestCanonSourceWithDataObjects(t *testing.T) { - Source := &yt2.YtSource{ - Cluster: os.Getenv("YT_PROXY"), - Proxy: os.Getenv("YT_PROXY"), - Paths: []string{"//home/cdc/junk/test_parent_dir"}, - YtToken: "", - RowIdxColumnName: "row_idx", - } - Source.WithDefaults() - - createTestData(t, Source, "//home/cdc/junk/test_parent_dir/nested_dir/some_table") - - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New(server.IsStrictSource(Source), validator.Canonizator(t)), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotOnly, - ) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{"//home/cdc/junk/test_parent_dir/nested_dir/some_table"}} - _ = helpers.Activate(t, transfer) -} - -func TestCanonSourceWithDirInDataObjects(t *testing.T) { - Source := &yt2.YtSource{ - Cluster: os.Getenv("YT_PROXY"), - Proxy: os.Getenv("YT_PROXY"), - Paths: []string{"//home/cdc/junk/test_parent_dir"}, - YtToken: "", - RowIdxColumnName: "row_idx", - } - Source.WithDefaults() - - createTestData(t, Source, "//home/cdc/junk/test_parent_dir/nested_dir2/nested_dir3/some_table2") - - transfer := helpers.MakeTransfer( - helpers.TransferID, - Source, - &server.MockDestination{ - SinkerFactory: validator.New(server.IsStrictSource(Source), validator.Canonizator(t)), - Cleanup: server.DisabledCleanup, - }, - abstract.TransferTypeSnapshotOnly, - ) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{"//home/cdc/junk/test_parent_dir/nested_dir2"}} - _ = helpers.Activate(t, transfer) -} - -func createTestData(t *testing.T, Source *yt2.YtSource, path string) { - ytc, err := ytclient.NewYtClientWrapper(ytclient.HTTP, nil, &yt.Config{Proxy: Source.Proxy}) - require.NoError(t, err) - _ = ytc.RemoveNode(context.Background(), ypath.NewRich(path).YPath(), nil) - - sch := schema.Schema{ - Strict: nil, - UniqueKeys: false, - Columns: YtColumns, - } - - ctx := context.Background() - wr, err := yt.WriteTable(ctx, ytc, ypath.NewRich(path).YPath(), yt.WithCreateOptions(yt.WithSchema(sch), yt.WithRecursive())) - require.NoError(t, err) - // var optint int64 = 10050 - for _, row := range TestData { - require.NoError(t, wr.Write(row)) - } - require.NoError(t, wr.Commit()) -} diff --git a/tests/canon/yt/canondata/result.json b/tests/canon/yt/canondata/result.json deleted file mode 100644 index 6839f4d9..00000000 --- a/tests/canon/yt/canondata/result.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "yt.yt.TestCanonSource/canon_0": { - "uri": "file://yt.yt.TestCanonSource_canon_0/extracted" - }, - "yt.yt.TestCanonSourceWithDataObjects/canon_0": { - "uri": "file://yt.yt.TestCanonSourceWithDataObjects_canon_0/extracted" - }, - "yt.yt.TestCanonSourceWithDirInDataObjects/canon_0": { - "uri": "file://yt.yt.TestCanonSourceWithDirInDataObjects_canon_0/extracted" - } -} diff --git a/tests/e2e/ch2ch/db_alt_name/check_db_test.go b/tests/e2e/ch2ch/db_alt_name/check_db_test.go deleted file mode 100644 index 17624db6..00000000 --- a/tests/e2e/ch2ch/db_alt_name/check_db_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *chrecipe.MustSource(chrecipe.WithInitFile("dump/src.sql"), chrecipe.WithDatabase("mtmobproxy")) - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase("mtmobproxy2"), chrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH source", Port: Source.NativePort}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - Target.AltNamesList = []server.AltName{ - { - From: "sample_table", - To: "sample_table2", - }, - } - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - helpers.Activate(t, transfer) - - dstStorage := helpers.GetSampleableStorageByModel(t, Target) - var res []abstract.ChangeItem - require.NoError(t, dstStorage.LoadTable(context.Background(), abstract.TableDescription{ - Name: "sample_table2", - Schema: "mtmobproxy2", - }, func(input []abstract.ChangeItem) error { - for _, row := range input { - if row.IsRowEvent() { - res = append(res, row) - } - } - return nil - })) - require.Len(t, res, 5) -} diff --git a/tests/e2e/ch2ch/db_alt_name/dump/dst.sql b/tests/e2e/ch2ch/db_alt_name/dump/dst.sql deleted file mode 100644 index 8c6d369f..00000000 --- a/tests/e2e/ch2ch/db_alt_name/dump/dst.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE mtmobproxy2; diff --git a/tests/e2e/ch2ch/db_alt_name/dump/src.sql b/tests/e2e/ch2ch/db_alt_name/dump/src.sql deleted file mode 100644 index 24040135..00000000 --- a/tests/e2e/ch2ch/db_alt_name/dump/src.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE DATABASE mtmobproxy; - -create table mtmobproxy.sample_table -( - `date` Date, - `value` String -) - ENGINE = MergeTree - Partition By toMonday(date) - ORDER BY date; - -insert into mtmobproxy.sample_table -(`date`, `value`) -values - ('2012-01-01', 'test1'), ('2012-01-01', 'test2'), ('2012-01-02', 'test3'), ('2012-01-02', 'test4'), ('2012-01-03', 'test5') -; diff --git a/tests/e2e/ch2ch/db_complex_name/check_db_test.go b/tests/e2e/ch2ch/db_complex_name/check_db_test.go deleted file mode 100644 index 9f2abe6f..00000000 --- a/tests/e2e/ch2ch/db_complex_name/check_db_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "mt-mob-proxy" - TransferType = abstract.TransferTypeSnapshotOnly - Source = *chrecipe.MustSource(chrecipe.WithInitFile("dump/src.sql"), chrecipe.WithDatabase(databaseName)) - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase(databaseName), chrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH source", Port: Source.NativePort}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - require.NoError(t, tasks.ActivateDelivery(context.Background(), nil, coordinator.NewFakeClient(), *transfer, solomon.NewRegistry(solomon.NewRegistryOpts()))) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/ch2ch/db_complex_name/dump/dst.sql b/tests/e2e/ch2ch/db_complex_name/dump/dst.sql deleted file mode 100644 index c392ec91..00000000 --- a/tests/e2e/ch2ch/db_complex_name/dump/dst.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `mt-mob-proxy`; diff --git a/tests/e2e/ch2ch/db_complex_name/dump/src.sql b/tests/e2e/ch2ch/db_complex_name/dump/src.sql deleted file mode 100644 index 9dfdcb63..00000000 --- a/tests/e2e/ch2ch/db_complex_name/dump/src.sql +++ /dev/null @@ -1,185 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `mt-mob-proxy`; - - --- MergeTree->MergeTree - - -CREATE TABLE `mt-mob-proxy`.logs_weekly__mt_mt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String -) -ENGINE = MergeTree() -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, ServerName, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - -INSERT INTO `mt-mob-proxy`.logs_weekly__mt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`) -VALUES -('my-server', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod') -; - - --- NotUpdateableReplacingMergeTree->MergeTree - - -CREATE TABLE `mt-mob-proxy`.logs_weekly__nurmt_mt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String, - `commit_time` UInt64, - `delete_time` UInt64 -) -ENGINE = ReplacingMergeTree(commit_time) -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, ServerName, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - --- insert 3 rows (my-server2/my-server3/my-server4) -INSERT INTO `mt-mob-proxy`.logs_weekly__nurmt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0 ),('my-server3', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server4', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0) -; - --- update 1 row (my-server2) -INSERT INTO `mt-mob-proxy`.logs_weekly__nurmt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 9, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 2, 0 ) -; - - --- NotUpdateableReplacingMergeTree->NotUpdateableReplacingMergeTree - - -CREATE TABLE `mt-mob-proxy`.logs_weekly__nurmt_nurmt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String, - `commit_time` UInt64, - `delete_time` UInt64 -) -ENGINE = ReplacingMergeTree(commit_time) -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, ServerName, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - --- insert 3 rows (my-server2/my-server3/my-server4) -INSERT INTO `mt-mob-proxy`.logs_weekly__nurmt_nurmt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0 ),('my-server3', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server4', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0) -; - --- update 1 row (my-server2) -INSERT INTO `mt-mob-proxy`.logs_weekly__nurmt_nurmt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 9, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 2, 0 ) -; - - --- UpdateableReplacingMergeTree->MergeTree - - -CREATE TABLE `mt-mob-proxy`.`.-logs_weekly__urmt_mt` -( - `Server-.-Name` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String, - `__data_transfer_commit_time` UInt64, - `__data_transfer_delete_time` UInt64 -) -ENGINE = ReplacingMergeTree(__data_transfer_commit_time) -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, `Server-.-Name`, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - --- insert 4 rows (my-server2/my-server3/my-server4) -INSERT INTO `mt-mob-proxy`.`.-logs_weekly__urmt_mt` -(`Server-.-Name`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `__data_transfer_commit_time`, `__data_transfer_delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0 ),('my-server3', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server4', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server5', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0) -; - --- update 1 row (my-server2) -INSERT INTO `mt-mob-proxy`.`.-logs_weekly__urmt_mt` -(`Server-.-Name`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `__data_transfer_commit_time`, `__data_transfer_delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 9, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 2, 0 ) -; - - - --- Empty table - - -CREATE TABLE `mt-mob-proxy`.empty -( - `BasePath` String DEFAULT 'misc', - `Code` UInt16 -) -ENGINE = MergeTree() -ORDER BY (BasePath) -SETTINGS index_granularity = 8192; diff --git a/tests/e2e/ch2ch/incremental_many_shards/check_db_test.go b/tests/e2e/ch2ch/incremental_many_shards/check_db_test.go deleted file mode 100644 index 5cd88250..00000000 --- a/tests/e2e/ch2ch/incremental_many_shards/check_db_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package snapshot - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "db" - tableName = "test_table" - TransferType = abstract.TransferTypeSnapshotOnly - Source = *chrecipe.MustSource(chrecipe.WithInitFile("dump/src.sql"), chrecipe.WithDatabase(databaseName)) - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase(databaseName), chrecipe.WithPrefix("DB0_")) -) - -const cursorField = "Birthday" -const cursorValue = "2019-01-03" - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) - Source.ShardsList = []model.ClickHouseShard{ - {Name: "_", Hosts: []string{"localhost"}}, - {Name: "[", Hosts: []string{"localhost"}}, - } - Target.Cleanup = server.DisabledCleanup -} - -func TestIncrementalSnapshot(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH source", Port: Source.NativePort}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - - transfer := helpers.MakeTransferForIncrementalSnapshot(helpers.TransferID, &Source, &Target, TransferType, databaseName, tableName, cursorField, cursorValue, 15) - transfer.Runtime = new(abstract.LocalRuntime) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewStatefulFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.WaitDestinationEqualRowsCount(databaseName, tableName, helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 7)) - // 7 and not 5, because we had to specify the same host in two shards - - conn, err := clickhouse.MakeConnection(Source.ToStorageParams(), transfer) - require.NoError(t, err) - - addData(t, conn) - - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.WaitDestinationEqualRowsCount(databaseName, tableName, helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 9)) - // 9 and not 8, because we had to specify the same host in two shards - - ids := readIdsFromTarget(t, helpers.GetSampleableStorageByModel(t, Target)) - require.True(t, slices.ContainsAll(ids, []uint16{1, 2, 3, 4, 5, 7})) -} - -func addData(t *testing.T, conn *sql.DB) { - query := fmt.Sprintf("INSERT INTO %s.%s (`Id`, `Name`, `Age`, `Birthday`) VALUES (7, 'Mary', 19, '2019-01-07');", databaseName, tableName) - _, err := conn.Exec(query) - require.NoError(t, err) -} - -func readIdsFromTarget(t *testing.T, storage abstract.SampleableStorage) []uint16 { - ids := make([]uint16, 0) - - require.NoError(t, storage.LoadTable(context.Background(), abstract.TableDescription{ - Name: tableName, - Schema: databaseName, - Filter: "", - EtaRow: 0, - Offset: 0, - }, func(items []abstract.ChangeItem) error { - for _, row := range items { - if !row.IsRowEvent() { - continue - } - id := row.ColumnNameIndex("Id") - ids = append(ids, row.ColumnValues[id].(uint16)) - } - return nil - })) - return ids -} diff --git a/tests/e2e/ch2ch/incremental_many_shards/dump/dst.sql b/tests/e2e/ch2ch/incremental_many_shards/dump/dst.sql deleted file mode 100644 index be8678f6..00000000 --- a/tests/e2e/ch2ch/incremental_many_shards/dump/dst.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE DATABASE IF NOT EXISTS db; - - -CREATE TABLE db.test_table -( - `Id` UInt16, - `Name` String, - `Age` UInt16, - `Birthday` Date -) - ENGINE = MergeTree() -ORDER BY (Age); - -INSERT INTO db.test_table -(`Id`, `Name`, `Age`, `Birthday`) -VALUES -(1, 'Bob', 20, '2019-01-01'), (2, 'Gwen', 25, '2019-01-02'), (3, 'John', 45, '2019-01-03') -; diff --git a/tests/e2e/ch2ch/incremental_many_shards/dump/src.sql b/tests/e2e/ch2ch/incremental_many_shards/dump/src.sql deleted file mode 100644 index cb559769..00000000 --- a/tests/e2e/ch2ch/incremental_many_shards/dump/src.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE DATABASE IF NOT EXISTS db; - - -CREATE TABLE db.test_table -( - `Id` UInt16, - `Name` String, - `Age` UInt16, - `Birthday` Date -) -ENGINE = MergeTree() -ORDER BY (Age); - -INSERT INTO db.test_table -(`Id`, `Name`, `Age`, `Birthday`) -VALUES -(1, 'Bob', 20, '2019-01-01'), (2, 'Gwen', 25, '2019-01-02'), (3, 'John', 45, '2019-01-03'), (4, 'Soul', 12, '2019-01-04'), (5, 'Em', 30, '2019-01-05') -; diff --git a/tests/e2e/ch2ch/incremental_one_shard/check_db_test.go b/tests/e2e/ch2ch/incremental_one_shard/check_db_test.go deleted file mode 100644 index 64945d52..00000000 --- a/tests/e2e/ch2ch/incremental_one_shard/check_db_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package snapshot - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "db" - tableName = "test_table" - TransferType = abstract.TransferTypeSnapshotOnly - Source = *chrecipe.MustSource(chrecipe.WithInitFile("dump/src.sql"), chrecipe.WithDatabase(databaseName)) - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase(databaseName), chrecipe.WithPrefix("DB0_")) -) - -const cursorField = "Birthday" -const cursorValue = "2019-01-03" - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) - Target.Cleanup = server.DisabledCleanup -} - -func TestIncrementalSnapshot(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH source", Port: Source.NativePort}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - - transfer := helpers.MakeTransferForIncrementalSnapshot(helpers.TransferID, &Source, &Target, TransferType, databaseName, tableName, cursorField, cursorValue, 15) - transfer.Runtime = new(abstract.LocalRuntime) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewStatefulFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.WaitDestinationEqualRowsCount(databaseName, tableName, helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 5)) - - conn, err := clickhouse.MakeConnection(Source.ToStorageParams(), transfer) - require.NoError(t, err) - - addData(t, conn) - - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.WaitDestinationEqualRowsCount(databaseName, tableName, helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 6)) - - ids := readIdsFromTarget(t, helpers.GetSampleableStorageByModel(t, Target)) - require.True(t, slices.ContainsAll(ids, []uint16{1, 2, 3, 4, 5, 7})) -} - -func addData(t *testing.T, conn *sql.DB) { - query := fmt.Sprintf("INSERT INTO %s.%s (`Id`, `Name`, `Age`, `Birthday`) VALUES (7, 'Mary', 19, '2019-01-07');", databaseName, tableName) - _, err := conn.Exec(query) - require.NoError(t, err) -} - -func readIdsFromTarget(t *testing.T, storage abstract.SampleableStorage) []uint16 { - ids := make([]uint16, 0) - - require.NoError(t, storage.LoadTable(context.Background(), abstract.TableDescription{ - Name: tableName, - Schema: databaseName, - Filter: "", - EtaRow: 0, - Offset: 0, - }, func(items []abstract.ChangeItem) error { - for _, row := range items { - if !row.IsRowEvent() { - continue - } - id := row.ColumnNameIndex("Id") - ids = append(ids, row.ColumnValues[id].(uint16)) - } - return nil - })) - return ids -} diff --git a/tests/e2e/ch2ch/incremental_one_shard/dump/dst.sql b/tests/e2e/ch2ch/incremental_one_shard/dump/dst.sql deleted file mode 100644 index be8678f6..00000000 --- a/tests/e2e/ch2ch/incremental_one_shard/dump/dst.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE DATABASE IF NOT EXISTS db; - - -CREATE TABLE db.test_table -( - `Id` UInt16, - `Name` String, - `Age` UInt16, - `Birthday` Date -) - ENGINE = MergeTree() -ORDER BY (Age); - -INSERT INTO db.test_table -(`Id`, `Name`, `Age`, `Birthday`) -VALUES -(1, 'Bob', 20, '2019-01-01'), (2, 'Gwen', 25, '2019-01-02'), (3, 'John', 45, '2019-01-03') -; diff --git a/tests/e2e/ch2ch/incremental_one_shard/dump/src.sql b/tests/e2e/ch2ch/incremental_one_shard/dump/src.sql deleted file mode 100644 index cb559769..00000000 --- a/tests/e2e/ch2ch/incremental_one_shard/dump/src.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE DATABASE IF NOT EXISTS db; - - -CREATE TABLE db.test_table -( - `Id` UInt16, - `Name` String, - `Age` UInt16, - `Birthday` Date -) -ENGINE = MergeTree() -ORDER BY (Age); - -INSERT INTO db.test_table -(`Id`, `Name`, `Age`, `Birthday`) -VALUES -(1, 'Bob', 20, '2019-01-01'), (2, 'Gwen', 25, '2019-01-02'), (3, 'John', 45, '2019-01-03'), (4, 'Soul', 12, '2019-01-04'), (5, 'Em', 30, '2019-01-05') -; diff --git a/tests/e2e/ch2ch/snapshot/check_db_test.go b/tests/e2e/ch2ch/snapshot/check_db_test.go deleted file mode 100644 index 186d043e..00000000 --- a/tests/e2e/ch2ch/snapshot/check_db_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package snapshot - -import ( - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/clickhouse/conn" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/tests/helpers" - proxy "github.com/doublecloud/tross/tests/helpers/http_proxy" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "mtmobproxy" - TransferType = abstract.TransferTypeSnapshotOnly - Source = *chrecipe.MustSource(chrecipe.WithInitFile("dump/src.sql"), chrecipe.WithDatabase(databaseName)) - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase(databaseName), chrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH source", Port: Source.NativePort}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - srcProxy := proxy.NewHTTPProxyWithPortAllocation(fmt.Sprintf("localhost:%d", Source.HTTPPort)) - srcProxy.WithLogger = true - srcWorker := srcProxy.RunAsync() - defer srcWorker.Close() - fmt.Printf("Source.HTTPPort:%d, srcProxy.ListenPort:%d\n", Source.HTTPPort, srcProxy.ListenPort) - Source.HTTPPort = srcProxy.ListenPort - - dstProxy := proxy.NewHTTPProxyWithPortAllocation(fmt.Sprintf("localhost:%d", Target.HTTPPort)) - dstProxy.WithLogger = true - dstWorker := dstProxy.RunAsync() - defer dstWorker.Close() - fmt.Printf("Target.HTTPPort:%d, dstProxy.ListenPort:%d\n", Target.HTTPPort, dstProxy.ListenPort) - Target.HTTPPort = dstProxy.ListenPort - - t.Run("default, CSV case", func(t *testing.T) { - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - helpers.Activate(t, transfer) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "FORMAT CSV")) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "timeout_before_checking_execution_speed=0")) - require.True(t, proxy.CheckRequestContains(dstProxy.GetSniffedData(), "FORMAT CSV")) - }) - - t.Run("drop", func(t *testing.T) { - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - db, err := conn.ConnectNative("localhost", Target.ToSinkParams(transfer)) - require.NoError(t, err) - - exec := func(query string) { - _, err := db.Exec(query) - require.NoError(t, err) - } - - exec(`drop table mtmobproxy.logs_weekly__mt_mt`) - exec(`drop table mtmobproxy.logs_weekly__nurmt_mt`) - exec(`drop table mtmobproxy.logs_weekly__nurmt_nurmt`) - exec("drop table mtmobproxy.`.-logs_weekly__urmt_mt`") - exec(`drop table mtmobproxy.empty`) - - srcProxy.ResetSniffedData() - dstProxy.ResetSniffedData() - }) - - t.Run("JSONCompactEachRow case", func(t *testing.T) { - Source.IOHomoFormat = model.ClickhouseIOFormatJSONCompact - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - helpers.Activate(t, transfer) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "FORMAT JSONCompactEachRow")) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "timeout_before_checking_execution_speed=0")) - require.True(t, proxy.CheckRequestContains(dstProxy.GetSniffedData(), "FORMAT JSONCompactEachRow")) - }) -} diff --git a/tests/e2e/ch2ch/snapshot/dump/dst.sql b/tests/e2e/ch2ch/snapshot/dump/dst.sql deleted file mode 100644 index d6547eec..00000000 --- a/tests/e2e/ch2ch/snapshot/dump/dst.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE IF NOT EXISTS mtmobproxy; diff --git a/tests/e2e/ch2ch/snapshot/dump/src.sql b/tests/e2e/ch2ch/snapshot/dump/src.sql deleted file mode 100644 index aedd38ec..00000000 --- a/tests/e2e/ch2ch/snapshot/dump/src.sql +++ /dev/null @@ -1,225 +0,0 @@ -CREATE DATABASE IF NOT EXISTS mtmobproxy; - - --- MergeTree->MergeTree - - -CREATE TABLE mtmobproxy.logs_weekly__mt_mt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String -) -ENGINE = MergeTree() -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, ServerName, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - -INSERT INTO mtmobproxy.logs_weekly__mt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`) -VALUES -('my-server', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod') -; - - --- NotUpdateableReplacingMergeTree->MergeTree - - -CREATE TABLE mtmobproxy.logs_weekly__nurmt_mt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String, - `commit_time` UInt64, - `delete_time` UInt64 -) -ENGINE = ReplacingMergeTree(commit_time) -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, ServerName, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - --- insert 3 rows (my-server2/my-server3/my-server4) -INSERT INTO mtmobproxy.logs_weekly__nurmt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0 ),('my-server3', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server4', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0) -; - --- update 1 row (my-server2) -INSERT INTO mtmobproxy.logs_weekly__nurmt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 9, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 2, 0 ) -; - - --- NotUpdateableReplacingMergeTree->NotUpdateableReplacingMergeTree - - -CREATE TABLE mtmobproxy.logs_weekly__nurmt_nurmt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String, - `commit_time` UInt64, - `delete_time` UInt64 -) -ENGINE = ReplacingMergeTree(commit_time) -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, ServerName, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - --- insert 3 rows (my-server2/my-server3/my-server4) -INSERT INTO mtmobproxy.logs_weekly__nurmt_nurmt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0 ),('my-server3', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server4', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0) -; - --- update 1 row (my-server2) -INSERT INTO mtmobproxy.logs_weekly__nurmt_nurmt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `commit_time`, `delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 9, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 2, 0 ) -; - - --- UpdateableReplacingMergeTree->MergeTree - - -CREATE TABLE mtmobproxy.`.-logs_weekly__urmt_mt` -( - `Server-.-Name` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String, - `__data_transfer_commit_time` UInt64, - `__data_transfer_delete_time` UInt64 -) -ENGINE = ReplacingMergeTree(__data_transfer_commit_time) -PARTITION BY toMonday(RequestDate) -ORDER BY (BasePath, Code, `Server-.-Name`, StringHash) -SAMPLE BY StringHash -SETTINGS index_granularity = 8192; - --- insert 4 rows (my-server2/my-server3/my-server4) -INSERT INTO mtmobproxy.`.-logs_weekly__urmt_mt` -(`Server-.-Name`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `__data_transfer_commit_time`, `__data_transfer_delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0 ),('my-server3', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server4', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0),('my-server5', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 1, 0) -; - --- update 1 row (my-server2) -INSERT INTO mtmobproxy.`.-logs_weekly__urmt_mt` -(`Server-.-Name`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`, `__data_transfer_commit_time`, `__data_transfer_delete_time`) -VALUES -('my-server2', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 9, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod', 2, 0 ) -; - - - --- Empty table - - -CREATE TABLE mtmobproxy.empty -( - `BasePath` String DEFAULT 'misc', - `Code` UInt16 -) -ENGINE = MergeTree() -ORDER BY (BasePath) -SETTINGS index_granularity = 8192; - - - --- Table with long line (exceeds 4096 chars) - - -CREATE TABLE mtmobproxy.long_line -( - `id` UInt64, - `BasePath` String -) -ENGINE = MergeTree() -ORDER BY (id) -SETTINGS index_granularity = 8192; - -INSERT INTO mtmobproxy.long_line -(`id`, `BasePath`) -VALUES -(1, repeat('a', 5000)) -; - - --- Table with values with fake backslash escaping - - -CREATE TABLE mtmobproxy.backslashes -( - `id` UInt64, - `v0` String, - `v1` UInt64 -) -ENGINE = MergeTree() -ORDER BY (id) -SETTINGS index_granularity = 8192; - -INSERT INTO mtmobproxy.backslashes -(`id`, `v0`, `v1`) -VALUES -(1, '\\',3) -; diff --git a/tests/e2e/ch2ch/snapshot_test_csv_different_values/check_db_test.go b/tests/e2e/ch2ch/snapshot_test_csv_different_values/check_db_test.go deleted file mode 100644 index 0dd24307..00000000 --- a/tests/e2e/ch2ch/snapshot_test_csv_different_values/check_db_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package snapshot - -import ( - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/clickhouse/conn" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/tests/helpers" - proxy "github.com/doublecloud/tross/tests/helpers/http_proxy" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "some_db" - TransferType = abstract.TransferTypeSnapshotOnly - Source = *chrecipe.MustSource(chrecipe.WithInitFile("dump/src.sql"), chrecipe.WithDatabase(databaseName)) - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase(databaseName), chrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH source", Port: Source.NativePort}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - srcProxy := proxy.NewHTTPProxyWithPortAllocation(fmt.Sprintf("localhost:%d", Source.HTTPPort)) - srcProxy.WithLogger = true - srcWorker := srcProxy.RunAsync() - defer srcWorker.Close() - fmt.Printf("Source.HTTPPort:%d, srcProxy.ListenPort:%d\n", Source.HTTPPort, srcProxy.ListenPort) - Source.HTTPPort = srcProxy.ListenPort - Source.BufferSize = 500 - - dstProxy := proxy.NewHTTPProxyWithPortAllocation(fmt.Sprintf("localhost:%d", Target.HTTPPort)) - dstProxy.WithLogger = true - dstWorker := dstProxy.RunAsync() - defer dstWorker.Close() - Target.HTTPPort = dstProxy.ListenPort - - t.Run("default, CSV case", func(t *testing.T) { - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - helpers.Activate(t, transfer) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "FORMAT CSV")) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "timeout_before_checking_execution_speed=0")) - require.True(t, proxy.CheckRequestContains(dstProxy.GetSniffedData(), "FORMAT CSV")) - }) - - t.Run("drop", func(t *testing.T) { - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - db, err := conn.ConnectNative("localhost", Target.ToSinkParams(transfer)) - require.NoError(t, err) - - exec := func(query string) { - _, err := db.Exec(query) - require.NoError(t, err) - } - - exec("drop table some_db.some_table") - - srcProxy.ResetSniffedData() - dstProxy.ResetSniffedData() - }) - - t.Run("JSONCompactEachRow case", func(t *testing.T) { - Source.IOHomoFormat = model.ClickhouseIOFormatJSONCompact - transfer := helpers.MakeTransfer("fake", &Source, &Target, abstract.TransferTypeSnapshotOnly) - helpers.Activate(t, transfer) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "FORMAT JSONCompactEachRow")) - require.True(t, proxy.CheckRequestContains(srcProxy.GetSniffedData(), "timeout_before_checking_execution_speed=0")) - require.True(t, proxy.CheckRequestContains(dstProxy.GetSniffedData(), "FORMAT JSONCompactEachRow")) - }) -} diff --git a/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/dst.sql b/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/dst.sql deleted file mode 100644 index 46ef64a0..00000000 --- a/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/dst.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE IF NOT EXISTS some_db; diff --git a/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/src.sql b/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/src.sql deleted file mode 100644 index 578d5f67..00000000 --- a/tests/e2e/ch2ch/snapshot_test_csv_different_values/dump/src.sql +++ /dev/null @@ -1,59 +0,0 @@ -CREATE DATABASE IF NOT EXISTS some_db; - -CREATE TABLE some_db.some_table -( - `StringVal` String, - `DateVal` Date, - `OneMoreStringVal` String, - `DateTimeVal` DateTime, - `StringWithDefaultVal` String DEFAULT 'misc', - `NullableStringVal` Nullable(String), - `UInt8Val` UInt8, - `Int16Val` Int16, - `Int32Val` Int32, - `UInt64Val` UInt64, - `Enum8Val` Enum8('false' = 0, 'true' = 1) -) - ENGINE = MergeTree() - PARTITION BY toMonday(DateVal) - ORDER BY (StringVal, DateVal, OneMoreStringVal) - SAMPLE BY OneMoreStringVal - SETTINGS index_granularity = 8192; - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES ('Death is a natural part of life. Rejoice for those around you who transform into the Force. Mourn them do not. Miss them do not. Attachment leads to jealously. The shadow of greed, that is.', '2019-01-01', 'simple string', 1546300800, 'qwe', null, 12, 1234, -4321, 123123, 'false'); - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES ('ab', '2019-01-02', 'bc', 1546300800, 'cd', 'de', 12, 34, 56, 78, 'true'); - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES (';', '2019-01-03', ';', 1546300800, ';', ';', 12, 34, 56, 78, 'false') -; - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES ('","', '2019-01-04', '""', 1546300800, '"', ';', 12, 34, 56, 78, 'true'); - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES ('"""', '2019-01-05', '"', 1546300800, '"""', '\n', 12, 34, 56, 78, 'false'); - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES ('""asd"a', '2019-01-06', 's,"ada', 1546300800, '"adads"er"', '\\n', 12, 34, 56, 78, 'true'); - -INSERT INTO some_db.some_table -(`StringVal`, `DateVal`, `OneMoreStringVal`, `DateTimeVal`, `StringWithDefaultVal`, - `NullableStringVal`, `UInt8Val`, `Int16Val`, `Int32Val`, `UInt64Val`, `Enum8Val`) -VALUES ('""klaz-klaz', '2019-01-07', '{PO{PI^D&CV,"()', 1546300800, '""_)&*^(&%#^%#@', '\\t\\t', 12, 34, 56, 78, 'true'); - -SET format_csv_delimiter = ';' diff --git a/tests/e2e/ch2yt/static_table/check_db_test.go b/tests/e2e/ch2yt/static_table/check_db_test.go deleted file mode 100644 index 714230f7..00000000 --- a/tests/e2e/ch2yt/static_table/check_db_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - ytcommon "github.com/doublecloud/tross/pkg/providers/yt" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestMain(m *testing.M) { - ytcommon.InitExe() - os.Exit(m.Run()) -} - -func TestClickhouseToYtStatic(t *testing.T) { - src := &model.ChSource{ - ShardsList: []model.ClickHouseShard{ - { - Name: "_", - Hosts: []string{ - "localhost", - }, - }, - }, - User: "default", - Password: "", - Database: "mtmobproxy", - HTTPPort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_HTTP_PORT"), - NativePort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_NATIVE_PORT"), - } - src.WithDefaults() - - dstModel := &ytcommon.YtDestination{ - Path: "//home/cdc/tests/e2e/pg2yt/yt_static", - Cluster: os.Getenv("YT_PROXY"), - CellBundle: "default", - PrimaryMedium: "default", - Static: false, - DisableDatetimeHack: true, - UseStaticTableOnSnapshot: false, // this test is not supposed to work for static table - } - dst := &ytcommon.YtDestinationWrapper{Model: dstModel} - dst.WithDefaults() - - t.Run("activate", func(t *testing.T) { - transfer := helpers.MakeTransfer("fake", src, dst, abstract.TransferTypeSnapshotOnly) - require.NoError(t, tasks.ActivateDelivery(context.Background(), nil, coordinator.NewFakeClient(), *transfer, solomon.NewRegistry(solomon.NewRegistryOpts()))) - require.NoError(t, helpers.CompareStorages(t, src, dst.LegacyModel(), helpers.NewCompareStorageParams().WithEqualDataTypes(func(lDataType, rDataType string) bool { - return true - }))) - }) -} diff --git a/tests/e2e/ch2yt/static_table/dump/src.sql b/tests/e2e/ch2yt/static_table/dump/src.sql deleted file mode 100644 index 49b49a40..00000000 --- a/tests/e2e/ch2yt/static_table/dump/src.sql +++ /dev/null @@ -1,36 +0,0 @@ -CREATE DATABASE IF NOT EXISTS mtmobproxy; - - --- MergeTree->MergeTree - - -CREATE TABLE mtmobproxy.logs_weekly__mt_mt -( - `ServerName` String, - `DC` FixedString(3), - `RequestDate` Date, - `RequestDateTime` DateTime, - `VirtualHost` String, - `Path` String, - `BasePath` String DEFAULT 'misc', - `Code` UInt16, - `RequestLengthBytes` UInt32, - `FullRequestTime` UInt16, - `UpstreamResponseTime` UInt16, - `IsUpstreamRequest` Enum8('false' = 0, 'true' = 1), - `SSLHanshakeTime` UInt16, - `IsKeepalive` Enum8('false' = 0, 'true' = 1), - `StringHash` UInt32, - `HTTPMethod` String -) - ENGINE = MergeTree() - PARTITION BY toMonday(RequestDate) - ORDER BY (BasePath, Code, ServerName, StringHash) - SAMPLE BY StringHash - SETTINGS index_granularity = 8192; - -INSERT INTO mtmobproxy.logs_weekly__mt_mt -(`ServerName`, `DC`, `RequestDate`, `RequestDateTime`, `VirtualHost`, `Path`, `BasePath`, `Code`, `RequestLengthBytes`, `FullRequestTime`, `UpstreamResponseTime`, `IsUpstreamRequest`, `SSLHanshakeTime`, `IsKeepalive`, `StringHash`, `HTTPMethod`) -VALUES - ('my-server', 'iva', 1546300800, 1546300800, 'my-virtual-host', 'a/b', 'a', 1, 2, 3, 4, 'true', 5, 'false', 6, 'HTTPMethod') -; diff --git a/tests/e2e/kafka2ch/blank_parser/ch_init.sql b/tests/e2e/kafka2ch/blank_parser/ch_init.sql deleted file mode 100644 index d34f4426..00000000 --- a/tests/e2e/kafka2ch/blank_parser/ch_init.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE mtmobproxy; diff --git a/tests/e2e/kafka2ch/blank_parser/check_db_test.go b/tests/e2e/kafka2ch/blank_parser/check_db_test.go deleted file mode 100644 index 63940912..00000000 --- a/tests/e2e/kafka2ch/blank_parser/check_db_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package blankparser - -import ( - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/parsers" - "github.com/doublecloud/tross/pkg/parsers/registry/blank" - "github.com/doublecloud/tross/pkg/parsers/registry/json" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/transformer" - "github.com/doublecloud/tross/pkg/transformer/registry/jsonparser" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -func TestLogs(t *testing.T) { - src, err := kafka.SourceRecipe() - require.NoError(t, err) - src.Topic = "logs" - require.NoError(t, kafka.CreateSourceTopicIfNotExist(src, src.Topic, logger.Log)) - dst, err := chrecipe.Target(chrecipe.WithInitFile("ch_init.sql"), chrecipe.WithDatabase("mtmobproxy")) - require.NoError(t, err) - - src.Topic = "logs" - parserConfigMap, err := parsers.ParserConfigStructToMap(new(blank.ParserConfigBlankLb)) - require.NoError(t, err) - src.ParserConfig = parserConfigMap - require.NoError(t, err) - transfer := &server.Transfer{ - ID: "e2e_test", - Src: src, - Dst: dst, - } - transfer.Transformation = &server.Transformation{ - Transformers: &transformer.Transformers{Transformers: []transformer.Transformer{{ - jsonparser.TransformerType: &jsonparser.Config{ - Parser: &json.ParserConfigJSONCommon{ - Fields: []abstract.ColSchema{ - {ColumnName: "msg", DataType: ytschema.TypeString.String()}, - }, - AddRest: false, - AddDedupeKeys: true, - }, - Topic: "logs", - }, - }}}, - } - - lgr, closer, err := logger.NewKafkaLogger(&logger.KafkaConfig{ - Broker: src.Connection.Brokers[0], - Topic: src.Topic, - User: src.Auth.User, - Password: src.Auth.Password, - }) - require.NoError(t, err) - - defer closer.Close() - // SEND TO KAFKA - go func() { - for i := 0; i < 50; i++ { - lgr.Infof("line:%v", i) - } - }() - w := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, solomon.NewRegistry(solomon.NewRegistryOpts()), logger.Log) - w.Start() - require.NoError(t, helpers.WaitDestinationEqualRowsCount(dst.Database, src.Topic, helpers.GetSampleableStorageByModel(t, dst), 60*time.Second, 50)) - require.NoError(t, w.Stop()) -} diff --git a/tests/e2e/kafka2ch/replication/canondata/replication.replication.TestReplication/extracted b/tests/e2e/kafka2ch/replication/canondata/replication.replication.TestReplication/extracted deleted file mode 100644 index 05374ed7..00000000 --- a/tests/e2e/kafka2ch/replication/canondata/replication.replication.TestReplication/extracted +++ /dev/null @@ -1,55 +0,0 @@ - -"public"."topic1" -{ - "meta": - [ - { - "name": "id", - "type": "Nullable(Int32)" - }, - { - "name": "level", - "type": "Nullable(String)" - }, - { - "name": "caller", - "type": "Nullable(String)" - }, - { - "name": "msg", - "type": "Nullable(String)" - }, - { - "name": "_timestamp", - "type": "DateTime64(6)" - }, - { - "name": "_partition", - "type": "String" - }, - { - "name": "_offset", - "type": "UInt64" - }, - { - "name": "_idx", - "type": "UInt32" - } - ], - - "data": - [ - { - "id": 1, - "level": "my_level", - "caller": "my_caller", - "msg": "my_msg", - "_timestamp": "2024-03-19 00:00:00.000000", - "_partition": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"topic1\"}", - "_offset": "0", - "_idx": 1 - } - ], - - "rows": 1 -} diff --git a/tests/e2e/kafka2ch/replication/canondata/result.json b/tests/e2e/kafka2ch/replication/canondata/result.json deleted file mode 100644 index 85b6d668..00000000 --- a/tests/e2e/kafka2ch/replication/canondata/result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "replication.replication.TestReplication": { - "uri": "file://replication.replication.TestReplication/extracted" - } -} diff --git a/tests/e2e/kafka2ch/replication/check_db_test.go b/tests/e2e/kafka2ch/replication/check_db_test.go deleted file mode 100644 index 160f6bc3..00000000 --- a/tests/e2e/kafka2ch/replication/check_db_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/parsers" - jsonparser "github.com/doublecloud/tross/pkg/parsers/registry/json" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/tests/canon/reference" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - kafkaTopic = "topic1" - source = *kafkasink.MustSourceRecipe() - - chDatabase = "public" - target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) - targetAsSource = *chrecipe.MustSource(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) - - timestampToUse = time.Date(2024, 03, 19, 0, 0, 0, 0, time.Local) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func includeAllTables(table abstract.TableID, schema abstract.TableColumns) bool { - return true -} - -func fixTimestampMiddleware(t *testing.T, items []abstract.ChangeItem) abstract.TransformerResult { - for _, item := range items { - for i, name := range item.ColumnNames { - if name == "_timestamp" { - // Fix timestamp to support canonization - item.ColumnValues[i] = timestampToUse - break - } - } - } - - return abstract.TransformerResult{ - Transformed: items, - } -} - -func TestReplication(t *testing.T) { - // prepare source - - parserConfigStruct := &jsonparser.ParserConfigJSONCommon{ - Fields: []abstract.ColSchema{ - {ColumnName: "id", DataType: ytschema.TypeInt32.String(), PrimaryKey: true}, - {ColumnName: "level", DataType: ytschema.TypeString.String()}, - {ColumnName: "caller", DataType: ytschema.TypeString.String()}, - {ColumnName: "msg", DataType: ytschema.TypeString.String()}, - }, - AddRest: false, - AddDedupeKeys: true, - } - parserConfigMap, err := parsers.ParserConfigStructToMap(parserConfigStruct) - require.NoError(t, err) - - source.ParserConfig = parserConfigMap - source.Topic = kafkaTopic - - // write to source topic - - k := []byte(`any_key`) - v := []byte(`{"id": "1", "level": "my_level", "caller": "my_caller", "msg": "my_msg"}`) - - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: source.Connection, - Auth: source.Auth, - Topic: source.Topic, - FormatSettings: server.SerializationFormat{ - Name: server.SerializationFormatJSON, - BatchingSettings: &server.Batching{ - Enabled: false, - Interval: 0, - MaxChangeItems: 0, - MaxMessageSize: 0, - }, - }, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{kafkasink.MakeKafkaRawMessage(source.Topic, time.Time{}, source.Topic, 0, 0, k, v)}) - require.NoError(t, err) - - // activate transfer - - transfer := helpers.MakeTransfer(helpers.TransferID, &source, &target, abstract.TransferTypeIncrementOnly) - // add transformation in order to control Kafka timestamp - err = transfer.AddExtraTransformer(helpers.NewSimpleTransformer(t, fixTimestampMiddleware, includeAllTables)) - require.NoError(t, err) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - // check results - - require.NoError(t, helpers.WaitDestinationEqualRowsCount( - target.Database, - kafkaTopic, - helpers.GetSampleableStorageByModel(t, target), - 60*time.Second, - 1, - )) - reference.Dump(t, &targetAsSource) -} diff --git a/tests/e2e/kafka2ch/replication/dump/ch/dump.sql b/tests/e2e/kafka2ch/replication/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/kafka2ch/replication/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/kafka2ch/replication_mv/canondata/replication.replication.TestReplication/extracted b/tests/e2e/kafka2ch/replication_mv/canondata/replication.replication.TestReplication/extracted deleted file mode 100644 index 05374ed7..00000000 --- a/tests/e2e/kafka2ch/replication_mv/canondata/replication.replication.TestReplication/extracted +++ /dev/null @@ -1,55 +0,0 @@ - -"public"."topic1" -{ - "meta": - [ - { - "name": "id", - "type": "Nullable(Int32)" - }, - { - "name": "level", - "type": "Nullable(String)" - }, - { - "name": "caller", - "type": "Nullable(String)" - }, - { - "name": "msg", - "type": "Nullable(String)" - }, - { - "name": "_timestamp", - "type": "DateTime64(6)" - }, - { - "name": "_partition", - "type": "String" - }, - { - "name": "_offset", - "type": "UInt64" - }, - { - "name": "_idx", - "type": "UInt32" - } - ], - - "data": - [ - { - "id": 1, - "level": "my_level", - "caller": "my_caller", - "msg": "my_msg", - "_timestamp": "2024-03-19 00:00:00.000000", - "_partition": "{\"cluster\":\"\",\"partition\":0,\"topic\":\"topic1\"}", - "_offset": "0", - "_idx": 1 - } - ], - - "rows": 1 -} diff --git a/tests/e2e/kafka2ch/replication_mv/canondata/result.json b/tests/e2e/kafka2ch/replication_mv/canondata/result.json deleted file mode 100644 index 85b6d668..00000000 --- a/tests/e2e/kafka2ch/replication_mv/canondata/result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "replication.replication.TestReplication": { - "uri": "file://replication.replication.TestReplication/extracted" - } -} diff --git a/tests/e2e/kafka2ch/replication_mv/check_db_test.go b/tests/e2e/kafka2ch/replication_mv/check_db_test.go deleted file mode 100644 index 6d7d9a1f..00000000 --- a/tests/e2e/kafka2ch/replication_mv/check_db_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/parsers" - jsonparser "github.com/doublecloud/tross/pkg/parsers/registry/json" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - kafkaTopic = "topic1" - source = *kafkasink.MustSourceRecipe() - - chDatabase = "public" - target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) - targetAsSource = *chrecipe.MustSource(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) - - timestampToUse = time.Date(2024, 03, 19, 0, 0, 0, 0, time.Local) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func includeAllTables(table abstract.TableID, schema abstract.TableColumns) bool { - return true -} - -func fixTimestampMiddleware(t *testing.T, items []abstract.ChangeItem) abstract.TransformerResult { - for _, item := range items { - for i, name := range item.ColumnNames { - if name == "_timestamp" { - // Fix timestamp to support canonization - item.ColumnValues[i] = timestampToUse - break - } - } - } - - return abstract.TransformerResult{ - Transformed: items, - } -} - -func TestReplication(t *testing.T) { - // prepare source - - target.Cleanup = server.DisabledCleanup - target.InsertParams = model.InsertParams{MaterializedViewsIgnoreErrors: true} - - parserConfigStruct := &jsonparser.ParserConfigJSONCommon{ - Fields: []abstract.ColSchema{ - {ColumnName: "id", DataType: ytschema.TypeInt32.String(), PrimaryKey: true}, - {ColumnName: "level", DataType: ytschema.TypeString.String()}, - {ColumnName: "caller", DataType: ytschema.TypeString.String()}, - {ColumnName: "msg", DataType: ytschema.TypeString.String()}, - }, - AddRest: false, - NullKeysAllowed: true, // ID can be null, but mat-view expect it not nullable - AddDedupeKeys: true, - } - parserConfigMap, err := parsers.ParserConfigStructToMap(parserConfigStruct) - require.NoError(t, err) - - source.ParserConfig = parserConfigMap - source.Topic = kafkaTopic - - // write to source topic - - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: source.Connection, - Auth: source.Auth, - Topic: source.Topic, - FormatSettings: server.SerializationFormat{ - Name: server.SerializationFormatJSON, - BatchingSettings: &server.Batching{ - Enabled: false, - Interval: 0, - MaxChangeItems: 0, - MaxMessageSize: 0, - }, - }, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{ - kafkasink.MakeKafkaRawMessage( - source.Topic, - time.Time{}, - source.Topic, - 0, - 1, - []byte(`any_key_2`), - []byte(`{"level": "my_level", "caller": "my_caller", "msg": "my_msg"}`), // no ID column, should fail matview. - ), - }) - require.NoError(t, err) - - // activate transfer - - transfer := helpers.MakeTransfer(helpers.TransferID, &source, &target, abstract.TransferTypeIncrementOnly) - // add transformation in order to control Kafka timestamp - err = transfer.AddExtraTransformer(helpers.NewSimpleTransformer(t, fixTimestampMiddleware, includeAllTables)) - require.NoError(t, err) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - // check results - - require.NoError(t, helpers.WaitDestinationEqualRowsCount( - target.Database, - kafkaTopic, - helpers.GetSampleableStorageByModel(t, target), - 60*time.Second, - 1, - )) -} diff --git a/tests/e2e/kafka2ch/replication_mv/dump/ch/dump.sql b/tests/e2e/kafka2ch/replication_mv/dump/ch/dump.sql deleted file mode 100644 index b80609b5..00000000 --- a/tests/e2e/kafka2ch/replication_mv/dump/ch/dump.sql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE DATABASE public; - -CREATE TABLE IF NOT EXISTS public.`topic1` -( - `id` Nullable(Int32), - `level` Nullable(String), - `caller` Nullable(String), - `msg` Nullable(String), - `_timestamp` DateTime64(6), - `_partition` String, - `_offset` UInt64, - `_idx` UInt32 -) -ENGINE=MergeTree() -ORDER BY (`id`, `_timestamp`, `_partition`, `_offset`, `_idx`) -SETTINGS allow_nullable_key = 1; - - -CREATE TABLE public.__test_aggr -( - `is_even` Int8, - `sum_id` UInt64 -) - ENGINE = SummingMergeTree() -ORDER BY (is_even); - -CREATE MATERIALIZED VIEW public.__test_mv -TO public.__test_aggr -AS -SELECT - coalesce(id / 2, 0) is_even, - sum(toInt32(_partition)) AS sumVal -- at replication we will try to insert null, it should fail sum -FROM public.topic1 -GROUP BY - is_even; diff --git a/tests/e2e/kafka2kafka/metering/canondata/result.json b/tests/e2e/kafka2kafka/metering/canondata/result.json deleted file mode 100644 index 6432a347..00000000 --- a/tests/e2e/kafka2kafka/metering/canondata/result.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "metering.metering.TestReplicationMetering": [ - { - "cloud_id": "", - "folder_id": "", - "resource_id": "dtt", - "schema": "datatransfer.data.output.v1", - "source_id": "", - "tags": { - "dst_type": "kafka", - "row_size": "small", - "runtime": "local", - "src_type": "kafka", - "transfer_type": "INCREMENT_ONLY" - }, - "usage": { - "quantity": 1, - "type": "delta", - "unit": "rows" - }, - "version": "v1alpha1" - }, - { - "cloud_id": "", - "folder_id": "", - "resource_id": "dtt", - "schema": "datatransfer.data.output.v1", - "source_id": "", - "tags": { - "dst_type": "kafka", - "row_size": "large", - "runtime": "local", - "src_type": "kafka", - "transfer_type": "INCREMENT_ONLY" - }, - "usage": { - "quantity": 0, - "type": "delta", - "unit": "rows" - }, - "version": "v1alpha1" - } - ] -} diff --git a/tests/e2e/kafka2kafka/metering/metering_test.go b/tests/e2e/kafka2kafka/metering/metering_test.go deleted file mode 100644 index cd33d2b5..00000000 --- a/tests/e2e/kafka2kafka/metering/metering_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package metering - -import ( - "context" - "encoding/json" - "errors" - "os" - "strings" - "syscall" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/config" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/metering" - kafka_writer "github.com/doublecloud/tross/pkg/metering/writer/kafka" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/segmentio/kafka-go" - "github.com/segmentio/kafka-go/sasl" - "github.com/stretchr/testify/require" -) - -var ( - srcTopic = "topic1" - dstTopic = "topic2" - meteringTopic = "topic3" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestReplicationMetering(t *testing.T) { - src, err := kafkasink.SourceRecipe() - require.NoError(t, err) - src.Topic = srcTopic - src.IsHomo = true - src.WithDefaults() - - dst, err := kafkasink.DestinationRecipe() - require.NoError(t, err) - dst.Topic = dstTopic - dst.FormatSettings = server.SerializationFormat{ - Name: server.SerializationFormatMirror, - } - dst.WithDefaults() - - dpconfig.Common = &dpconfig.CommonConfig{ - Metering: &config.MeteringEnabled{ - Writers: []config.WriterEntry{ - { - Writer: &kafka_writer.Kafka{ - Brokers: src.Connection.Brokers, - User: src.Auth.User, - Password: config.Secret(src.Auth.Password), - TLSMode: &config.TLSModeDisabled{}, - AuthMechanism: src.Auth.Mechanism, - }, - MetricTopics: map[string]string{"datatransfer.data.output.v1": meteringTopic}, - }, - }, - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - require.NoError(t, metering.Agent().RunPusher(ctx, time.Minute)) - defer metering.Agent().Stop() - - // write to source topic - - k := []byte(`my_key`) - v := []byte(`test`) - - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: src.Connection, - Auth: src.Auth, - Topic: src.Topic, - FormatSettings: dst.FormatSettings, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{kafkasink.MakeKafkaRawMessage(srcTopic, time.Time{}, srcTopic, 0, 0, k, v)}) - require.NoError(t, err) - - // activate main transfer - helpers.InitSrcDst(helpers.TransferID, src, dst, abstract.TransferTypeIncrementOnly) - transfer := helpers.MakeTransfer(helpers.TransferID, src, dst, abstract.TransferTypeIncrementOnly) - transfer.Runtime = &abstract.LocalRuntime{} - metering.Initialize(transfer, nil) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, solomon.NewRegistry(solomon.NewRegistryOpts()), logger.Log) - localWorker.Start() - defer localWorker.Stop() - - authMech, err := src.Auth.GetAuthMechanism() - require.NoError(t, err) - - // wait for metering message - meteringMessage, err := readMessages(authMech, src.Connection.Brokers, meteringTopic) - - require.NoError(t, err) - require.NotNil(t, meteringMessage) - - values, err := removeDynamicFields(meteringMessage.Value) - require.NoError(t, err) - - canon.SaveJSON(t, values) -} - -func readMessages(auth sasl.Mechanism, brokers []string, topic string) (*kafka.Message, error) { - dialer := &kafka.Dialer{ - SASLMechanism: auth, - } - - r := kafka.NewReader(kafka.ReaderConfig{ - Brokers: brokers, - Topic: topic, - Dialer: dialer, - }) - - defer r.Close() - - for { - m, err := r.ReadMessage(context.Background()) - if err != nil { - if errors.Is(err, syscall.ECONNRESET) { - time.Sleep(time.Second * 5) - continue - } - return nil, err - } - if m.Key != nil { - return &m, nil - } - } -} - -func removeDynamicFields(data []byte) ([]ReceivedKafkaMetering, error) { - var results []ReceivedKafkaMetering - - jsonLines := strings.Split(string(data), "\n") - for _, line := range jsonLines { - var metering ReceivedKafkaMetering - err := json.Unmarshal([]byte(line), &metering) - if err != nil { - return nil, err - } - results = append(results, metering) - } - - return results, nil -} - -type ReceivedKafkaMetering struct { - CloudID string `json:"cloud_id"` - FolderID string `json:"folder_id"` - ResourceID string `json:"resource_id"` - Schema string `json:"schema"` - SourceID string `json:"source_id"` - Tags Tags `json:"tags"` - Usage Usage `json:"usage"` - Version string `json:"version"` -} - -type Tags struct { - DstType string `json:"dst_type"` - RowSize string `json:"row_size"` - Runtime string `json:"runtime"` - SrcType string `json:"src_type"` - TransferType string `json:"transfer_type"` -} -type Usage struct { - Quantity int `json:"quantity"` - Type string `json:"type"` - Unit string `json:"unit"` -} diff --git a/tests/e2e/kafka2kafka/mirror/mirror_test.go b/tests/e2e/kafka2kafka/mirror/mirror_test.go deleted file mode 100644 index b7211ac3..00000000 --- a/tests/e2e/kafka2kafka/mirror/mirror_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/util" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestReplication(t *testing.T) { - srcTopic := "topic1" - dstTopic := "topic2" - - src, err := kafkasink.SourceRecipe() - require.NoError(t, err) - src.Topic = srcTopic - src.IsHomo = true - - dst, err := kafkasink.DestinationRecipe() - require.NoError(t, err) - dst.Topic = dstTopic - dst.FormatSettings = server.SerializationFormat{Name: server.SerializationFormatMirror} - - // write to source topic - - k := []byte(`my_key`) - v := []byte(`blablabla`) - - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: src.Connection, - Auth: src.Auth, - Topic: src.Topic, - FormatSettings: dst.FormatSettings, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{kafkasink.MakeKafkaRawMessage(srcTopic, time.Time{}, srcTopic, 0, 0, k, v)}) - require.NoError(t, err) - - // prepare additional transfer: from dst to mock - - result := make([]abstract.ChangeItem, 0) - mockSink := &helpers.MockSink{ - PushCallback: func(in []abstract.ChangeItem) { - abstract.Dump(in) - result = append(result, in...) - }, - } - mockTarget := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return mockSink }, - Cleanup: server.DisabledCleanup, - } - additionalTransfer := helpers.MakeTransfer("additional", &kafkasink.KafkaSource{ - Connection: dst.Connection, - Auth: dst.Auth, - GroupTopics: []string{dst.Topic}, - IsHomo: true, - }, &mockTarget, abstract.TransferTypeIncrementOnly) - - // activate main transfer - - helpers.InitSrcDst(helpers.TransferID, src, dst, abstract.TransferTypeIncrementOnly) - transfer := helpers.MakeTransfer(helpers.TransferID, src, dst, abstract.TransferTypeIncrementOnly) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, solomon.NewRegistry(solomon.NewRegistryOpts()), log.With(logger.Log, log.Any("transfer", "main"))) - localWorker.Start() - defer localWorker.Stop() - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - go func() { - for { - // restart transfer if error - errCh := make(chan error, 1) - w, err := helpers.ActivateErr(additionalTransfer, func(err error) { - errCh <- err - }) - require.NoError(t, err) - _, ok := util.Receive(ctx, errCh) - if !ok { - return - } - w.Close(t) - } - }() - - st := time.Now() - for time.Since(st) < time.Second*30 { - if len(result) == 1 { - require.Equal(t, k, kafkasink.GetKafkaRawMessageKey(&result[0])) - require.Equal(t, v, kafkasink.GetKafkaRawMessageData(&result[0])) - break - } - - time.Sleep(time.Second) - } -} diff --git a/tests/e2e/kafka2kafka/multi_topic/canondata/result.json b/tests/e2e/kafka2kafka/multi_topic/canondata/result.json deleted file mode 100644 index 2dc1c088..00000000 --- a/tests/e2e/kafka2kafka/multi_topic/canondata/result.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "multi_topic.multi_topic.TestReplication": { - "\"topic1\"": { - "data": "blablabla", - "key": "my_key" - }, - "\"topic2\"": { - "data": "blablabla", - "key": "my_key" - } - } -} diff --git a/tests/e2e/kafka2kafka/multi_topic/mirror_test.go b/tests/e2e/kafka2kafka/multi_topic/mirror_test.go deleted file mode 100644 index 6e6eeb96..00000000 --- a/tests/e2e/kafka2kafka/multi_topic/mirror_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestReplication(t *testing.T) { - src, err := kafkasink.SourceRecipe() - require.NoError(t, err) - src.IsHomo = true - - dst, err := kafkasink.DestinationRecipe() - require.NoError(t, err) - dst.FormatSettings = server.SerializationFormat{Name: server.SerializationFormatMirror} - - // write to source topic - k := []byte(`my_key`) - v := []byte(`blablabla`) - - pushData(t, *src, "topic1", *dst, k, v) - pushData(t, *src, "topic2", *dst, k, v) - - // prepare additional transfer: from dst to mock - - result := make([]abstract.ChangeItem, 0) - mockSink := &helpers.MockSink{ - PushCallback: func(in []abstract.ChangeItem) { - result = append(result, in...) - }, - } - mockTarget := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return mockSink }, - Cleanup: server.DisabledCleanup, - } - additionalTransfer := helpers.MakeTransfer("additional", &kafkasink.KafkaSource{ - Connection: dst.Connection, - Auth: dst.Auth, - GroupTopics: []string{"topic1", "topic2"}, - IsHomo: true, - }, &mockTarget, abstract.TransferTypeIncrementOnly) - - localAdditionalWorker := local.NewLocalWorker(coordinator.NewFakeClient(), additionalTransfer, solomon.NewRegistry(solomon.NewRegistryOpts()), logger.Log) - localAdditionalWorker.Start() - defer localAdditionalWorker.Stop() - - //----------------------------------------------------------------------------------------------------------------- - st := time.Now() - for time.Since(st) < time.Minute { - if len(result) < 2 { - time.Sleep(time.Second) - continue - } - break - } - readedData := map[string]map[string]string{} - for _, ci := range result { - readedData[ci.TableID().String()] = map[string]string{ - "key": string(kafkasink.GetKafkaRawMessageKey(&ci)), - "data": string(kafkasink.GetKafkaRawMessageData(&ci)), - } - } - require.Len(t, result, 2) - canon.SaveJSON(t, readedData) -} - -func pushData(t *testing.T, src kafkasink.KafkaSource, srcTopic string, dst kafkasink.KafkaDestination, k []byte, v []byte) { - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: src.Connection, - Auth: src.Auth, - Topic: srcTopic, - FormatSettings: dst.FormatSettings, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{kafkasink.MakeKafkaRawMessage(srcTopic, time.Time{}, srcTopic, 0, 0, k, v)}) - require.NoError(t, err) - require.NoError(t, srcSink.Close()) -} diff --git a/tests/e2e/kafka2mongo/replication/check_db_test.go b/tests/e2e/kafka2mongo/replication/check_db_test.go deleted file mode 100644 index 3a3616d7..00000000 --- a/tests/e2e/kafka2mongo/replication/check_db_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/parsers" - jsonparser "github.com/doublecloud/tross/pkg/parsers/registry/json" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/providers/logbroker" - mongodataagent "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - source = kafkasink.KafkaSource{ - Connection: &kafkasink.KafkaConnectionOptions{ - TLS: logbroker.DisabledTLS, - Brokers: []string{os.Getenv("KAFKA_RECIPE_BROKER_LIST")}, - }, - Auth: &kafkasink.KafkaAuth{Enabled: false}, - Topic: "topic1", - Transformer: nil, - BufferSize: server.BytesSize(1024), - SecurityGroupIDs: nil, - ParserConfig: nil, - IsHomo: false, - } - target = mongodataagent.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - Database: "db1", - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestReplication(t *testing.T) { - // prepare source - - parserConfigStruct := &jsonparser.ParserConfigJSONCommon{ - Fields: []abstract.ColSchema{ - {ColumnName: "id", DataType: ytschema.TypeInt32.String(), PrimaryKey: true}, - {ColumnName: "level", DataType: ytschema.TypeString.String()}, - {ColumnName: "caller", DataType: ytschema.TypeString.String()}, - {ColumnName: "msg", DataType: ytschema.TypeString.String()}, - }, - AddRest: false, - AddDedupeKeys: true, - } - parserConfigMap, err := parsers.ParserConfigStructToMap(parserConfigStruct) - require.NoError(t, err) - - source.ParserConfig = parserConfigMap - - // write to source topic - - k := []byte(`any_key`) - v := []byte(`{"id": "1", "level": "my_level", "caller": "my_caller", "msg": "my_msg"}`) - - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: source.Connection, - Auth: source.Auth, - Topic: source.Topic, - FormatSettings: server.SerializationFormat{ - Name: server.SerializationFormatJSON, - BatchingSettings: &server.Batching{ - Enabled: false, - Interval: 0, - MaxChangeItems: 0, - MaxMessageSize: 0, - }, - }, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{kafkasink.MakeKafkaRawMessage(source.Topic, time.Time{}, source.Topic, 0, 0, k, v)}) - require.NoError(t, err) - - // activate transfer - - transfer := helpers.MakeTransfer(helpers.TransferID, &source, &target, abstract.TransferTypeIncrementOnly) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - // check results - - require.NoError(t, helpers.WaitDestinationEqualRowsCount( - target.Database, - "topic1", - helpers.GetSampleableStorageByModel(t, target), - 60*time.Second, - 1, - )) -} diff --git a/tests/e2e/kafka2mongo/replication/dump/date_time.sql b/tests/e2e/kafka2mongo/replication/dump/date_time.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/e2e/kafka2mysql/filter_rows/check_db_test.go b/tests/e2e/kafka2mysql/filter_rows/check_db_test.go deleted file mode 100644 index c77700b1..00000000 --- a/tests/e2e/kafka2mysql/filter_rows/check_db_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/parsers" - jsonparser "github.com/doublecloud/tross/pkg/parsers/registry/json" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/providers/logbroker" - filterrows "github.com/doublecloud/tross/pkg/transformer/registry/filter_rows" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - topicName = "testTopic" - - source = kafkasink.KafkaSource{ - Connection: &kafkasink.KafkaConnectionOptions{ - TLS: logbroker.DisabledTLS, - Brokers: []string{os.Getenv("KAFKA_RECIPE_BROKER_LIST")}, - }, - Auth: &kafkasink.KafkaAuth{Enabled: false}, - Topic: topicName, - Transformer: nil, - BufferSize: server.BytesSize(1024), - SecurityGroupIDs: nil, - ParserConfig: nil, - IsHomo: false, - } - target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestReplication(t *testing.T) { - - // prepare source - parserConfigStruct := &jsonparser.ParserConfigJSONCommon{ - Fields: []abstract.ColSchema{ - {ColumnName: "id", DataType: ytschema.TypeInt32.String(), PrimaryKey: true}, - {ColumnName: "i64", DataType: ytschema.TypeInt64.String()}, - {ColumnName: "f32", DataType: ytschema.TypeFloat32.String()}, - {ColumnName: "str", DataType: ytschema.TypeString.String()}, - {ColumnName: "date", DataType: ytschema.TypeDate.String()}, - {ColumnName: "datetime", DataType: ytschema.TypeDatetime.String()}, - {ColumnName: "time", DataType: ytschema.TypeTimestamp.String()}, - {ColumnName: "null", DataType: ytschema.TypeString.String()}, - {ColumnName: "notNull", DataType: ytschema.TypeString.String()}, - }, - AddRest: false, - AddDedupeKeys: false, - } - parserConfigMap, err := parsers.ParserConfigStructToMap(parserConfigStruct) - require.NoError(t, err) - - source.ParserConfig = parserConfigMap - - // activate transfer - filter := strings.Join([]string{ - "id > 1", - "i64 < 9223372036854775807", "i64 > -9223372036854775808", - "f32 <= 0.3", - `str ~ "name"`, `str !~ "bad"`, - "date > 1999-01-04", "date <= 2000-03-04", - "datetime = 2010-01-01T00:00:00", - "time = 2010-01-01T00:00:00", - "null = NULL", - "notNull != NULL", - }, " AND ") - - transfer := helpers.MakeTransfer(helpers.TransferID, &source, &target, abstract.TransferTypeIncrementOnly) - transformer, err := filterrows.NewFilterRowsTransformer( - filterrows.Config{Filter: filter}, - logger.Log, - ) - require.NoError(t, err) - require.NoError(t, transfer.AddExtraTransformer(transformer)) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - // write to source topic - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: source.Connection, - Auth: source.Auth, - Topic: source.Topic, - FormatSettings: server.SerializationFormat{ - Name: server.SerializationFormatJSON, - BatchingSettings: &server.Batching{ - Enabled: false, - Interval: 0, - MaxChangeItems: 0, - MaxMessageSize: 0, - }, - }, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - - v1 := []byte(`{"id": "1", "i64": "9223372036854775807", "f32": "0.1", "str": "badname", "time": "2000-01-01T00:00:00", "datetime": "2000-01-01 00:00:00 +0000 UTC", "date": "1999-01-04", "null": null, "notNull": null}`) - v2 := []byte(`{"id": "2", "i64": "200", "f32": "0.2", "str": "name", "time": "2010-01-01T00:00:00", "datetime": "2010-01-01T00:00:00 +0000 UTC", "date": "2000-03-04", "null": null, "notNull": "str"}`) - v3 := []byte(`{"id": "3", "i64": "-9223372036854775808", "f32": "0.3", "str": "other", "time": "2005-01-01T00:00:00", "datetime": "2005-01-01T00:00:00 +0000 UTC", "date": "2000-03-05", "null": "str", "notNull": "str"}`) - - require.NoError(t, srcSink.Push([]abstract.ChangeItem{ - kafkasink.MakeKafkaRawMessage(source.Topic, time.Time{}, source.Topic, 0, 0, []byte("_"), v1), - kafkasink.MakeKafkaRawMessage(source.Topic, time.Time{}, source.Topic, 0, 0, []byte("_"), v2), - kafkasink.MakeKafkaRawMessage(source.Topic, time.Time{}, source.Topic, 0, 0, []byte("_"), v3), - })) - - // check results - expected := []abstract.ChangeItem{{ - ColumnNames: []string{ - "id", - "date", - "datetime", - "f32", - "i64", - "notNull", - "null", - "str", - "time", - }, - ColumnValues: []interface{}{ - int32(2), - time.Date(2000, time.March, 4, 0, 0, 0, 0, time.UTC), - time.Date(2010, time.January, 1, 0, 0, 0, 0, time.Local), - json.Number("0.2"), - int64(200), - "str", - nil, - "name", - time.Date(2010, time.January, 1, 0, 0, 0, 0, time.Local), - }, - }} - - dst := helpers.GetSampleableStorageByModel(t, target) - err = helpers.WaitDestinationEqualRowsCount(target.Database, topicName, dst, 300*time.Second, uint64(len(expected))) - require.NoError(t, err) - - var actual []abstract.ChangeItem - - dst = helpers.GetSampleableStorageByModel(t, target) - require.NoError(t, dst.LoadTable(context.Background(), abstract.TableDescription{ - Schema: target.Database, - Name: topicName, - }, func(input []abstract.ChangeItem) error { - for _, row := range input { - if row.Kind != abstract.InsertKind { - continue - } - item := abstract.ChangeItem{ - ColumnNames: row.ColumnNames, - ColumnValues: row.ColumnValues, - } - actual = append(actual, helpers.RemoveColumnsFromChangeItem( - item, []string{"_idx", "_offset", "_partition", "_timestamp"})) - } - return nil - })) - - require.Equal(t, expected, actual) -} diff --git a/tests/e2e/kafka2mysql/filter_rows/dump/date_time.sql b/tests/e2e/kafka2mysql/filter_rows/dump/date_time.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/e2e/kafka2mysql/replication/check_db_test.go b/tests/e2e/kafka2mysql/replication/check_db_test.go deleted file mode 100644 index f7f97ca7..00000000 --- a/tests/e2e/kafka2mysql/replication/check_db_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/parsers" - jsonparser "github.com/doublecloud/tross/pkg/parsers/registry/json" - kafkasink "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/providers/logbroker" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - source = kafkasink.KafkaSource{ - Connection: &kafkasink.KafkaConnectionOptions{ - TLS: logbroker.DisabledTLS, - Brokers: []string{os.Getenv("KAFKA_RECIPE_BROKER_LIST")}, - }, - Auth: &kafkasink.KafkaAuth{Enabled: false}, - Topic: "topic1", - Transformer: nil, - BufferSize: server.BytesSize(1024), - SecurityGroupIDs: nil, - ParserConfig: nil, - IsHomo: false, - } - target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestReplication(t *testing.T) { - // prepare source - - parserConfigStruct := &jsonparser.ParserConfigJSONCommon{ - Fields: []abstract.ColSchema{ - {ColumnName: "id", DataType: ytschema.TypeInt32.String(), PrimaryKey: true}, - {ColumnName: "level", DataType: ytschema.TypeString.String()}, - {ColumnName: "caller", DataType: ytschema.TypeString.String()}, - {ColumnName: "msg", DataType: ytschema.TypeString.String()}, - }, - AddRest: false, - AddDedupeKeys: false, - } - parserConfigMap, err := parsers.ParserConfigStructToMap(parserConfigStruct) - require.NoError(t, err) - - source.ParserConfig = parserConfigMap - - // write to source topic - - k := []byte(`any_key`) - v := []byte(`{"id": "1", "level": "my_level", "caller": "my_caller", "msg": "my_msg"}`) - - srcSink, err := kafkasink.NewReplicationSink( - &kafkasink.KafkaDestination{ - Connection: source.Connection, - Auth: source.Auth, - Topic: source.Topic, - FormatSettings: server.SerializationFormat{ - Name: server.SerializationFormatJSON, - BatchingSettings: &server.Batching{ - Enabled: false, - Interval: 0, - MaxChangeItems: 0, - MaxMessageSize: 0, - }, - }, - ParralelWriterCount: 10, - }, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - ) - require.NoError(t, err) - err = srcSink.Push([]abstract.ChangeItem{kafkasink.MakeKafkaRawMessage(source.Topic, time.Time{}, source.Topic, 0, 0, k, v)}) - require.NoError(t, err) - - // activate transfer - - transfer := helpers.MakeTransfer(helpers.TransferID, &source, &target, abstract.TransferTypeIncrementOnly) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - // check results - - require.NoError(t, helpers.WaitDestinationEqualRowsCount( - target.Database, - "topic1", - helpers.GetSampleableStorageByModel(t, target), - 60*time.Second, - 1, - )) -} diff --git a/tests/e2e/kafka2mysql/replication/dump/date_time.sql b/tests/e2e/kafka2mysql/replication/dump/date_time.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/e2e/mongo2ch/snapshot/check_db_test.go b/tests/e2e/mongo2ch/snapshot/check_db_test.go deleted file mode 100644 index 351cfe39..00000000 --- a/tests/e2e/mongo2ch/snapshot/check_db_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package snapshot - -import ( - "context" - "encoding/json" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/canon/mongo" - "github.com/doublecloud/tross/tests/helpers" - "github.com/spf13/cast" - "github.com/stretchr/testify/require" -) - -const databaseName string = "db" - -var ( - Source = mongocommon.RecipeSource() - Target = chrecipe.MustTarget(chrecipe.WithInitFile("dump.sql"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -func MakeDstClient(t *mongocommon.MongoDestination) (*mongocommon.MongoClientWrapper, error) { - return mongocommon.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH HTTP target", Port: Target.HTTPPort}, - helpers.LabeledPort{Label: "CH Native target", Port: Target.NativePort}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Snapshot", Snapshot) - }) -} - -func Ping(t *testing.T) { - client, err := mongocommon.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - Source.Collections = []mongocommon.MongoCollection{ - {DatabaseName: databaseName, CollectionName: "test_data"}, - {DatabaseName: databaseName, CollectionName: "empty"}, - } - - require.NoError(t, mongo.InsertDocs(context.Background(), Source, databaseName, "test_data", mongo.SnapshotDocuments...)) - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotOnly) - transfer.TypeSystemVersion = 7 - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - snapshotLoader := tasks.NewSnapshotLoader(client2.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - err = helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(func(lDataType, rDataType string) bool { - return true - }).WithPriorityComparators(func(lVal interface{}, lSchema abstract.ColSchema, rVal interface{}, rSchema abstract.ColSchema, intoArray bool) (comparable bool, result bool, err error) { - ld, _ := json.Marshal(lVal) - return true, string(ld) == cast.ToString(rVal), nil - })) - require.NoError(t, err) -} diff --git a/tests/e2e/mongo2ch/snapshot/dump.sql b/tests/e2e/mongo2ch/snapshot/dump.sql deleted file mode 100644 index e042557e..00000000 --- a/tests/e2e/mongo2ch/snapshot/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE db; diff --git a/tests/e2e/mongo2ch/snapshot_flatten/canondata/result.json b/tests/e2e/mongo2ch/snapshot_flatten/canondata/result.json deleted file mode 100644 index 2d945300..00000000 --- a/tests/e2e/mongo2ch/snapshot_flatten/canondata/result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "snapshot_flatten.snapshot_flatten.TestGroup/Group_after_port_check/Snapshot": { - "uri": "file://snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted" - } -} diff --git a/tests/e2e/mongo2ch/snapshot_flatten/canondata/snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted b/tests/e2e/mongo2ch/snapshot_flatten/canondata/snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted deleted file mode 100644 index 6e5387dc..00000000 --- a/tests/e2e/mongo2ch/snapshot_flatten/canondata/snapshot_flatten.snapshot_flatten.TestGroup_Group_after_port_check_Snapshot/extracted +++ /dev/null @@ -1,40 +0,0 @@ - -"db"."test_data" -{ - "meta": - [ - { - "name": "_id", - "type": "String" - }, - { - "name": "floors_as_string_array", - "type": "Array(String)" - }, - { - "name": "value_from_floors", - "type": "Array(Float64)" - }, - { - "name": "currency_from_floors", - "type": "Array(String)" - }, - { - "name": "floors_as_string", - "type": "String" - } - ], - - "data": - [ - { - "_id": "D1AAD9AB", - "floors_as_string_array": ["{\"currency\":\"EUR\",\"value\":0.2,\"countryIds\":[\"IT\"]}","{\"currency\":\"EUR\",\"value\":0.3,\"countryIds\":[\"FR\",\"GB\"]}"], - "value_from_floors": [0.2,0.3], - "currency_from_floors": ["EUR","EUR"], - "floors_as_string": "[{\"currency\":\"EUR\",\"value\":0.2,\"countryIds\":[\"IT\"]},{\"currency\":\"EUR\",\"value\":0.3,\"countryIds\":[\"FR\",\"GB\"]}]" - } - ], - - "rows": 1 -} diff --git a/tests/e2e/mongo2ch/snapshot_flatten/check_db_test.go b/tests/e2e/mongo2ch/snapshot_flatten/check_db_test.go deleted file mode 100644 index c182180c..00000000 --- a/tests/e2e/mongo2ch/snapshot_flatten/check_db_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/transformer" - "github.com/doublecloud/tross/pkg/transformer/registry/clickhouse" - "github.com/doublecloud/tross/pkg/transformer/registry/filter" - "github.com/doublecloud/tross/tests/canon/mongo" - "github.com/doublecloud/tross/tests/canon/reference" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -const databaseName string = "db" - -var ( - Source = mongocommon.RecipeSource() - Target = chrecipe.MustTarget(chrecipe.WithInitFile("dump.sql"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - dpconfig.Cloud = &dpconfig.CloudConfig{} - dpconfig.Common = &dpconfig.CommonConfig{ - ClickhouseLocal: os.Getenv("RECIPE_CLICKHOUSE_BIN"), - } -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH HTTP target", Port: Target.HTTPPort}, - helpers.LabeledPort{Label: "CH Native target", Port: Target.NativePort}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Snapshot", Snapshot) - }) -} - -func Ping(t *testing.T) { - client, err := mongocommon.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - Source.Collections = []mongocommon.MongoCollection{ - {DatabaseName: databaseName, CollectionName: "test_data"}, - } - Target.ChClusterName = "" - - doc := `{ - "_id": "D1AAD9AB", - "floors": [ - { - "currency": "EUR", - "value": 0.2, - "countryIds": [ - "IT" - ] - }, - { - "currency": "EUR", - "value": 0.3, - "countryIds": [ - "FR", - "GB" - ] - } - ] -}` - var masterDoc bson.D - require.NoError(t, bson.UnmarshalExtJSON([]byte(doc), false, &masterDoc)) - - require.NoError(t, mongo.InsertDocs(context.Background(), Source, databaseName, "test_data", masterDoc)) - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.TypeSystemVersion = 7 - transfer.Transformation = &server.Transformation{Transformers: &transformer.Transformers{ - DebugMode: false, - Transformers: []transformer.Transformer{{ - clickhouse.Type: clickhouse.Config{ - Tables: filter.Tables{ - IncludeTables: []string{fmt.Sprintf("%s.%s", databaseName, "test_data")}, - }, - Query: ` -SELECT _id, - JSONExtractArrayRaw(document,'floors') as floors_as_string_array, - arrayMap(x -> JSONExtractFloat(x, 'value'), JSONExtractArrayRaw(document,'floors')) as value_from_floors, - arrayMap(x -> JSONExtractString(x, 'currency'), JSONExtractArrayRaw(document,'floors')) as currency_from_floors, - JSONExtractRaw(assumeNotNull(document),'floors') AS floors_as_string -FROM table -SETTINGS - function_json_value_return_type_allow_nullable = true, - function_json_value_return_type_allow_complex = true -`, - }, - }}, - ErrorsOutput: nil, - }} - helpers.Activate(t, transfer) - - canon.SaveJSON(t, reference.FromClickhouse(t, &model.ChSource{ - Database: databaseName, - ShardsList: []model.ClickHouseShard{{Name: "_", Hosts: []string{"localhost"}}}, - NativePort: Target.NativePort, - HTTPPort: Target.HTTPPort, - User: Target.User, - }, true)) -} diff --git a/tests/e2e/mongo2ch/snapshot_flatten/dump.sql b/tests/e2e/mongo2ch/snapshot_flatten/dump.sql deleted file mode 100644 index e042557e..00000000 --- a/tests/e2e/mongo2ch/snapshot_flatten/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE db; diff --git a/tests/e2e/mongo2mock/slots/slot_test.go b/tests/e2e/mongo2mock/slots/slot_test.go deleted file mode 100644 index f7e8a879..00000000 --- a/tests/e2e/mongo2mock/slots/slot_test.go +++ /dev/null @@ -1,362 +0,0 @@ -package slots - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/randutil" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -const ( - testDB1 string = "test_db1" - testDB2 string = "test_db2" - testDB3 string = "test_db3" - collectionName string = "collection" - transferSlotID string = "dttqegn8908aata701lu" -) - -var ( - allDBs = []string{testDB1, testDB2, testDB3} - - port = helpers.GetIntFromEnv("MONGO_LOCAL_PORT") - userName = os.Getenv("MONGO_LOCAL_USER") - userPassword = os.Getenv("MONGO_LOCAL_PASSWORD") -) - -func getSource(collection ...mongo.MongoCollection) *mongo.MongoSource { - return &mongo.MongoSource{ - Hosts: []string{"localhost"}, - Port: port, - User: userName, - Password: server.SecretString(userPassword), - Collections: collection, - } -} - -func getTransfer(source *mongo.MongoSource) server.Transfer { - tr := server.Transfer{ - ID: transferSlotID, - Type: abstract.TransferTypeSnapshotAndIncrement, - Src: source, - Dst: &server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return new(mockSinker) }, - Cleanup: server.Drop, - }, - } - tr.FillDependentFields() - return tr -} - -func connect(source *mongo.MongoSource) (*mongo.MongoClientWrapper, error) { - client, err := mongo.Connect(context.TODO(), source.ConnectionOptions([]string{}), nil) - if err != nil { - return nil, err - } - return client, nil -} - -func init() { - cpconfig.Cloud = &cpconfig.CloudConfig{} - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -func setOplogSize(ctx context.Context, client *mongo.MongoClientWrapper, sizeInSeconds, sizeInMegabytes int) error { - hourOplogSize := float64(sizeInSeconds) / (60.0 * 60.0) - - cmdParams := bson.D{ - bson.E{Key: "replSetResizeOplog", Value: 1}, - bson.E{Key: "size", Value: float64(sizeInMegabytes)}, - bson.E{Key: "minRetentionHours", Value: hourOplogSize}, - } - singleRes := client.Database("admin").RunCommand(ctx, cmdParams) - if singleRes.Err() != nil { - return singleRes.Err() - } - return nil -} - -// just mock sinker -type mockSinker struct{} - -func (m mockSinker) Close() error { return nil } -func (m mockSinker) Push(items []abstract.ChangeItem) error { return nil } - -// controlplane that catches replication failure -type mockCPFailRepl struct { - cpclient.FakeClient - err error -} - -// test data structure -type Pepe struct { - DayOfTheWeek string - DayOfTheWeekID int - InsertDate time.Time -} - -func (f *mockCPFailRepl) FailReplication(transferID string, err error) error { - f.err = err - return nil -} - -func snapshotPhase(t *testing.T, ctx context.Context, source *mongo.MongoSource) { - sourceDBs := []string{} - for _, coll := range source.Collections { - sourceDBs = append(sourceDBs, coll.DatabaseName) - } - - client, err := connect(source) - require.NoError(t, err) - defer client.Close(ctx) - - // prepare oplog - // we need more than 11Mb/S RPS to exhaust oplog by time - // note, that we need retention for 90 seconds to catch up lag (oplog flushes every 60 seconds) - err = setOplogSize(ctx, client, 30, 990) - require.NoError(t, err, "cannot configure oplog size") - - // drop slot info - for _, sourceDB := range allDBs { - _ = client.Database(sourceDB).Collection(mongo.ClusterTimeCollName).Drop(context.Background()) - } - - // insert first records - for i, dbName := range sourceDBs { - db := client.Database(dbName) - _ = db.Collection(collectionName).Drop(context.Background()) - err = db.CreateCollection(context.Background(), collectionName) - require.NoError(t, err) - - coll := db.Collection(collectionName) - - _, err = coll.InsertOne(context.Background(), - Pepe{"Wednesday", i, time.Now()}) - require.NoError(t, err) - } - - // start worker - transfer := getTransfer(source) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - require.NoError(t, err) -} - -func incrementPhaseWithRestart(t *testing.T, ctx context.Context, source *mongo.MongoSource, - updatableCollections []string, sourceAfterRestart *mongo.MongoSource) error { - - client, err := connect(source) - require.NoError(t, err) - defer client.Close(ctx) - - transfer := getTransfer(source) - - // start replication - func() { - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - // full speed generation -- spam oplog - timeStart := time.Now() - for { - for i, dbName := range updatableCollections { - db := client.Database(dbName) - coll := db.Collection(collectionName) - - _, err = coll.InsertOne(context.Background(), - Pepe{randutil.GenerateString("abcdefghijklmnopqrstuvwxyz", 4*1024*1024), - i, time.Now()}) - require.NoError(t, err) - } - - // note: admin rights required - oplogFromTS, _, err := mongo.GetLocalOplogInterval(ctx, client) - require.NoError(t, err) - if timeStart.Before(mongo.FromMongoTimestamp(oplogFromTS)) { - // when oplog rotation happened -- terminate - break - } - time.Sleep(100 * time.Millisecond) - } - // wait a little bit - time.Sleep(5 * time.Second) - }() - - // change source params (if any) - if sourceAfterRestart != nil { - transfer.Src = sourceAfterRestart - transfer.FillDependentFields() - } - - // restart replication - newMockCP := mockCPFailRepl{} - localWorker := local.NewLocalWorker(&newMockCP, &transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - time.Sleep(3 * time.Second) - err = localWorker.Stop() //nolint - require.NoError(t, err) - require.NoError(t, localWorker.Error()) - return newMockCP.err -} - -func findSlots( - t *testing.T, - ctx context.Context, - source *mongo.MongoSource, -) (primitive.Timestamp, primitive.Timestamp, primitive.Timestamp) { - client, err := connect(source) - require.NoError(t, err) - defer client.Close(ctx) - - // find slots for all DBs after - var oplog1, oplog2, oplog3 primitive.Timestamp - for dbName, oplogRef := range map[string]*primitive.Timestamp{ - testDB1: &oplog1, - testDB2: &oplog2, - testDB3: &oplog3, - } { - var pu mongo.ParallelizationUnit - if source.ReplicationSource == mongo.MongoReplicationSourceOplog { - pu = mongo.MakeParallelizationUnitOplog(source.TechnicalDatabase, source.SlotID) - } else { - pu = mongo.MakeParallelizationUnitDatabase(source.TechnicalDatabase, source.SlotID, dbName) - } - clusterTime, err := pu.GetClusterTime(ctx, client) - if err == nil { - *oplogRef = *clusterTime - } - } - return oplog1, oplog2, oplog3 -} - -func TestMongoSlot(t *testing.T) { - ctx := context.Background() - t.Run("StaleDB", func(t *testing.T) { - source := getSource( - mongo.MongoCollection{ - DatabaseName: testDB1, - CollectionName: collectionName, - }, - mongo.MongoCollection{ - DatabaseName: testDB2, - CollectionName: collectionName, - }, - mongo.MongoCollection{ - DatabaseName: testDB3, - CollectionName: collectionName, - }) - source.WithDefaults() - - // start snapshot phase - snapshotPhase(t, ctx, source) - - // find slots for all DBs after activation - oplogAfterSnapshot1, oplogAfterSnapshot2, oplogAfterSnapshot3 := findSlots(t, ctx, source) - require.False(t, oplogAfterSnapshot1.IsZero()) - require.False(t, oplogAfterSnapshot2.IsZero()) - require.False(t, oplogAfterSnapshot3.IsZero()) - - // start increment phase - err := incrementPhaseWithRestart(t, ctx, source, []string{testDB1, testDB2}, nil) - require.NoError(t, err) - - // find slots for all DBs after - oplogAfterRestart1, oplogAfterRestart2, oplogAfterRestart3 := findSlots(t, ctx, source) - require.False(t, oplogAfterRestart1.IsZero()) - require.False(t, oplogAfterRestart2.IsZero()) - require.False(t, oplogAfterRestart3.IsZero()) - - // check that slots has been updated - require.False(t, oplogAfterSnapshot1.Equal(oplogAfterRestart1), "Slot 1 should change during replication") - require.False(t, oplogAfterSnapshot2.Equal(oplogAfterRestart2), "Slot 2 should change during replication") - require.False(t, oplogAfterSnapshot3.Equal(oplogAfterRestart3), "Slot 3 should change during replication") - - }) - t.Run("StaleDBOplog", func(t *testing.T) { - source := getSource( - mongo.MongoCollection{ - DatabaseName: testDB3, - CollectionName: collectionName, - }) - source.ReplicationSource = mongo.MongoReplicationSourceOplog - source.WithDefaults() - - // start snapshot phase - snapshotPhase(t, ctx, source) - - // find slots for all DBs after activation - _, _, oplogAfterSnapshot3 := findSlots(t, ctx, source) - require.False(t, oplogAfterSnapshot3.IsZero()) - - // start increment phase - err := incrementPhaseWithRestart(t, ctx, source, []string{testDB1, testDB2}, nil) - require.NoError(t, err) - - // find slots for all DBs after - _, _, oplogAfterRestart3 := findSlots(t, ctx, source) - require.False(t, oplogAfterRestart3.IsZero()) - - // check that slots has been updated - require.False(t, oplogAfterSnapshot3.Equal(oplogAfterRestart3), "Slot 3 should change during replication") - }) - // this needed for checking ChangeStreamHistoryLost - t.Run("CheckOplogFailure", func(t *testing.T) { - sourceBefore := getSource(mongo.MongoCollection{ - DatabaseName: testDB1, - CollectionName: collectionName, - }, mongo.MongoCollection{ - DatabaseName: testDB2, - CollectionName: collectionName, - }) - sourceBefore.WithDefaults() - sourceAfter := getSource(mongo.MongoCollection{ - DatabaseName: testDB1, - CollectionName: collectionName, - }, mongo.MongoCollection{ - DatabaseName: testDB2, - CollectionName: collectionName, - }, mongo.MongoCollection{ - DatabaseName: testDB3, - CollectionName: collectionName, - }) - sourceAfter.WithDefaults() - - // start snapshot phase - snapshotPhase(t, ctx, sourceBefore) - - // find slots for all DBs after activation - oplogAfterSnapshot1, oplogAfterSnapshot2, oplogAfterSnapshot3 := findSlots(t, ctx, sourceBefore) - require.False(t, oplogAfterSnapshot1.IsZero()) - require.False(t, oplogAfterSnapshot2.IsZero()) - require.True(t, oplogAfterSnapshot3.IsZero()) - - // start increment phase, and change source parameters after restart (note sourceAfter parameter) - err := incrementPhaseWithRestart(t, ctx, sourceBefore, []string{testDB1, testDB2, testDB3}, sourceAfter) - require.ErrorContains(t, err, "Cannot get cluster time for database 'test_db3', try to Activate transfer again.") - - // find slots for all DBs after - oplogAfterRestart1, oplogAfterRestart2, oplogAfterRestart3 := findSlots(t, ctx, sourceAfter) - require.False(t, oplogAfterRestart1.IsZero()) - require.False(t, oplogAfterRestart2.IsZero()) - require.True(t, oplogAfterRestart3.IsZero()) - - // check that slots has been updated - require.False(t, oplogAfterSnapshot1.Equal(oplogAfterRestart1), "Slot 1 should have change during replication") - require.False(t, oplogAfterSnapshot2.Equal(oplogAfterRestart2), "Slot 2 should have change during replication") - }) -} diff --git a/tests/e2e/mongo2mock/tech_db_permission/permission_test.go b/tests/e2e/mongo2mock/tech_db_permission/permission_test.go deleted file mode 100644 index 4ca5f6ba..00000000 --- a/tests/e2e/mongo2mock/tech_db_permission/permission_test.go +++ /dev/null @@ -1,240 +0,0 @@ -package permissiontest - -import ( - "context" - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -const ( - readOnlyDatabase string = "read_only_db" - technicalDatabase string = "data_transfer" - collectionName string = "collection" -) - -var ( - port = helpers.GetIntFromEnv("ADMIN_MONGO_LOCAL_PORT") - adminUserName = os.Getenv("ADMIN_MONGO_LOCAL_USER") - adminUserPassword = os.Getenv("ADMIN_MONGO_LOCAL_PASSWORD") - transferUserName = os.Getenv("TRANSFER_USER_NAME") - transferUserPassword = os.Getenv("TRANSFER_USER_PASSWORD") -) - -func getSource(user, password string, collection ...mongocommon.MongoCollection) *mongocommon.MongoSource { - return &mongocommon.MongoSource{ - Hosts: []string{"localhost"}, - Port: port, - User: user, - Password: server.SecretString(password), - Collections: collection, - } -} - -var ( - adminUserSource = getSource(adminUserName, adminUserPassword) -) - -func init() { - cpconfig.Cloud = &cpconfig.CloudConfig{} - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -func connect(source *mongocommon.MongoSource) (*mongocommon.MongoClientWrapper, error) { - client, err := mongocommon.Connect(context.Background(), source.ConnectionOptions([]string{}), nil) - if err != nil { - return nil, err - } - return client, nil -} - -func makeReadOnlyUser(ctx context.Context, adminSource *mongocommon.MongoSource, userName, userPassword string) error { - client, err := connect(adminSource) - if err != nil { - return err - } - defer client.Close(ctx) - - // https://mongoing.com/docs/reference/command/createUser.html#dbcmd.createUser - // - // db.runCommand("createUser", {createUser:"asdf", pwd:"kek", roles: [ - // { role: "", db: "" } | "", - // ... - // ],}) - // - // db.runCommand("createUser", {createUser:"asdf", pwd:"kek", roles: ["read", {db: readWrite}]}) - cmdParams := bson.D{ - bson.E{Key: "createUser", Value: userName}, - bson.E{Key: "pwd", Value: userPassword}, - bson.E{Key: "roles", Value: bson.A{ - bson.D{ - bson.E{Key: "role", Value: "read"}, - bson.E{Key: "db", Value: readOnlyDatabase}, - }, - bson.D{ - bson.E{Key: "role", Value: "readWrite"}, - bson.E{Key: "db", Value: technicalDatabase}, - }, - }}, - } - singleRes := client.Database("admin").RunCommand(ctx, cmdParams) - if singleRes.Err() != nil { - return singleRes.Err() - } - return nil -} - -// TODO(@kry127) refactor doubles: https://github.com/doublecloud/tross/arc_vcs/transfer_manager/go/pkg/worker/tasks/e2e/load_sharded_snapshot_test.go?rev=r9868991#L111 -type permissionSinker struct { - bannedCollections []mongocommon.MongoCollection -} - -func (d permissionSinker) Close() error { return nil } -func (d permissionSinker) Push(items []abstract.ChangeItem) error { - for _, item := range items { - for _, bc := range d.bannedCollections { - if bc.DatabaseName == item.Schema { - if bc.CollectionName == item.Table || bc.CollectionName == "*" { - return xerrors.Errorf("error: item should not be uploaded: %v", item) - } - } - } - } - return nil -} - -func makePermissionSinker(bannedCollections ...mongocommon.MongoCollection) *permissionSinker { - return &permissionSinker{ - bannedCollections: bannedCollections, - } -} - -func snapshotAndIncrement(t *testing.T, ctx context.Context, source *mongocommon.MongoSource, permissionSinker *permissionSinker, - sourceDB, collection string, expectError bool) { - adminClient, err := connect(adminUserSource) - require.NoError(t, err) - defer adminClient.Close(ctx) - - //------------------------------------------------------------------------------------ - // insert one record - - adminDB := adminClient.Database(sourceDB) - defer func() { - // clear collection in the end (for local debug) - _ = adminDB.Collection(collection).Drop(context.Background()) - }() - err = adminDB.CreateCollection(context.Background(), collection) - require.NoError(t, err) - - adminColl := adminDB.Collection(collection) - - type Myamlya struct { - Name string - Age int - TableToDrop string - } - - _, err = adminColl.InsertOne(context.Background(), - Myamlya{"Eugene", 3, "connector_endpoints"}) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - transfer := server.Transfer{ - Type: abstract.TransferTypeSnapshotAndIncrement, - Src: source, - Dst: &server.MockDestination{ - SinkerFactory: func() abstract.Sinker { - return permissionSinker - }, - Cleanup: server.Drop, - }, - ID: helpers.TransferID, - } - - accessErrorChecker := func(err error) { - if expectError { - require.Error(t, err, "error should happen: expected that user has not enough permission") - expectedMessage := fmt.Sprintf("(Unauthorized) not authorized on %s to execute command", sourceDB) - require.ErrorContainsf(t, err, expectedMessage, - "error should be about unauthorized on source database. Expected message: %s", sourceDB) - } else { - msg := "expected, that user has permission to upload object, but got error: %v" - require.NoError(t, err, fmt.Sprintf(msg, err)) - } - } - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - accessErrorChecker(err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // replicate one record - - _, err = adminColl.InsertOne(context.Background(), Myamlya{"Victor", 2, "public"}) - require.NoError(t, err) -} - -func TestMongoPermissions(t *testing.T) { - ctx := context.Background() - - err := makeReadOnlyUser(ctx, adminUserSource, transferUserName, transferUserPassword) - expectedError := fmt.Sprintf("(Location51003) User \"%s@admin\" already exists", transferUserName) - switch { - case err != nil && err.Error() == expectedError: // OK - default: - require.NoError(t, err, "unable to create read-only user") - } - - t.Run("AttemptToWriteToReadonlyTest", func(t *testing.T) { - t.Skip("Skipped when fixing TM-4906, can be turned on again when auth will be working") - src := getSource(transferUserName, transferUserPassword, - mongocommon.MongoCollection{DatabaseName: readOnlyDatabase, CollectionName: collectionName}, - ) - dst := makePermissionSinker() - snapshotAndIncrement(t, ctx, src, dst, readOnlyDatabase, collectionName, true) - }) - - t.Run("ReadFromReadOnlyWriteToTechnicalTest", func(t *testing.T) { - src := getSource(transferUserName, transferUserPassword, - mongocommon.MongoCollection{DatabaseName: readOnlyDatabase, CollectionName: collectionName}, - ) - src.TechnicalDatabase = technicalDatabase - dst := makePermissionSinker(mongocommon.MongoCollection{DatabaseName: technicalDatabase, CollectionName: "*"}) - snapshotAndIncrement(t, ctx, src, dst, readOnlyDatabase, collectionName, false) - }) - t.Run("ReadFromLegacyOplog", func(t *testing.T) { - src := getSource(adminUserName, adminUserPassword, - mongocommon.MongoCollection{DatabaseName: readOnlyDatabase, CollectionName: collectionName}, - ) - src.ReplicationSource = mongocommon.MongoReplicationSourceOplog - dst := makePermissionSinker(mongocommon.MongoCollection{DatabaseName: mongocommon.DataTransferSystemDatabase, CollectionName: "*"}) - snapshotAndIncrement(t, ctx, src, dst, readOnlyDatabase, collectionName, false) - }) - t.Run("ReadFromLegacyOplogOverrideDB", func(t *testing.T) { - src := getSource(adminUserName, adminUserPassword, - mongocommon.MongoCollection{DatabaseName: readOnlyDatabase, CollectionName: collectionName}, - ) - src.TechnicalDatabase = technicalDatabase - src.ReplicationSource = mongocommon.MongoReplicationSourceOplog - dst := makePermissionSinker(mongocommon.MongoCollection{DatabaseName: technicalDatabase, CollectionName: "*"}) - snapshotAndIncrement(t, ctx, src, dst, readOnlyDatabase, collectionName, false) - }) -} diff --git a/tests/e2e/mongo2mongo/add_db_on_snapshot/check_db_test.go b/tests/e2e/mongo2mongo/add_db_on_snapshot/check_db_test.go deleted file mode 100644 index 74a1e4f6..00000000 --- a/tests/e2e/mongo2mongo/add_db_on_snapshot/check_db_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongodataagent "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/mongo" -) - -const ( - GoodDatabase = "lawful_good_db" - BadDatabase = "yolo234_database" - Collection = "some_collection817" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = mongodataagent.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongodataagent.MongoCollection{ - {DatabaseName: GoodDatabase, CollectionName: "*"}, - {DatabaseName: BadDatabase, CollectionName: "*"}, - }, - } - Target = mongodataagent.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -type DummyData struct { - Value int -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongodataagent.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongodataagent.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongodataagent.MongoDestination) (*mongodataagent.MongoClientWrapper, error) { - return mongodataagent.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -func clearSrc(t *testing.T, client *mongodataagent.MongoClientWrapper) { - t.Helper() - for _, dbName := range []string{GoodDatabase, BadDatabase} { - db := client.Database(dbName) - _ = db.Drop(context.Background()) - } -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("CheckDBAdditionOnSnapshot", CheckDBAdditionOnSnapshot) - }) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(&Source) - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func CheckDBAdditionOnSnapshot(t *testing.T) { - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = client.Close(context.Background()) }() - - clearSrc(t, client) - dbOk := client.Database(GoodDatabase) - collOk := dbOk.Collection(Collection) - dbNotOk := client.Database(BadDatabase) - collNotOk := dbNotOk.Collection(Collection) - - logger.Log.Info("prefill both collection with entities in snapshot") - collectionCount := 20000 - var dummyDataSlice []interface{} - for i := 0; i < collectionCount; i++ { - dummyDataSlice = append(dummyDataSlice, DummyData{Value: i}) - } - var im *mongo.InsertManyResult - im, err = collOk.InsertMany(context.Background(), dummyDataSlice) - require.NoError(t, err) - require.Len(t, im.InsertedIDs, collectionCount) - im, err = collNotOk.InsertMany(context.Background(), dummyDataSlice) - require.NoError(t, err) - require.Len(t, im.InsertedIDs, collectionCount) - - logger.Log.Info("start replication") - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - errChan := make(chan error, 1) - go func() { - errChan <- localWorker.Run() - }() - defer func() { _ = localWorker.Stop() }() - - logger.Log.Info("just after worker started, let's drop second database and recreate it during snapshot") - _ = dbNotOk.Drop(context.Background()) - for i := 0; i < 20; i++ { - _, err := collNotOk.InsertOne(context.Background(), DummyData{Value: i}) - require.NoError(t, err) - } - - logger.Log.Info("wait for replication fatal error with exact message") - timer := time.NewTimer(30 * time.Second) - select { - case err := <-errChan: - require.True(t, abstract.IsFatal(err), "should be fatal") - expectMessage := fmt.Sprintf("Cannot get cluster time for database '%s', try to Activate transfer again. ", BadDatabase) - require.Contains(t, err.Error(), expectMessage, "Error should be about cluster time for new collection") - require.Contains(t, err.Error(), BadDatabase, "Should contain bad database name") - case <-timer.C: - t.Fatal("Couldn't wait for error from worker") - } -} diff --git a/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go b/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go deleted file mode 100644 index bc6e2459..00000000 --- a/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go +++ /dev/null @@ -1,382 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "math/rand" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongostorage "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/mongo" -) - -const ( - DB1 = "db1" - DB2 = "db2" - CollectionGood = "kry127_good" - CollectionBsonTooLarge = "kry127_bson_too_large" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = mongostorage.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongostorage.MongoCollection{ - {DatabaseName: DB1, CollectionName: CollectionGood}, - {DatabaseName: DB1, CollectionName: CollectionBsonTooLarge}, - {DatabaseName: DB2, CollectionName: "*"}, // this is almost the same - }, - } - Target = mongostorage.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -type KV struct { - Key string `bson:"_id"` - Value string -} - -const Alphabet = "abcdefghijklmnopqrstuvwxyz" - -func randString(size int) string { - ret := make([]byte, size) - for i := range ret { - ret[i] = Alphabet[int(rand.Uint32())%len(Alphabet)] - } - return string(ret) -} - -func NewKV(keysize, valsize int) *KV { - return &KV{Key: randString(keysize), Value: randString(valsize)} -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongostorage.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongostorage.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongostorage.MongoDestination) (*mongostorage.MongoClientWrapper, error) { - return mongostorage.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - // Test two different modes. - // NOTE: heavily dependent on mongo version, be careful with recipes - t.Run("Load_FromChangeStream", LoadFromchangestream) - t.Run("Load_FromPureCursor", LoadFrompurecursor) - t.Run("Load_FromOplog", LoadFromoplog) - }) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(&Source) - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func clearSrc(t *testing.T, client *mongostorage.MongoClientWrapper) { - t.Helper() - var err error - for _, dbName := range []string{DB1, DB2} { - db := client.Database(dbName) - _ = db.Collection(CollectionGood).Drop(context.Background()) - err = db.CreateCollection(context.Background(), CollectionGood) - require.NoError(t, err) - _ = db.Collection(CollectionBsonTooLarge).Drop(context.Background()) - err = db.CreateCollection(context.Background(), CollectionBsonTooLarge) - require.NoError(t, err) - } -} - -func LoadFromchangestream(t *testing.T) { - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = client.Close(context.Background()) }() - - // recreate collections on source - clearSrc(t, client) - db1 := client.Database(DB1) - coll1good := db1.Collection(CollectionGood) - coll1toolarge := db1.Collection(CollectionBsonTooLarge) - db2 := client.Database(DB2) - coll2good := db2.Collection(CollectionGood) - coll2toolarge := db2.Collection(CollectionBsonTooLarge) - - // wait a little bit for oplog to shake up - time.Sleep(5 * time.Second) // TODO(@kry127) is it needed - - // start replication - Source.ReplicationSource = mongostorage.MongoReplicationSourcePerDatabaseFullDocument // set fetch mode - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - errChan := make(chan error, 1) - go func() { - errChan <- localWorker.Run() // like .Start(), but we in control for processing error in test - }() - defer func() { _ = localWorker.Stop() }() - - // replicate good records - dstStorage, err := mongostorage.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - var goodInsertionsSize uint64 = 2940 - goodInsertionsCount := 20 - for _, coll := range []*mongo.Collection{coll1good, coll2good, coll1toolarge, coll2toolarge} { - for i := 0; i < goodInsertionsCount; i++ { - _, err = coll.InsertOne(context.Background(), NewKV(20, 100)) - require.NoError(t, err) - } - } - time.Sleep(time.Second) - - tryingsCount := 30 - tries := 0 - var dstTableSize uint64 = 0 - for tries = 0; tries < tryingsCount; tries++ { - allOk := true - for _, td := range []abstract.TableDescription{ - {Schema: DB1, Name: CollectionGood}, - {Schema: DB2, Name: CollectionGood}, - {Schema: DB1, Name: CollectionBsonTooLarge}, - {Schema: DB2, Name: CollectionBsonTooLarge}, - } { - dstTableSize, err = dstStorage.TableSizeInBytes(td.ID()) - require.NoError(t, err) - - t.Logf("Table %s, calculating size. Expected %d, actual %d", td.String(), goodInsertionsSize, dstTableSize) - if dstTableSize != goodInsertionsSize { - allOk = false - break - } - } - if allOk { - break - } - time.Sleep(time.Second) - } - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - - // insert large documents - for _, coll := range []*mongo.Collection{coll1toolarge, coll2toolarge} { - _, err = coll.InsertOne(context.Background(), NewKV(4*1024*1024, 10*1024*1024)) // should be processable - require.NoError(t, err) - _, err = coll.InsertOne(context.Background(), NewKV(5*1024*1024, 10)) // should fail in full document mode - require.NoError(t, err) - } - - // wait for appropriate error from replication - timer := time.NewTimer(30 * time.Second) - select { - case err := <-errChan: - require.True(t, abstract.IsFatal(err), "should be fatal") - require.Contains(t, err.Error(), "BSONObjectTooLarge", "Error should be about too large object in oplog") - containsProperName := strings.Contains(err.Error(), coll1toolarge.Name()) || strings.Contains(err.Error(), coll2toolarge.Name()) - require.True(t, containsProperName, "Should contain collection name") - case <-timer.C: - t.Fatal("Couldn't wait for error from worker") - } -} - -func LoadFrompurecursor(t *testing.T) { - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = client.Close(context.Background()) }() - - // recreate collections on source - clearSrc(t, client) - db1 := client.Database(DB1) - coll1good := db1.Collection(CollectionGood) - coll1toolarge := db1.Collection(CollectionBsonTooLarge) - db2 := client.Database(DB2) - coll2good := db2.Collection(CollectionGood) - coll2toolarge := db2.Collection(CollectionBsonTooLarge) - - // wait a little bit for oplog to shake up - time.Sleep(5 * time.Second) // TODO(@kry127) is it needed - - // start replication - Source.ReplicationSource = mongostorage.MongoReplicationSourcePerDatabase - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer func() { _ = localWorker.Stop() }() - - // replicate good records - dstStorage, err := mongostorage.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - const goodInsertionsCount = 20 - const goodInsertionsSize uint64 = 2940 - for _, coll := range []*mongo.Collection{coll1good, coll2good, coll1toolarge, coll2toolarge} { - for i := 0; i < goodInsertionsCount; i++ { - _, err = coll.InsertOne(context.Background(), NewKV(20, 100)) - require.NoError(t, err) - } - } - time.Sleep(time.Second) - - const tryingsCount int = 30 - var dstTableSize uint64 = 0 - for tries := 0; tries < tryingsCount; tries++ { - allOk := true - for _, td := range []abstract.TableDescription{ - {Schema: DB1, Name: CollectionGood}, - {Schema: DB2, Name: CollectionGood}, - {Schema: DB1, Name: CollectionBsonTooLarge}, - {Schema: DB2, Name: CollectionBsonTooLarge}, - } { - dstTableSize, err = dstStorage.TableSizeInBytes(td.ID()) - require.NoError(t, err) - - t.Logf("Table %s, calculating size. Expected %d, actual %d", td.String(), goodInsertionsSize, dstTableSize) - if dstTableSize != goodInsertionsSize { - allOk = false - break - } - } - if allOk { - break - } - time.Sleep(time.Second) - } - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - - // insert large documents - const badInsertionsSize uint64 = 19923008 - for _, coll := range []*mongo.Collection{coll1toolarge, coll2toolarge} { - _, err = coll.InsertOne(context.Background(), NewKV(4*1024*1024, 10*1024*1024)) // should be processable - require.NoError(t, err) - _, err = coll.InsertOne(context.Background(), NewKV(5*1024*1024, 10)) // also should be processed with pure cursor - require.NoError(t, err) - //_, err = coll.InsertOne(context.Background(), NewKV(5*1024*1024 + 512 * 1024, 0)) // you shall not pass :D - //require.NoError(t, err) - } - - for tries := 0; tries < tryingsCount; tries++ { - allOk := true - for _, td := range []abstract.TableDescription{ - {Schema: DB1, Name: CollectionBsonTooLarge}, - {Schema: DB2, Name: CollectionBsonTooLarge}, - } { - dstTableSize, err = dstStorage.TableSizeInBytes(td.ID()) - require.NoError(t, err) - - t.Logf("Table %s, calculating size. Expected %d, actual %d", td.String(), goodInsertionsSize+badInsertionsSize, dstTableSize) - if dstTableSize != goodInsertionsSize+badInsertionsSize { - allOk = false - break - } - } - if allOk { - break - } - time.Sleep(time.Second) - } - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func LoadFromoplog(t *testing.T) { - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = client.Close(context.Background()) }() - - // recreate collections on source - clearSrc(t, client) - db := client.Database(DB1) - coll := db.Collection(CollectionBsonTooLarge) - - // start replication - Source.ReplicationSource = mongostorage.MongoReplicationSourceOplog // set replication source - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer func() { _ = localWorker.Stop() }() - - // insert large documents - _, err = coll.InsertOne(context.Background(), NewKV(4*1024*1024, 10*1024*1024)) // should be processable - require.NoError(t, err) - _, err = coll.InsertOne(context.Background(), NewKV(5*1024*1024, 10)) // also should be processed with pure cursor - require.NoError(t, err) - _, err = coll.InsertOne(context.Background(), NewKV(5*1024*1024+512*1024, 0)) // you shall not pass :D - require.NoError(t, err) - _, err = coll.InsertOne(context.Background(), NewKV(15*1024*1024, 30)) // ??? - require.NoError(t, err) - // wait for large document insertion - time.Sleep(5 * time.Second) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mongo2mongo/bson_order/reorder_test.go b/tests/e2e/mongo2mongo/bson_order/reorder_test.go deleted file mode 100644 index a6b0780d..00000000 --- a/tests/e2e/mongo2mongo/bson_order/reorder_test.go +++ /dev/null @@ -1,327 +0,0 @@ -package reorder - -import ( - "context" - "fmt" - "math/rand" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/internal/slices" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -func makeSource(t *testing.T, database, collection string) *mongocommon.MongoSource { - return &mongocommon.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: helpers.GetEnvOfFail(t, "MONGO_LOCAL_USER"), - Password: server.SecretString(helpers.GetEnvOfFail(t, "MONGO_LOCAL_PASSWORD")), - Collections: []mongocommon.MongoCollection{ - {DatabaseName: database, CollectionName: collection}, - }, - BatchingParams: &mongocommon.BatcherParameters{ - BatchSizeLimit: mongocommon.DefaultBatchSizeLimit, - KeySizeThreshold: mongocommon.DefaultKeySizeThreshold, - BatchFlushInterval: mongocommon.DefaultBatchFlushInterval, - }, - } -} - -func makeTarget(t *testing.T, targetDatabase string) *mongocommon.MongoDestination { - return &mongocommon.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: helpers.GetEnvOfFail(t, "MONGO_LOCAL_USER"), - Password: server.SecretString(helpers.GetEnvOfFail(t, "MONGO_LOCAL_PASSWORD")), - Database: targetDatabase, - } -} - -const alphabet = "abcdefghijklmnopqrstuvwxyz" - -func randString(size int) string { - ret := make([]byte, size) - for i := range ret { - ret[i] = alphabet[rand.Intn(len(alphabet))] - } - return string(ret) -} - -func makeBson() bson.D { - result := bson.D{} - keyCount := 2 + rand.Intn(3) - ids := []int{} - for i := 0; i < len(alphabet); i++ { - ids = append(ids, i) - } - slices.Shuffle(ids, rand.NewSource(time.Now().Unix())) - for i := 0; i < keyCount; i++ { - result = append(result, bson.E{Key: string(alphabet[ids[i]]), Value: randString(int(rand.Uint32()%4 + 2))}) - } - return result -} - -type BsonAsID struct { - Key bson.D `bson:"_id"` - Value string `bson:"value"` -} - -type BsonAsIndex struct { - Index bson.D `bson:"index"` - Value string `bson:"value"` -} - -func bsonsEqual(a, b bson.D) int { - if len(a) < len(b) { - return -1 - } - if len(a) > len(b) { - return 1 - } - - for i := range a { - if a[i].Key < b[i].Key { - return -1 - } - if a[i].Key > b[i].Key { - return 1 - } - - // assume values are string-only =) - if a[i].Value.(string) < b[i].Value.(string) { - return -1 - } - if a[i].Value.(string) > b[i].Value.(string) { - return 1 - } - } - return 0 -} - -func bsonPermEqual(a, b bson.D) bool { - aM := a.Map() - bM := b.Map() - checkXincludedInY := func(x, y map[string]interface{}) bool { - for keyX, valX := range x { - valY, ok := y[keyX] - if !ok || valX != valY { - return false - } - } - return true - } - if !checkXincludedInY(aM, bM) { - return false - } - if !checkXincludedInY(bM, aM) { - return false - } - return true -} - -func bsonASubB(a []bson.D, b []bson.D) []interface{} { - res := []interface{}{} - for _, aa := range a { - hasEqual := false - for _, bb := range b { - if bsonsEqual(aa, bb) == 0 || bsonPermEqual(aa, bb) { - hasEqual = true - } - } - if !hasEqual { - res = append(res, aa) - } - } - return res -} - -func fetchPermutations(a []bson.D, b []bson.D) []interface{} { - res := []interface{}{} - for _, aa := range a { - hasPerm := false - for _, bb := range b { - if bsonsEqual(aa, bb) != 0 && bsonPermEqual(aa, bb) { - hasPerm = true - } - } - if hasPerm { - res = append(res, aa) - } - } - return res -} - -type collectionGenerator func(int) []interface{} - -func generateBsonAsID(amount int) []interface{} { - var documents []interface{} - for i := 0; i < amount; i++ { - documents = append(documents, BsonAsID{ - Key: makeBson(), - Value: randString(2), - }) - } - return documents -} - -func generateBsonAsIndex(amount int) []interface{} { - var documents []interface{} - for i := 0; i < amount; i++ { - documents = append(documents, BsonAsIndex{ - Index: makeBson(), - Value: randString(2), - }) - } - return documents -} - -type transferStage func(t *testing.T, inserter func() uint64, transfer *server.Transfer, targetDatabase, targetCollection string) - -func snapshotOnlyStage(t *testing.T, inserter func() uint64, transfer *server.Transfer, _, _ string) { - _ = inserter() - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - snapshotLoader := tasks.NewSnapshotLoader(cpclient.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) -} - -func replicationOnlyStage(t *testing.T, inserter func() uint64, transfer *server.Transfer, targetDatabase, targetCollection string) { - err := tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - amount := inserter() - - err = helpers.WaitDestinationEqualRowsCount(targetDatabase, targetCollection, helpers.GetSampleableStorageByModel(t, transfer.Dst), 60*time.Second, amount) - require.NoError(t, err) -} - -func TestBsonOrdering(t *testing.T) { - t.Run("Test ID snapshot only", mkBsonTester("snapshot_id", generateBsonAsID, snapshotOnlyStage, abstract.TransferTypeSnapshotOnly).RunTest) - t.Run("Test Index snapshot only", mkBsonTester("snapshot_index", generateBsonAsIndex, snapshotOnlyStage, abstract.TransferTypeSnapshotOnly).RunTest) - t.Run("Test ID replication only", mkBsonTester("replication_id", generateBsonAsID, replicationOnlyStage, abstract.TransferTypeIncrementOnly).RunTest) - t.Run("Test Index replication only", mkBsonTester("replication_index", generateBsonAsIndex, replicationOnlyStage, abstract.TransferTypeIncrementOnly).RunTest) -} - -type bsonOrderingTester struct { - collGenerator collectionGenerator - stage transferStage - trType abstract.TransferType - - collectionName string -} - -func mkBsonTester(collectionName string, - collGenerator collectionGenerator, - stage transferStage, - trType abstract.TransferType, -) *bsonOrderingTester { - return &bsonOrderingTester{ - collGenerator: collGenerator, - stage: stage, - trType: trType, - collectionName: collectionName, - } -} - -func (b *bsonOrderingTester) RunTest(t *testing.T) { - ctx := context.Background() - - sourceDB := "tm3500" - targetDB := fmt.Sprintf("%s_d", sourceDB) - src := makeSource(t, sourceDB, b.collectionName) - dst := makeTarget(t, targetDB) - - sourceClient, err := mongocommon.Connect(context.Background(), src.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - targetClient, err := mongocommon.Connect(context.Background(), dst.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - mongoSourceCollection := sourceClient.Database(sourceDB).Collection(b.collectionName) - mongoTargetCollection := targetClient.Database(targetDB).Collection(b.collectionName) - - _ = mongoSourceCollection.Drop(ctx) - _ = mongoTargetCollection.Drop(ctx) - - err = sourceClient.Database(sourceDB).CreateCollection(ctx, b.collectionName) - require.NoError(t, err) - - documents := b.collGenerator(20) - - tr := helpers.MakeTransfer("dttztm3500ztestzid", src, dst, b.trType) - b.stage(t, func() uint64 { - res, err := mongoSourceCollection.InsertMany(ctx, documents) - require.NoError(t, err) - return uint64(len(res.InsertedIDs)) - }, tr, targetDB, b.collectionName) - - cursor, err := mongoTargetCollection.Find(ctx, bson.D{}) - require.NoError(t, err) - - var newDocuments []bson.D - err = cursor.All(ctx, &newDocuments) - require.NoError(t, err) - - keyAsID, indexAsID := false, false - docsToBsons := slices.Map(documents, func(doc interface{}) bson.D { - switch d := doc.(type) { - case BsonAsID: - keyAsID = true - return d.Key - case BsonAsIndex: - indexAsID = true - return d.Index - default: - t.Fatalf("unexpected type of document: '%T'", doc) - return nil - } - }) - if keyAsID && indexAsID { - t.Fatalf("Collection of heterogeneous type! choose only one of them") - } - newDocumentsToBsons := slices.Map(newDocuments, func(doc bson.D) bson.D { - if keyAsID { - return doc.Map()["_id"].(bson.D) - } - if indexAsID { - return doc.Map()["index"].(bson.D) - } - t.Fatalf("Illegal: collection should have certain type") - return nil - }) - - // compare slices - perms := fetchPermutations(docsToBsons, newDocumentsToBsons) - lhsMiss := bsonASubB(docsToBsons, newDocumentsToBsons) - rhsMiss := bsonASubB(newDocumentsToBsons, docsToBsons) - - if len(perms) > 0 { - t.Errorf("%d permutations found: %v", len(perms), perms) - } - if len(lhsMiss) > 0 { - t.Errorf("%d documents dropped during transfer: %v", len(lhsMiss), lhsMiss) - } - if len(rhsMiss) > 0 { - t.Errorf("%d documents appeared during transfer: %v", len(rhsMiss), rhsMiss) - } - require.Empty(t, perms, "some documents shuffled fields during transfer") - require.Empty(t, lhsMiss, "some documents dropped during transfer") - require.Empty(t, rhsMiss, "some documents appeared during transfer") -} diff --git a/tests/e2e/mongo2mongo/db_rename/check_db_test.go b/tests/e2e/mongo2mongo/db_rename/check_db_test.go deleted file mode 100644 index edcef820..00000000 --- a/tests/e2e/mongo2mongo/db_rename/check_db_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "strings" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -var ( - ctx = context.Background() - Source = mongocommon.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongocommon.MongoCollection{}, - } - Target = mongocommon.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - Database: "custom_target_db", - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - } -) - -func init() { - _ = os.Setenv("YC", "1") // Do not go to vanga - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -//--------------------------------------------------------------------------------------------------------------------- -// Utils - -func LogMongoSource(s *mongocommon.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongocommon.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongocommon.MongoDestination) (*mongocommon.MongoClientWrapper, error) { - return mongocommon.Connect(ctx, t.ConnectionOptions([]string{}), nil) -} - -//--------------------------------------------------------------------------------------------------------------------- -// Source db name NOT given and target db name given - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Snapshot", Snapshot) - }) -} - -func Ping(t *testing.T) { - // Ping src - LogMongoSource(&Source) - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - err = client.Ping(ctx, nil) - require.NoError(t, err) - - // Ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - require.NoError(t, err) - err = client2.Ping(ctx, nil) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // Insert one record into each db in the source - // They must later show up in the target (in a single custom-named db) - - originalDB1 := client.Database("original_db_1") - err = originalDB1.CreateCollection(ctx, "grass_pokemon") - require.NoError(t, err) - originalDB2 := client.Database("original_db_2") - err = originalDB2.CreateCollection(ctx, "fire_pokemon") - require.NoError(t, err) - - grassPokemon := originalDB1.Collection("grass_pokemon") - firePokemon := originalDB2.Collection("fire_pokemon") - - bulbasaur := bson.D{{ - Key: "Name", - Value: "Bulbasaur", - }} - charmander := bson.D{{ - Key: "Name", - Value: "Charmander", - }} - - _, err = grassPokemon.InsertOne(ctx, bulbasaur) - require.NoError(t, err) - _, err = firePokemon.InsertOne(ctx, charmander) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // Upload snapshot - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(client2.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(ctx, tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // Check results - - targetClient, err := mongocommon.Connect(ctx, Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - // Both original dbs must be absent in target (contents must appear in renamed db) - - originalDB1 = targetClient.Database("original_db_1") - res, err := originalDB1.ListCollectionNames(ctx, bson.D{}) - require.NoError(t, err) - require.Len(t, res, 0) - - originalDB2 = targetClient.Database("original_db_2") - res, err = originalDB2.ListCollectionNames(ctx, bson.D{}) - require.NoError(t, err) - require.Len(t, res, 0) - - renamedDB := targetClient.Database("custom_target_db") - res, err = renamedDB.ListCollectionNames(ctx, bson.D{}) - require.NoError(t, err) - resStr := strings.Join(res, ", ") - // Both collections (from the 2 original dbs) must appear here - require.Len(t, res, 2, "Collections: %s", resStr) - require.Contains(t, resStr, "grass_pokemon") - require.Contains(t, resStr, "fire_pokemon") - - grassColl := renamedDB.Collection("grass_pokemon") - fireColl := renamedDB.Collection("fire_pokemon") - - var docResult bson.M - err = grassColl.FindOne(ctx, bulbasaur).Decode(&docResult) - require.NoError(t, err) - err = fireColl.FindOne(ctx, charmander).Decode(&docResult) - require.NoError(t, err) -} diff --git a/tests/e2e/mongo2mongo/db_rename_rep/check_db_test.go b/tests/e2e/mongo2mongo/db_rename_rep/check_db_test.go deleted file mode 100644 index b862c996..00000000 --- a/tests/e2e/mongo2mongo/db_rename_rep/check_db_test.go +++ /dev/null @@ -1,242 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -var ( - ctx = context.Background() - targetDBName = "custom_db_name" - TransferType = abstract.TransferTypeIncrementOnly - Source = mongocommon.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongocommon.MongoCollection{}, - } - Target = mongocommon.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - Database: targetDBName, - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - _ = os.Setenv("YC", "1") // Do not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -//--------------------------------------------------------------------------------------------------------------------- -// Utils - -func LogMongoSource(s *mongocommon.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongocommon.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongocommon.MongoDestination) (*mongocommon.MongoClientWrapper, error) { - return mongocommon.Connect(ctx, t.ConnectionOptions([]string{}), nil) -} - -//--------------------------------------------------------------------------------------------------------------------- -// Both source db name and target db name given - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Replication test", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Load", Load) - }) -} - -func Ping(t *testing.T) { - // Ping src - LogMongoSource(&Source) - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(ctx) }() - require.NoError(t, err) - err = client.Ping(ctx, nil) - require.NoError(t, err) - - // Ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(ctx) }() - require.NoError(t, err) - err = client2.Ping(ctx, nil) - require.NoError(t, err) -} - -func Load(t *testing.T) { - client, err := mongocommon.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // Insert one record into each db in the source - // They must later show up in the target (in a single custom-named db) - - originalDB1 := client.Database("original_db_1") - err = originalDB1.CreateCollection(ctx, "grass_pokemon") - require.NoError(t, err) - originalDB2 := client.Database("original_db_2") - err = originalDB2.CreateCollection(ctx, "fire_pokemon") - require.NoError(t, err) - - srcGrassColl := originalDB1.Collection("grass_pokemon") - srcFireColl := originalDB2.Collection("fire_pokemon") - - bulbasaur := bson.D{{ - Key: "Name", - Value: "Bulbasaur", - }} - charmander := bson.D{{ - Key: "Name", - Value: "Charmander", - }} - - _, err = srcGrassColl.InsertOne(ctx, bulbasaur) - require.NoError(t, err) - _, err = srcFireColl.InsertOne(ctx, charmander) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // Start worker - - transfer := server.Transfer{ - Type: abstract.TransferTypeSnapshotAndIncrement, - Src: &Source, - Dst: &Target, - ID: helpers.TransferID, - } - - err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // Add data to existing collections for replication - - ivysaur := bson.D{{ - Key: "Name", - Value: "Ivysaur", - }} - charmeleon := bson.D{{ - Key: "Name", - Value: "Charmeleon", - }} - - _, err = srcGrassColl.InsertOne(ctx, ivysaur) - require.NoError(t, err) - _, err = srcFireColl.InsertOne(ctx, charmeleon) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // Wait for documents to appear in source - - docCount := map[string]int64{} - expectedCtGrass := int64(2) - expectedCtFire := int64(2) - for ct, lim := 1, 14; ct <= lim; ct++ { - docCount["grass_pokemon"], err = srcGrassColl.CountDocuments(ctx, bson.D{}) - require.NoError(t, err) - docCount["fire_pokemon"], err = srcFireColl.CountDocuments(ctx, bson.D{}) - require.NoError(t, err) - if docCount["grass_pokemon"] == expectedCtGrass && docCount["fire_pokemon"] == expectedCtFire { - break - } - time.Sleep(3 * time.Second) - } - - require.Equal(t, expectedCtGrass, docCount["grass_pokemon"], "Wrong doc count in grass_pokemon in source") - require.Equal(t, expectedCtFire, docCount["fire_pokemon"], "Wrong doc count in fire_pokemon in source") - - //------------------------------------------------------------------------------------ - // Check results - - targetClient, err := mongocommon.Connect(ctx, Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - renamedDB := targetClient.Database(targetDBName) - - trgGrassColl := renamedDB.Collection("grass_pokemon") - trgFireColl := renamedDB.Collection("fire_pokemon") - - docCount = map[string]int64{} - // Wait for documents to appear in target - for ct, lim := 1, 14; ct <= lim; ct++ { - docCount["grass_pokemon"], err = trgGrassColl.CountDocuments(ctx, bson.D{}) - require.NoError(t, err) - docCount["fire_pokemon"], err = trgFireColl.CountDocuments(ctx, bson.D{}) - require.NoError(t, err) - if docCount["grass_pokemon"] == expectedCtGrass && docCount["fire_pokemon"] == expectedCtFire { - break - } - time.Sleep(3 * time.Second) - } - - require.Equal(t, expectedCtGrass, docCount["grass_pokemon"], "Wrong doc count in grass_pokemon in target") - require.Equal(t, expectedCtFire, docCount["fire_pokemon"], "Wrong doc count in fire_pokemon in target") - - // Check that data have appeared in target - var docResult bson.M - err = trgGrassColl.FindOne(ctx, bulbasaur).Decode(&docResult) - require.NoError(t, err, "No Bulbasaur in target :(") - err = trgGrassColl.FindOne(ctx, ivysaur).Decode(&docResult) - require.NoError(t, err, "No Ivysaur in target :(") - err = trgFireColl.FindOne(ctx, charmander).Decode(&docResult) - require.NoError(t, err, "No Charmander in target :(") - err = trgFireColl.FindOne(ctx, charmeleon).Decode(&docResult) - require.NoError(t, err, "No Charmeleon in target :(") - - // Both original dbs must be absent in target (contents must appear in renamed db) - - originalDB1 = targetClient.Database("original_db_1") - res, err := originalDB1.ListCollectionNames(ctx, bson.D{}) - require.NoError(t, err) - require.Len(t, res, 0) - - originalDB2 = targetClient.Database("original_db_2") - res, err = originalDB2.ListCollectionNames(ctx, bson.D{}) - require.NoError(t, err) - require.Len(t, res, 0) -} diff --git a/tests/e2e/mongo2mongo/replication/check_db_test.go b/tests/e2e/mongo2mongo/replication/check_db_test.go deleted file mode 100644 index 26761934..00000000 --- a/tests/e2e/mongo2mongo/replication/check_db_test.go +++ /dev/null @@ -1,385 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "sync" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongodataagent "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = mongodataagent.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongodataagent.MongoCollection{{DatabaseName: "db", CollectionName: "timmyb32r_test"}}, - } - Target = mongodataagent.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongodataagent.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongodataagent.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongodataagent.MongoDestination) (*mongodataagent.MongoClientWrapper, error) { - return mongodataagent.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Load", Load) - t.Run("ReplicationShutdownTest", ReplicationShutdownTest) - t.Run("ReplicationOfDropDatabaseTest", ReplicationOfDropDatabaseTest) - }) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(&Source) - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func Load(t *testing.T) { - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = client.Close(context.Background()) }() - - //------------------------------------------------------------------------------------ - // insert one record - - db := client.Database("db") - defer func() { - // clear collection in the end (for local debug) - _ = db.Collection("timmyb32r_test").Drop(context.Background()) - }() - err = db.CreateCollection(context.Background(), "timmyb32r_test") - require.NoError(t, err) - - coll := db.Collection("timmyb32r_test") - - type Trainer struct { - Name string - Age int - City string - } - - _, err = coll.InsertOne(context.Background(), Trainer{"a", 1, "aa"}) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - transfer := server.Transfer{ - Type: abstract.TransferTypeSnapshotAndIncrement, - Src: &Source, - Dst: &Target, - ID: helpers.TransferID, - } - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // replicate one record - - _, err = coll.InsertOne(context.Background(), Trainer{"b", 2, "bb"}) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // check results - - require.NoError(t, helpers.WaitEqualRowsCount(t, "db", "timmyb32r_test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -// define mock sinker to report error -var MockSinkerError = xerrors.New("You picked the wrong house, fool!") - -type mockSinker struct { - batchesTillErr int -} - -func newMockSinker(batchesTillErr int) *mockSinker { - return &mockSinker{ - batchesTillErr: batchesTillErr, - } -} - -func (m *mockSinker) dec() { - m.batchesTillErr-- -} - -func (m *mockSinker) Close() error { - return nil -} - -func (m *mockSinker) Push(input []abstract.ChangeItem) error { - defer m.dec() - if m.batchesTillErr <= 1 { - return MockSinkerError - } - return nil -} - -func ReplicationShutdownTest(t *testing.T) { - ctx := context.Background() - - logger.Log.Info("Connect to mongo source database") - clientSource, err := mongodataagent.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = clientSource.Close(context.Background()) }() - - type Parquet struct{ X, Y int } - - logger.Log.Info("Prepare mongo source database") - db, collection := "db100500", "shutdowntest" - dbSource := clientSource.Database(db) - collectionSource := dbSource.Collection(collection) - defer func() { - _ = collectionSource.Drop(context.Background()) - }() - _, err = collectionSource.InsertOne(context.Background(), Parquet{X: 5, Y: 10}) - require.NoError(t, err) - - slotID := "shutdowntransfer" - logger.Log.Info("Specify replication parameters") - source := Source - source.Collections = []mongodataagent.MongoCollection{{DatabaseName: db, CollectionName: collection}} - source.SlotID = slotID - transfer := server.Transfer{ - Type: abstract.TransferTypeIncrementOnly, - Src: &source, - Dst: &server.MockDestination{ - SinkerFactory: func() abstract.Sinker { - return newMockSinker(3) - }, - }, - ID: slotID, - } - - logger.Log.Info("Activate transfer") - err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - logger.Log.Info("Start local worker for activation") - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - errChan := make(chan error, 1) - var waitForLocalWorker sync.WaitGroup - waitForLocalWorker.Add(1) - go func() { - waitForLocalWorker.Done() - errChan <- localWorker.Run() // like .Start(), but we in control for processing error in test - }() - logger.Log.Info("Wait for worker to start") - waitForLocalWorker.Wait() - - logger.Log.Info("Add some documents to make replication busy") - for round := 0; round < 20; round++ { - select { - case err := <-errChan: - require.ErrorIs(t, err, MockSinkerError) - return - default: - } - parquetAmount := 3 - var parquetList []interface{} - for i := 0; i < parquetAmount; i++ { - parquetList = append(parquetList, Parquet{X: 2 * (33 + parquetAmount - i), Y: 10 * i}) - } - insertManyRes, err := collectionSource.InsertMany(context.Background(), parquetList) - require.NoError(t, err) - require.Equal(t, len(insertManyRes.InsertedIDs), parquetAmount, "Amount of inserted documents didn't match requested amount") - time.Sleep(1 * time.Second) // every 1 second sinker accepts new batch - } - - logger.Log.Info("Wait for appropriate error on replication") - tmr := time.NewTimer(5 * time.Second) - select { - case err := <-errChan: - require.ErrorIs(t, err, MockSinkerError) - case <-tmr.C: - logger.Log.Error("Too long no shutdown! Replication hanged on deadlock (possibly)") - t.Fail() - } -} - -func ReplicationOfDropDatabaseTest(t *testing.T) { - t.Run("PerDatabase", func(t *testing.T) { - ReplicationOfDropDatabaseFromReplSourceTest(t, mongodataagent.MongoReplicationSourcePerDatabase) - }) - t.Run("Oplog", func(t *testing.T) { - ReplicationOfDropDatabaseFromReplSourceTest(t, mongodataagent.MongoReplicationSourceOplog) - }) -} - -func ReplicationOfDropDatabaseFromReplSourceTest(t *testing.T, replSource mongodataagent.MongoReplicationSource) { - logger.Log.Info("Checking that dropping collection in source is replicated in target") - logger.Log.Infof("Replication source: %s", replSource) - ctx := context.Background() - - logger.Log.Info("Connect to mongo source database") - clientSource, err := mongodataagent.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = clientSource.Close(context.Background()) }() - - logger.Log.Info("Connect to mongo target database") - clientTarget, err := MakeDstClient(&Target) - require.NoError(t, err) - defer func() { _ = clientTarget.Close(context.Background()) }() - - logger.Log.Info("Prepare mongo source database") - db, collection := "db_that_will_die", "shutdowntest" - dbSource := clientSource.Database(db) - dbTarget := clientTarget.Database(db) - collectionSource := dbSource.Collection(collection) - defer func() { - _ = collectionSource.Drop(context.Background()) - }() - logger.Log.Infof("Drop database '%s' on target if exists before test", db) - err = dbTarget.Drop(context.Background()) - require.NoError(t, err) - - logger.Log.Infof("Insert document in database '%s' collection '%s' in order to create db and subscribe for changes", db, collection) - _, err = collectionSource.InsertOne(context.Background(), struct{ Val int }{Val: 9}) - require.NoError(t, err) - - srcList, err := clientSource.ListDatabaseNames(ctx, bson.D{{Key: "name", Value: db}}) - require.NoError(t, err) - require.Len(t, srcList, 1, "Database should exist before replication start") - - slotID := "dropdatabase" - logger.Log.Info("Specify replication parameters") - source := Source - source.Collections = []mongodataagent.MongoCollection{{DatabaseName: db, CollectionName: collection}} - source.SlotID = slotID - source.ReplicationSource = replSource - transfer := server.Transfer{ - Type: abstract.TransferTypeIncrementOnly, - Src: &source, - Dst: &Target, - ID: slotID, - } - - logger.Log.Info("Activate transfer") - err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - logger.Log.Info("Start local worker for activation") - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer func(localWorker *local.LocalWorker) { - _ = localWorker.Stop() - }(localWorker) - - time.Sleep(2 * time.Second) - - logger.Log.Infof("Insert another document in database '%s' collection '%s'", db, collection) - _, err = collectionSource.InsertOne(context.Background(), struct{ Val int }{Val: 1}) - require.NoError(t, err) - - logger.Log.Info("Wait for db creation on target") - for retryCount, maxRetryCount := 1, 14; retryCount <= maxRetryCount; retryCount++ { - logger.Log.Infof("Attempt %d of %d for database '%s' to appear", retryCount, maxRetryCount, db) - list, err := clientTarget.ListDatabaseNames(ctx, bson.D{{Key: "name", Value: db}}) - require.NoError(t, err) - if len(list) == 1 { - logger.Log.Infof("Database '%s' appeared successfully", db) - break - } - if retryCount == maxRetryCount { - require.Failf(t, "Didn't wait until database '%s' appear", db) - } - time.Sleep(3 * time.Second) - } - - logger.Log.Infof("Drop database %s", db) - err = dbSource.Drop(context.Background()) - require.NoError(t, err) - - time.Sleep(3 * time.Second) - - logger.Log.Info("Wait until database will perish from destination") - - for retryCount, maxRetryCount := 1, 14; retryCount <= maxRetryCount; retryCount++ { - logger.Log.Infof("Attempt %d of %d for database '%s' to drop on target", retryCount, maxRetryCount, db) - list, err := clientTarget.ListDatabaseNames(ctx, bson.D{{Key: "name", Value: db}}) - require.NoError(t, err) - if len(list) == 0 { - logger.Log.Infof("Database '%s' dropped successfully", db) - break - } - if retryCount == maxRetryCount { - require.Failf(t, "Database '%s' should be dropped on target during replication", db) - } - time.Sleep(3 * time.Second) - } -} diff --git a/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go b/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go deleted file mode 100644 index 535af09e..00000000 --- a/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package replication_filter_test - -import ( - "context" - "math/rand" - "os" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongodataagent "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -// creates source from environment settings/recipe -func sourceFromConfig() (*mongodataagent.MongoSource, error) { - srcPort, err := strconv.Atoi(os.Getenv("MONGO_LOCAL_PORT")) - if err != nil { - return nil, err - } - ret := new(mongodataagent.MongoSource) - ret.Hosts = []string{"localhost"} - ret.Port = srcPort - ret.User = os.Getenv("MONGO_LOCAL_USER") - ret.Password = server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")) - ret.WithDefaults() - return ret, nil -} - -func targetFromConfig() (*mongodataagent.MongoDestination, error) { - trgPort, err := strconv.Atoi(os.Getenv("DB0_MONGO_LOCAL_PORT")) - if err != nil { - return nil, err - } - ret := new(mongodataagent.MongoDestination) - ret.Hosts = []string{"localhost"} - ret.Port = trgPort - ret.User = os.Getenv("DB0_MONGO_LOCAL_USER") - ret.Password = server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")) - ret.Cleanup = server.Drop - return ret, nil -} - -func makeTransfer(id string, source *mongodataagent.MongoSource, target *mongodataagent.MongoDestination) *server.Transfer { - source.SlotID = id // set slot ID in order to get valid cluster time on ActivateDelivery - - transfer := new(server.Transfer) - transfer.Type = abstract.TransferTypeIncrementOnly - transfer.Src = source - transfer.Dst = target - transfer.ID = id - transfer.WithDefault() - transfer.FillDependentFields() - return transfer -} - -func TestGroup(t *testing.T) { - sourcePort, err := strconv.Atoi(os.Getenv("MONGO_LOCAL_PORT")) - require.NoError(t, err) - targetPort, err := strconv.Atoi(os.Getenv("DB0_MONGO_LOCAL_PORT")) - require.NoError(t, err) - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: sourcePort}, - helpers.LabeledPort{Label: "Mongo target", Port: targetPort}, - )) - }() - - _ = os.Setenv("YC", "1") // to not go to vanga - - dpconfig.Cloud = &dpconfig.CloudConfig{} - - t.Run("Empty Collection List Means Include All", testEmptyCollectionListIncludesAll) - t.Run("Include All Collections Test", testCollectionFilterIncludeWholeDB) - t.Run("Empty Set Collections Test", testCollectionFilterAllIncludesExcluded) - t.Run("Exclude Star Wins Include Star", testCollectionFilterWholeDBExcludedExcludesCollection) -} - -func testCollectionFilterIncludeWholeDB(t *testing.T) { - t.Parallel() - - src, err := sourceFromConfig() - require.NoError(t, err) - tgt, err := targetFromConfig() - require.NoError(t, err) - - src.Collections = []mongodataagent.MongoCollection{ - {DatabaseName: "db1", CollectionName: "*"}, - } - - transfer := makeTransfer("transfer1", src, tgt) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - if strings.Contains(err.Error(), "replication") { - require.EqualError(t, err, "Failed in accordance with configuration: Some tables whose replication was requested are missing in the source database. Include directives with no matching tables: [db1.*]") - } else { - require.EqualError(t, err, "Unable to find any tables") - } -} - -func testCollectionFilterAllIncludesExcluded(t *testing.T) { - t.Parallel() - - src, err := sourceFromConfig() - require.NoError(t, err) - tgt, err := targetFromConfig() - require.NoError(t, err) - - src.Collections = []mongodataagent.MongoCollection{ - {DatabaseName: "db1", CollectionName: "coll1"}, - {DatabaseName: "db1", CollectionName: "coll2"}, - {DatabaseName: "db2", CollectionName: "A"}, - {DatabaseName: "db2", CollectionName: "B"}, - } - // exclude elides all included collections - src.ExcludedCollections = []mongodataagent.MongoCollection{ - {DatabaseName: "db2", CollectionName: "B"}, - {DatabaseName: "db2", CollectionName: "C"}, - {DatabaseName: "db1", CollectionName: "coll3"}, - {DatabaseName: "db1", CollectionName: "coll1"}, - {DatabaseName: "db2", CollectionName: "A"}, - {DatabaseName: "db1", CollectionName: "coll2"}, - } - - logger.Log.Info("start replication") - transfer := makeTransfer("transfer2", src, tgt) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.Error(t, err) - if strings.Contains(err.Error(), "replication") { - require.Contains(t, err.Error(), "Failed in accordance with configuration: Some tables whose replication was requested are missing in the source database. Include directives with no matching tables:") - require.Contains(t, err.Error(), "db1.coll1") - require.Contains(t, err.Error(), "db1.coll2") - require.Contains(t, err.Error(), "db2.A") - require.Contains(t, err.Error(), "db2.B") - } else { - require.Contains(t, err.Error(), "Unable to find any tables") - } -} - -func testEmptyCollectionListIncludesAll(t *testing.T) { - logger.Log.Warn("Waring -- this test can NOT be run in parallel") - - src, err := sourceFromConfig() - require.NoError(t, err) - tgt, err := targetFromConfig() - require.NoError(t, err) - - srcClient, err := mongodataagent.Connect(context.Background(), src.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - ldb, err := srcClient.ListDatabases(context.Background(), bson.D{}) - require.NoError(t, err) - for _, db := range ldb.Databases { - _ = srcClient.Database(db.Name).Drop(context.Background()) - } - - db1, db2 := "A", "B" - coll1, coll2 := "C", "D" - type PingData struct{ version int } - - insertRandomDocuments := func() { - t.Helper() - for _, db := range []string{db1, db2} { - for _, coll := range []string{coll1, coll2} { - _, err = srcClient.Database(db).Collection(coll).InsertOne(context.Background(), PingData{version: rand.Int()}) - if err != nil { - require.NoError(t, err, "Couldn't insert into database one item. Producing goroutine stops.") - return - } - } - } - } - - logger.Log.Info("Create databases to save cluster time") - insertRandomDocuments() - - logger.Log.Info("Create and activate transfer") - transfer := makeTransfer("transfer3", src, tgt) - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - logger.Log.Info("Insert documents after activation") - insertRandomDocuments() - - logger.Log.Info("Start replication worker") - replicationWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, solomon.NewRegistry(nil), logger.Log) - errChan := make(chan error, 1) - var wgWaitForStart sync.WaitGroup - wgWaitForStart.Add(1) - go func() { - wgWaitForStart.Done() - errChan <- replicationWorker.Run() - }() - logger.Log.Info("wait for goroutine to start") - wgWaitForStart.Wait() - - logger.Log.Info("wait for appropriate error from replication") - timeToWait := 10 * time.Second - timer := time.NewTimer(timeToWait) - select { - case err := <-errChan: - require.NoError(t, err, "Should be no error") - case <-timer.C: - logger.Log.Infof("OK, replication didn't fail within time interval '%v'", timeToWait) - break - } -} - -func testCollectionFilterWholeDBExcludedExcludesCollection(t *testing.T) { - t.Parallel() - - src, err := sourceFromConfig() - require.NoError(t, err) - tgt, err := targetFromConfig() - require.NoError(t, err) - - src.Collections = []mongodataagent.MongoCollection{ - {DatabaseName: "db1", CollectionName: "coll1"}, - } - // exclude elides previous collections - src.ExcludedCollections = []mongodataagent.MongoCollection{ - {DatabaseName: "db1", CollectionName: "*"}, - } - - logger.Log.Info("start replication") - transfer := makeTransfer("transfer4", src, tgt) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - if strings.Contains(err.Error(), "replication") { - require.EqualError(t, err, "Failed in accordance with configuration: Some tables whose replication was requested are missing in the source database. Include directives with no matching tables: [db1.coll1]") - } else { - require.EqualError(t, err, "Unable to find any tables") - } -} diff --git a/tests/e2e/mongo2mongo/replication_update_model/check_db_test.go b/tests/e2e/mongo2mongo/replication_update_model/check_db_test.go deleted file mode 100644 index b98e9df9..00000000 --- a/tests/e2e/mongo2mongo/replication_update_model/check_db_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongodataagent "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = mongodataagent.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongodataagent.MongoCollection{{DatabaseName: "db", CollectionName: "timmyb32r_test"}}, - ReplicationSource: mongodataagent.MongoReplicationSourcePerDatabaseUpdateDocument, - } - Target = mongodataagent.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongodataagent.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongodataagent.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongodataagent.MongoDestination) (*mongodataagent.MongoClientWrapper, error) { - return mongodataagent.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Load", Load) - }) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(&Source) - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func Load(t *testing.T) { - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = client.Close(context.Background()) }() - - //------------------------------------------------------------------------------------ - // insert one record - - db := client.Database("db") - defer func() { - // clear collection in the end (for local debug) - _ = db.Collection("timmyb32r_test").Drop(context.Background()) - }() - err = db.CreateCollection(context.Background(), "timmyb32r_test") - require.NoError(t, err) - - coll := db.Collection("timmyb32r_test") - - type Trainer struct { - Name string - Age int - City string - } - - _, err = coll.InsertOne(context.Background(), Trainer{"a", 1, "aa"}) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - transfer := server.Transfer{ - Type: abstract.TransferTypeSnapshotAndIncrement, - Src: &Source, - Dst: &Target, - ID: helpers.TransferID, - } - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // replicate one record - - _, err = coll.InsertOne(context.Background(), Trainer{"b", 2, "bb"}) - require.NoError(t, err) - - _, err = coll.UpdateOne(context.Background(), bson.D{{Key: "name", Value: "b"}}, bson.D{{Key: "$set", Value: bson.D{{Key: "name", Value: "bb"}, {Key: "age", Value: 21}}}}) - require.NoError(t, err) - - _, err = coll.InsertOne(context.Background(), Trainer{"c", 2, "aa"}) - require.NoError(t, err) - _, err = coll.UpdateOne(context.Background(), bson.D{{Key: "name", Value: "c"}}, bson.D{{Key: "$set", Value: bson.D{{Key: "name", Value: "cc"}, {Key: "age", Value: 21}}}}) - require.NoError(t, err) - _, err = coll.UpdateOne(context.Background(), bson.D{{Key: "name", Value: "cc"}}, bson.D{{Key: "$set", Value: bson.D{{Key: "name", Value: "ccc"}, {Key: "age", Value: 21}}}}) - require.NoError(t, err) - - _, err = coll.UpdateMany(context.Background(), bson.M{"age": bson.M{"$lte": 21}}, bson.D{{Key: "$set", Value: bson.D{{Key: "City", Value: "Gotham"}}}}) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // check results - - require.NoError(t, helpers.WaitEqualRowsCount(t, "db", "timmyb32r_test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go b/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go deleted file mode 100644 index b1c19efa..00000000 --- a/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go +++ /dev/null @@ -1,382 +0,0 @@ -package replication - -// Author: kry127 -// This test check replication correct in case of intensive RPS on mongo collection -// Lifetime span of object in database is less than second, requests per minute should be approx 45MB per second - -// expected statistics (25.08.2021) -// startrek:PRIMARY> db.onetimeJobs.stats() -//{ -// "ns" : "startrek.onetimeJobs", -// "size" : 1036123603, -// "count" : 256612, -// "avgObjSize" : 4037, -// "storageSize" : 509001728, -// ... -// } - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongostorage "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/e2e/mongo2mongo/rps" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga -} - -func TestGroup(t *testing.T) { - t.Skip("TM-5255 temporary skip tests") - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("RpsTest", RpsTest) - }) -} - -const ( - slotIDAkaTransferID = "dttintensiveupdatingcollection" - DB = "startrek" // tribute to StarTrek database - Collection = "onetimeJobs" // tribute to StarTrek collection -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = &mongostorage.MongoSource{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("MONGO_LOCAL_PORT"), - User: os.Getenv("MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("MONGO_LOCAL_PASSWORD")), - Collections: []mongostorage.MongoCollection{ - {DatabaseName: DB, CollectionName: Collection}, - }, - SlotID: slotIDAkaTransferID, - } - Target = mongostorage.MongoDestination{ - Hosts: []string{"localhost"}, - Port: helpers.GetIntFromEnv("DB0_MONGO_LOCAL_PORT"), - User: os.Getenv("DB0_MONGO_LOCAL_USER"), - Password: server.SecretString(os.Getenv("DB0_MONGO_LOCAL_PASSWORD")), - Cleanup: server.Drop, - } -) - -func init() { - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongostorage.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongostorage.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongostorage.MongoDestination) (*mongostorage.MongoClientWrapper, error) { - return mongostorage.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(Source) - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func clearSrc(t *testing.T, client *mongostorage.MongoClientWrapper) { - t.Helper() - var err error - - db := client.Database(DB) - _ = db.Collection(Collection).Drop(context.Background()) - err = db.CreateCollection(context.Background(), Collection) - require.NoError(t, err) -} - -type RpsTestParameters struct { - SrcParamGen func() *mongostorage.MongoSource -} - -func RpsTest(t *testing.T) { - Source.WithDefaults() - - for testName, testParam := range map[string]RpsTestParameters{ - "PerDatabase": {SrcParamGen: func() *mongostorage.MongoSource { - src := *Source - src.ReplicationSource = mongostorage.MongoReplicationSourcePerDatabase - return &src - }}, - "PerDatabaseFullDocument": {SrcParamGen: func() *mongostorage.MongoSource { - src := *Source - src.ReplicationSource = mongostorage.MongoReplicationSourcePerDatabaseFullDocument - return &src - }}, - "PerDatabaseUpdateDocument": {SrcParamGen: func() *mongostorage.MongoSource { - src := *Source - src.ReplicationSource = mongostorage.MongoReplicationSourcePerDatabaseUpdateDocument - return &src - }}, - "Oplog": {SrcParamGen: func() *mongostorage.MongoSource { - src := *Source - src.ReplicationSource = mongostorage.MongoReplicationSourceOplog - return &src - }}, - } { - t.Run(testName, RpsTestFactory(testParam)) - } -} - -func RpsTestFactory(testParameters RpsTestParameters) func(t *testing.T) { - return func(t *testing.T) { - ctx := context.Background() - - clientSource, err := mongostorage.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = clientSource.Close(context.Background()) }() - - // recreate collections on source - clearSrc(t, clientSource) - dbSource := clientSource.Database(DB) - collectionSource := dbSource.Collection(Collection) - - mongoSource := testParameters.SrcParamGen() - transfer := helpers.MakeTransfer(helpers.TransferID, mongoSource, &Target, TransferType) - - // activate transfer - err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - // start local worker for activation - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - errChan := make(chan error, 1) - go func() { - errChan <- localWorker.Run() // like .Start(), but we in control for processing error in test - }() - - dstStorage, err := mongostorage.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - // configure desired RPS - rpsModel := rps.NewRpsModel(ctx, &rps.RpsCallbacks{ - OnDelete: func(ctx context.Context, key string) { - filter := bson.D{{Key: "_id", Value: key}} - result, err := collectionSource.DeleteOne(ctx, filter, nil) - require.NoError(t, err) - require.Equal(t, int64(1), result.DeletedCount) - }, - OnCreate: func(ctx context.Context, entity rps.KV) { - _, err := collectionSource.InsertOne(ctx, entity) - require.NoError(t, err) - }, - OnUpdate: func(ctx context.Context, previous rps.KV, actual rps.KV) { - opts := options.Update() - filter := bson.D{{Key: "_id", Value: previous.Key}} - update := bson.D{{Key: "$set", Value: bson.D{{Key: "document", Value: actual.Document}}}} - result, err := collectionSource.UpdateOne(ctx, filter, update, opts) - require.NoError(t, err) - require.Equal(t, int64(1), result.ModifiedCount) - }, - OnReplace: func(ctx context.Context, previous rps.KV, actual rps.KV) { - opts := options.Replace() - filter := bson.D{{Key: "_id", Value: previous.Key}} - result, err := collectionSource.ReplaceOne(ctx, filter, actual, opts) - require.NoError(t, err) - require.Equal(t, int64(1), result.ModifiedCount) - }, - Tick: func(ctx context.Context, tickId int, model *rps.RpsModel) bool { - normalMode := &rps.RpsSpec{ - DeleteCount: 100, - CreateCount: 100, - UpdateCount: 50, - ReplaceCount: 0, - KVConstructor: func() rps.KV { - return rps.GenerateKV(800, 2000) - }, - Delay: 100 * time.Millisecond, - } - logger.Log.Info(fmt.Sprintf("Delay iteration %d, in: %d, out: %d", tickId, len(model.Persistent), len(model.NonPersistent))) - - var currentSpec *rps.RpsSpec // when changed from nil, reconfigure happens - message := "" - switch tickId { - case 0: - message = "create initial entries" - currentSpec = &rps.RpsSpec{ - DeleteCount: 0, - CreateCount: 4000, - UpdateCount: 0, - ReplaceCount: 0, - KVConstructor: func() rps.KV { - return rps.GenerateKV(200, 200) - }, - Delay: 5 * time.Second, - } - case 1: - message = "then equalize insert and delete rates with normal mode" - currentSpec = normalMode - case 9: - message = "make outlier in one Delay with heavy documents" - currentSpec = &rps.RpsSpec{ - DeleteCount: 0, - CreateCount: 40, - UpdateCount: 20, - ReplaceCount: 0, - KVConstructor: func() rps.KV { - return rps.GenerateKV(1000, 5000) - }, - Delay: 100 * time.Millisecond, - } - case 10: - message = "back to normal mode" - currentSpec = normalMode - case 20: - message = "intensify update rate up to 800 requests per 10 millisecond == 80000 RPS" - // more intensive by time pushes - currentSpec = &rps.RpsSpec{ - DeleteCount: 200, - CreateCount: 300, - UpdateCount: 200, - ReplaceCount: 100, - KVConstructor: func() rps.KV { - return rps.GenerateKV(300, 400) - }, - Delay: 10 * time.Millisecond, - } - case 30: - message = "maximum intensity" - // more intensive by time pushes - currentSpec = &rps.RpsSpec{ - DeleteCount: 500, - CreateCount: 600, - UpdateCount: 900, - ReplaceCount: 300, - KVConstructor: func() rps.KV { - return rps.GenerateKV(50, 50) - }, - Delay: 2 * time.Millisecond, - } - case 40: - // stop generation on last Delay - logger.Log.Info("RPS stopping", log.Int("tickId", tickId)) - return false - } - - if currentSpec != nil { - logger.Log.Info("RPS Reconfigure", log.String("message", message), log.Any("config", currentSpec)) - model.SetSpec(currentSpec) - } - return true - }, - }) - - logger.Log.Info("Start RPS generator") - rpsModelDone := make(chan struct{}) - go func() { - defer rpsModel.Close() - defer close(rpsModelDone) - rpsModel.Start() - }() - select { - case <-rpsModelDone: - break - case <-ctx.Done(): - t.Fatal("Couldn't wait for RPS to close") - } - - // wait for replication to catch up lag - rowCount := uint64(len(rpsModel.Persistent)) - tryingsCount := 30 - tries := 0 - for tries = 0; tries < tryingsCount; tries++ { - td := abstract.TableID{Namespace: DB, Name: Collection} - dstTableSize, err := dstStorage.ExactTableRowsCount(td) // TODO(@kry127;@timmyb32r) TM2409 change on GetRowsCount() - require.NoError(t, err) - - t.Logf("Table: %s, count rows. Expected: %d, actual: %d", td.Fqtn(), rowCount, dstTableSize) - if dstTableSize == rowCount { - break - } - time.Sleep(time.Second) - } - if tries == tryingsCount { - // nevermind, further test is unpassable - t.Logf("Tries are over: %d out of %d", tries, tryingsCount) - } - - // wait a little bit (push batch delay is recomended) - time.Sleep(3 * mongostorage.DefaultBatchFlushInterval) - - // stop worker - logger.Log.Info("Stop local worker") - err = localWorker.Stop() - require.NoError(t, err) - - // wait for appropriate error from replication - select { - case err := <-errChan: - require.NoError(t, err) - case <-ctx.Done(): - t.Fatalf("Couldn't wait until replication ended: %v", ctx.Err()) - } - - // make connection to the target - clientTarget, err := mongostorage.Connect(ctx, Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - dbTarget := clientTarget.Database(DB) - collectionTarget := dbTarget.Collection(Collection) - - // check that 'persistent' is present in source and target, and they values are equal - // and check that 'not persistent' neither on source nor target - logger.Log.Info("Validation of source and target databases") - for fromWhere, coll := range map[string]*mongo.Collection{"source": collectionSource, "target": collectionTarget} { - rpsModel.CheckValid(t, ctx, fromWhere, coll) - } - - logger.Log.Info("All values validated, tear down") - } -} diff --git a/tests/e2e/mongo2mongo/rps/rps.go b/tests/e2e/mongo2mongo/rps/rps.go deleted file mode 100644 index 761844d0..00000000 --- a/tests/e2e/mongo2mongo/rps/rps.go +++ /dev/null @@ -1,309 +0,0 @@ -package rps - -// Author: kry127 -// rps.go -- common file for RPS tests - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/randutil" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" -) - -// KV is Key-Document object for key-value database (Mongo or in-memory) -type KV struct { - Key string `bson:"_id"` - Document interface{} `bson:"document"` -} - -func stringOrDots(input string) string { - if len(input) < 64 { - return input - } else { - return input[:61] + "..." - } -} - -func (k *KV) String() string { - return fmt.Sprintf("KV{%s: %s}", stringOrDots(k.Key), stringOrDots(fmt.Sprintf("%v", k.Document))) -} - -// GenerateKV produces random Key-Document object with key and value as -// string of `keysize` and `valsize` length respectfully -func GenerateKV(keysize, valsize int) KV { - return KV{ - Key: randutil.GenerateAlphanumericString(keysize), - Document: bson.D{{Key: "value", Value: randutil.GenerateAlphanumericString(valsize)}}, - } -} - -// RpsSpec defines specification of requests count per Delay -// applied in order: delete, create, update -type RpsSpec struct { - DeleteCount, CreateCount, UpdateCount, ReplaceCount uint - KVConstructor func() KV `json:"-"` - Delay time.Duration -} - -// RpsNotifier contains callbacks for different events of RPS generator -type RpsCallbacks struct { - Tick func(ctx context.Context, index int, rps *RpsModel) bool - OnDelete func(ctx context.Context, key string) - OnCreate func(ctx context.Context, entity KV) - OnUpdate func(ctx context.Context, previous KV, actual KV) - OnReplace func(ctx context.Context, previous KV, actual KV) -} - -var ( - // default spec immediately gives control to user - defaultRpsSpec = RpsSpec{ - KVConstructor: func() KV { - return GenerateKV(10, 10) - }, - Delay: 0 * time.Millisecond, - } - defaultRpsNotifier = RpsCallbacks{ - Tick: func(ctx context.Context, index int, model *RpsModel) bool { return false }, - OnDelete: func(ctx context.Context, key string) {}, - OnCreate: func(ctx context.Context, entity KV) {}, - OnUpdate: func(ctx context.Context, previous KV, actual KV) {}, - OnReplace: func(ctx context.Context, previous KV, actual KV) {}, - } -) - -type historyDescription struct { - optype string // insert, update, delete - opkey string - value, oldValue *KV -} - -func (h *historyDescription) String() string { - switch h.optype { - case "insert": - return fmt.Sprintf("%s %s", h.optype, h.value.String()) - case "update": - return fmt.Sprintf("%s %s=>%s", h.optype, h.oldValue.String(), h.value.String()) - case "delete": - return fmt.Sprintf("%s %s", h.optype, stringOrDots(h.opkey)) - default: - return fmt.Sprintf("%s %s %s %s", h.optype, stringOrDots(h.opkey), h.value.String(), h.oldValue.String()) - } -} - -func HistoryToString(history []historyDescription) string { - var hd []string - for _, entry := range history { - hd = append(hd, entry.String()) - } - return strings.Join(hd, "\n") -} - -// RpsModel is a: -// 1. rate-limiter for requests -// 2. in-mem KV storage -// -// Anyone can access in-memory Persistent state, NonPersistent(deleted) state, and change history as ModelHistory -type RpsModel struct { - // we will use this for generating RPS: every 'period' milliseconds the 'timer' hits - timer *time.Timer - specification *RpsSpec - callbacks *RpsCallbacks - ctx context.Context - ctxCancel func() - // we should uniformly distribute documents between this two categories: - Persistent map[string]interface{} // present KV - NonPersistent map[string]struct{} // sometimes deleted KV - ModelHistory map[string][]historyDescription -} - -func (r *RpsModel) Close() { - r.ctxCancel() - r.timer.Stop() -} - -// SetSpec sets RPS specification -func (r *RpsModel) SetSpec(spec *RpsSpec) { - if spec == nil { - spec = &defaultRpsSpec - } - r.specification = spec - - r.timer.Reset(r.specification.Delay) -} - -func (r *RpsModel) CheckValid(t *testing.T, ctx context.Context, label string, coll *mongo.Collection) { - cursor, err := coll.Find(ctx, bson.D{}) - require.NoError(t, err) - hasInCursor := map[string]struct{}{} - for cursor.Next(ctx) { - var kv KV - err := cursor.Decode(&kv) - require.NoError(t, err) - hasInCursor[kv.Key] = struct{}{} - history := r.ModelHistory[kv.Key] - actualVal, persist := r.Persistent[kv.Key] - _, nonPersist := r.NonPersistent[kv.Key] - require.True(t, persist, fmt.Sprintf("Entity with label '%s' should persist in model. History: \n%s\n", label, HistoryToString(history))) - require.False(t, nonPersist, fmt.Sprintf("Entity with label '%s' should not be deleted in model. History: \n%s\n", label, HistoryToString(history))) - require.Equal(t, actualVal, kv.Document, "Values in label '%s' and model should match. History: \n%s\n", label, HistoryToString(history)) - } - // extra check on completeness - for key := range r.Persistent { - _, ok := hasInCursor[key] - require.True(t, ok, fmt.Sprintf("All values inserted in model should be presented in database labeled '%s'", label)) - } - for key := range r.NonPersistent { - _, notOk := hasInCursor[key] - require.False(t, notOk, fmt.Sprintf("All values deleted from model should not be presented in database labeled '%s'", label)) - } -} - -func (r *RpsModel) Start() { - tickIndex := 0 - for { - select { - case <-r.timer.C: - shouldContinue := r.callbacks.Tick(r.ctx, tickIndex, r) - if !shouldContinue { - return - } - tickIndex++ - - // make deletes, inserts and updates - toDelete := r.specification.DeleteCount - for key, oldValue := range r.Persistent { - if toDelete == 0 { - break - } - delete(r.Persistent, key) - r.NonPersistent[key] = struct{}{} - r.callbacks.OnDelete(r.ctx, key) - oldKv := KV{Key: key, Document: oldValue} - r.ModelHistory[key] = append(r.ModelHistory[key], historyDescription{ - optype: "delete", - opkey: key, - value: nil, - oldValue: &oldKv, - }) - toDelete-- - } - - // make inserts and updates - toCreate := r.specification.CreateCount - toReplace := r.specification.ReplaceCount - toUpdate := r.specification.UpdateCount - retryLimit, retryLimitID := 20, 0 - for { - if toCreate == 0 { - break - } - - kv := r.specification.KVConstructor() - if oldValue, ok := r.Persistent[kv.Key]; ok { - // this is an update - if toUpdate > 0 { - r.Persistent[kv.Key] = kv.Document - oldKv := KV{Key: kv.Key, Document: oldValue} - r.callbacks.OnUpdate(r.ctx, oldKv, kv) - r.ModelHistory[kv.Key] = append(r.ModelHistory[kv.Key], historyDescription{ - optype: "update", - opkey: kv.Key, - value: &kv, - oldValue: &oldKv, - }) - toUpdate-- - } else { - retryLimitID++ - if retryLimitID == retryLimit { - // give up on inserting, to many collisions - logger.Log.Warn("Too many collisions on RPS insert", log.Int("RetryLimit", retryLimit)) - break - } - } - } else { - if toCreate > 0 { - // this is an insert - r.Persistent[kv.Key] = kv.Document - delete(r.NonPersistent, kv.Key) - r.callbacks.OnCreate(r.ctx, kv) - r.ModelHistory[kv.Key] = append(r.ModelHistory[kv.Key], historyDescription{ - optype: "insert", - opkey: kv.Key, - value: &kv, - oldValue: nil, - }) - toCreate-- - } else if toReplace > 0 { - // this is an update - for replaceWhatID, replaceWhatValue := range r.Persistent { - r.Persistent[kv.Key] = kv.Document - delete(r.NonPersistent, kv.Key) - delete(r.Persistent, replaceWhatID) - r.NonPersistent[replaceWhatID] = struct{}{} - r.callbacks.OnReplace(r.ctx, KV{Key: replaceWhatID, Document: replaceWhatValue}, kv) - toReplace-- - break - } - } - } - } - - // make the rest of updates - for key, oldValue := range r.Persistent { - if toUpdate == 0 { - break - } - newValue := r.specification.KVConstructor().Document - r.Persistent[key] = newValue - oldKv := KV{Key: key, Document: oldValue} - newKv := KV{Key: key, Document: newValue} - r.callbacks.OnUpdate(r.ctx, oldKv, newKv) - r.ModelHistory[key] = append(r.ModelHistory[key], historyDescription{ - optype: "update", - opkey: key, - value: &newKv, - oldValue: &oldKv, - }) - toUpdate-- - } - - r.timer.Reset(r.specification.Delay) - case <-r.ctx.Done(): - return - } - } -} - -// NewRpsModel creates RPS model -// use initialSpec to set frequency and value of operations -// use onCreate, onDelete and onUpdate from RpsCallbacks to make actions (e.g. with database) -// use Delay in RpsCallbacks to reconfigure RPS in time -func NewRpsModel(ctx context.Context, notifiers *RpsCallbacks) *RpsModel { - newCtx, newCtxCancel := context.WithCancel(ctx) - - if notifiers == nil { - notifiers = &defaultRpsNotifier - } - - r := &RpsModel{ - timer: time.NewTimer(defaultRpsSpec.Delay), - specification: &defaultRpsSpec, - callbacks: notifiers, - ctx: newCtx, - ctxCancel: newCtxCancel, - Persistent: map[string]interface{}{}, - NonPersistent: map[string]struct{}{}, - ModelHistory: map[string][]historyDescription{}, - } - - // start periodic goroutine - return r -} diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db1.yaml b/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db1.yaml deleted file mode 100644 index 29148518..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db1.yaml +++ /dev/null @@ -1,12 +0,0 @@ -envPrefix: "DB1_" -configReplicaSet: - amount: 3 -shards: - amount: 2 - shardReplicaSet: - amount: 3 -postSteps: - createAdminUser: - user: user1 - password: P@ssw0rd1 - authSource: db1 diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db2.yaml b/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db2.yaml deleted file mode 100644 index 9f139b2b..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/db2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -envPrefix: "DB2_" -configReplicaSet: - amount: 2 -shards: - amount: 3 - shardReplicaSet: - amount: 2 -postSteps: - createAdminUser: - user: user2 - password: P@ssw0rd2 - authSource: db2 diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go b/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go deleted file mode 100644 index 5f361c59..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go +++ /dev/null @@ -1,356 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongostorage "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/randutil" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - mongoshardedcluster "github.com/doublecloud/tross/recipe/mongo/pkg/cluster" - "github.com/doublecloud/tross/tests/e2e/mongo2mongo/rps" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga -} - -func TestGroup(t *testing.T) { - t.Skip("TM-5255 temporary skip tests") - - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("RpsTest", RpsTest) - }) -} - -const ( - slotIDAkaTransferID = "dtt_shard_to_shard" - DB = "db1" - Collection = "coll1" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = &mongostorage.MongoSource{ - Hosts: []string{os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: server.SecretString(os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Collections: []mongostorage.MongoCollection{ - {DatabaseName: DB, CollectionName: Collection}, - }, - SlotID: slotIDAkaTransferID, - } - Target = mongostorage.MongoDestination{ - Hosts: []string{os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: server.SecretString(os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Cleanup: server.DisabledCleanup, - } -) - -func init() { - dpconfig.Cloud = &dpconfig.CloudConfig{} - Source.WithDefaults() -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongostorage.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongostorage.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongostorage.MongoDestination) (*mongostorage.MongoClientWrapper, error) { - return mongostorage.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -func ShardSourceCollection(t *testing.T, client *mongostorage.MongoClientWrapper) { - adminDB := client.Database("admin") - - res := adminDB.RunCommand(context.TODO(), - bson.D{ - {Key: "enableSharding", Value: DB}, - }) - require.NoError(t, res.Err()) - - var runCmdResult bson.M - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection)}, - {Key: "key", Value: bson.D{ - {Key: "document.a", Value: "hashed"}, - {Key: "document.b", Value: 1}, - {Key: "document.c", Value: 1}, - }}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) -} - -func ShardTargetCollection(t *testing.T, client *mongostorage.MongoClientWrapper) { - adminDB := client.Database("admin") - - res := adminDB.RunCommand(context.TODO(), - bson.D{ - {Key: "enableSharding", Value: DB}, - }) - require.NoError(t, res.Err()) - - key := bson.D{ - {Key: "document.x", Value: "hashed"}, - {Key: "document.y", Value: 1}, - {Key: "document.z", Value: 1}, - } - - var runCmdResult bson.M - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection)}, - {Key: "key", Value: key}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(Source) - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func clearStorage(t *testing.T, client *mongostorage.MongoClientWrapper) { - t.Helper() - var err error - - db := client.Database(DB) - _ = db.Collection(Collection).Drop(context.Background()) - err = db.CreateCollection(context.Background(), Collection) - require.NoError(t, err) -} - -func RpsTest(t *testing.T) { - for _, rsName := range []mongostorage.MongoReplicationSource{ - mongostorage.MongoReplicationSourcePerDatabaseFullDocument, - mongostorage.MongoReplicationSourcePerDatabaseUpdateDocument, - } { - t.Run(string(rsName), func(t *testing.T) { - RpsTestForRS(t, rsName) - }) - } -} - -func RpsTestForRS(t *testing.T, rs mongostorage.MongoReplicationSource) { - ctx := context.Background() - - clientSource, err := mongostorage.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = clientSource.Close(context.Background()) }() - - // recreate collections on source - clearStorage(t, clientSource) - dbSource := clientSource.Database(DB) - collectionSource := dbSource.Collection(Collection) - - // make connection to the target - clientTarget, err := mongostorage.Connect(ctx, Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - // drop collection on target before sharding - clearStorage(t, clientTarget) - - // shard source - //ShardSourceCollection(t, clientSource) //IS it recipe limitation? - // shard target - ShardTargetCollection(t, clientTarget) - - mongoSource := Source - mongoSource.ReplicationSource = rs - transfer := helpers.MakeTransfer(helpers.TransferID, mongoSource, &Target, TransferType) - - // activate transfer - err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - // start local worker for activation - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - errChan := make(chan error, 1) - go func() { - errChan <- localWorker.Run() // like .Start(), but we in control for processing error in test - }() - - dstStorage, err := mongostorage.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - // configure desired RPS - rpsContext, rpsCancel := context.WithCancel(ctx) - defer rpsCancel() - rpsModel := rps.NewRpsModel(rpsContext, &rps.RpsCallbacks{ - OnCreate: func(ctx context.Context, entity rps.KV) { - _, err := collectionSource.InsertOne(ctx, entity) - require.NoError(t, err) - }, - OnUpdate: func(ctx context.Context, previous rps.KV, actual rps.KV) { - opts := options.Update() - doc, ok := previous.Document.(bson.D) - require.True(t, ok) - filter := bson.D{ - {Key: "_id", Value: previous.Key}, - {Key: "document.a", Value: doc.Map()["a"]}, - {Key: "document.b", Value: doc.Map()["b"]}, - {Key: "document.c", Value: doc.Map()["c"]}, - } - update := bson.D{{Key: "$set", Value: bson.D{{Key: "document", Value: actual.Document}}}} - result, err := collectionSource.UpdateOne(ctx, filter, update, opts) - require.NoError(t, err) - require.Equal(t, int64(1), result.ModifiedCount) - }, - OnReplace: func(ctx context.Context, previous rps.KV, actual rps.KV) { - opts := options.Replace() - filter := bson.D{{Key: "_id", Value: previous.Key}} - result, err := collectionSource.ReplaceOne(ctx, filter, actual, opts) - require.NoError(t, err) - require.Equal(t, int64(1), result.ModifiedCount) - }, - OnDelete: func(ctx context.Context, key string) { - filter := bson.D{{Key: "_id", Value: key}} - result, err := collectionSource.DeleteOne(ctx, filter, nil) - require.NoError(t, err) - require.Equal(t, int64(1), result.DeletedCount) - }, - Tick: func(ctx context.Context, tickId int, model *rps.RpsModel) bool { - if tickId > 12 { - // stop generation on last Delay - logger.Log.Info("RPS stopping", log.Int("tickId", tickId)) - return false - } - return true - }, - }) - - rpsModel.SetSpec(&rps.RpsSpec{ - DeleteCount: 100, - CreateCount: 100, - UpdateCount: 100, - ReplaceCount: 100, - KVConstructor: func() rps.KV { - return rps.KV{ - Key: randutil.GenerateAlphanumericString(16), - Document: bson.D{ - {Key: "a", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "b", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "c", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "x", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "y", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "z", Value: randutil.GenerateAlphanumericString(8)}, - }, - } - }, - Delay: 0, - }) - - logger.Log.Info("Start RPS generator") - rpsModelDone := make(chan struct{}) - go func() { - defer rpsModel.Close() - defer close(rpsModelDone) - rpsModel.Start() - }() - select { - case <-rpsModelDone: - break - case <-ctx.Done(): - t.Fatal("Couldn't wait for RPS to close") - } - - // wait for replication to catch up lag - rowCount := uint64(len(rpsModel.Persistent)) - tryingsCount := 30 - tries := 0 - for tries = 0; tries < tryingsCount; tries++ { - td := abstract.TableID{Namespace: DB, Name: Collection} - dstTableSize, err := dstStorage.ExactTableRowsCount(td) // TODO(@kry127;@timmyb32r) TM2409 change on GetRowsCount() - require.NoError(t, err) - - t.Logf("Table: %s, count rows. Expected: %d, actual: %d", td.Fqtn(), rowCount, dstTableSize) - if dstTableSize == rowCount { - break - } - time.Sleep(time.Second) - } - if tries == tryingsCount { - // nevermind, further test is unpassable - t.Logf("Tries are over: %d out of %d", tries, tryingsCount) - } - - // wait a little bit (push batch delay is recomended) - time.Sleep(3 * mongostorage.DefaultBatchFlushInterval) - - // stop worker - logger.Log.Info("Stop local worker") - err = localWorker.Stop() - require.NoError(t, err) - - // wait for appropriate error from replication - select { - case err := <-errChan: - require.NoError(t, err) - case <-ctx.Done(): - t.Fatalf("Couldn't wait until replication ended: %v", ctx.Err()) - } - - dbTarget := clientTarget.Database(DB) - collectionTarget := dbTarget.Collection(Collection) - - // check that 'persistent' is present in source and target, and they values are equal - // and check that 'not persistent' neither on source nor target - logger.Log.Info("Validation of source and target databases") - for fromWhere, coll := range map[string]*mongo.Collection{"source": collectionSource, "target": collectionTarget} { - rpsModel.CheckValid(t, ctx, fromWhere, coll) - } - - logger.Log.Info("All values validated, tear down") -} diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db1.yaml b/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db1.yaml deleted file mode 100644 index 29148518..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db1.yaml +++ /dev/null @@ -1,12 +0,0 @@ -envPrefix: "DB1_" -configReplicaSet: - amount: 3 -shards: - amount: 2 - shardReplicaSet: - amount: 3 -postSteps: - createAdminUser: - user: user1 - password: P@ssw0rd1 - authSource: db1 diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db2.yaml b/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db2.yaml deleted file mode 100644 index 9f139b2b..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/db2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -envPrefix: "DB2_" -configReplicaSet: - amount: 2 -shards: - amount: 3 - shardReplicaSet: - amount: 2 -postSteps: - createAdminUser: - user: user2 - password: P@ssw0rd2 - authSource: db2 diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go b/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go deleted file mode 100644 index e8a601db..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongodataagent "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - mongoshardedcluster "github.com/doublecloud/tross/recipe/mongo/pkg/cluster" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Load", Load) - }) -} - -const ( - slotIDAkaTransferID = "dtt_shard_to_shard" - DB = "db1" - Collection1 = "coll1" - Collection2 = "coll2" - Collection3 = "coll3" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = &mongodataagent.MongoSource{ - Hosts: []string{os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: server.SecretString(os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Collections: []mongodataagent.MongoCollection{ - {DatabaseName: DB, CollectionName: Collection1}, - {DatabaseName: DB, CollectionName: Collection2}, - {DatabaseName: DB, CollectionName: Collection3}, - }, - SlotID: slotIDAkaTransferID, - } - Target = mongodataagent.MongoDestination{ - Hosts: []string{os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: server.SecretString(os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Cleanup: server.DisabledCleanup, - } -) - -func init() { - dpconfig.Cloud = &dpconfig.CloudConfig{} - Source.WithDefaults() -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongodataagent.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongodataagent.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func ShardTargetCollections(t *testing.T, client *mongodataagent.MongoClientWrapper) { - adminDB := client.Database("admin") - - res := adminDB.RunCommand(context.TODO(), - bson.D{ - {Key: "enableSharding", Value: DB}, - }) - require.NoError(t, res.Err()) - - key1 := bson.D{ - {Key: "_id.x", Value: "hashed"}, - } - - key2 := bson.D{ - {Key: "_id.x", Value: "hashed"}, - {Key: "city", Value: 1}, - } - - key3 := bson.D{ - {Key: "_id.x", Value: "hashed"}, - {Key: "_id.y", Value: 1}, - } - - var runCmdResult bson.M - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection1)}, - {Key: "key", Value: key1}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) - - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection2)}, - {Key: "key", Value: key2}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) - - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection3)}, - {Key: "key", Value: key3}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(Source) - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := mongodataagent.Connect(context.Background(), Target.ConnectionOptions([]string{}), nil) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func insertOne(t *testing.T, coll *mongo.Collection, row any) { - _, err := coll.InsertOne(context.Background(), row) - require.NoError(t, err) -} - -func updateOne(t *testing.T, coll *mongo.Collection, filter, update bson.D) { - _, err := coll.UpdateOne(context.Background(), filter, update) - require.NoError(t, err) -} - -func Load(t *testing.T) { - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // create source collections - - db := client.Database(DB) - err = db.CreateCollection(context.Background(), Collection1) - require.NoError(t, err) - err = db.CreateCollection(context.Background(), Collection2) - require.NoError(t, err) - err = db.CreateCollection(context.Background(), Collection3) - require.NoError(t, err) - - coll1 := db.Collection(Collection1) - coll2 := db.Collection(Collection2) - coll3 := db.Collection(Collection3) - - type CompositeID struct { - X string `bson:"x,omitempty"` - Y string `bson:"y,omitempty"` - } - - type Citizen struct { - ID CompositeID `bson:"_id"` - Name string `bson:"name,omitempty"` - Age int `bson:"age,omitempty"` - City string `bson:"city,omitempty"` - } - - jamesGordon := Citizen{ - ID: CompositeID{"x1", "y1"}, - Name: "James Gordon", - Age: 33, - City: "Gotham", - } - - insertOne(t, coll1, jamesGordon) - insertOne(t, coll2, jamesGordon) - insertOne(t, coll3, jamesGordon) - - //------------------------------------------------------------------------------------ - // shard target collections - - targetClient, err := mongodataagent.Connect(context.Background(), Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - targetDB := targetClient.Database(DB) - err = targetDB.CreateCollection(context.Background(), Collection1) - require.NoError(t, err) - err = targetDB.CreateCollection(context.Background(), Collection2) - require.NoError(t, err) - err = targetDB.CreateCollection(context.Background(), Collection3) - require.NoError(t, err) - - ShardTargetCollections(t, targetClient) - - //------------------------------------------------------------------------------------ - // activate - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // replicate update one record and insert one - - leslieThompkins := Citizen{ - ID: CompositeID{"x2", "y2"}, - Name: "Leslie Thompkins", - Age: 29, - City: "Gotham", - } - - insertOne(t, coll1, leslieThompkins) - insertOne(t, coll2, leslieThompkins) - insertOne(t, coll3, leslieThompkins) - - leslieFilter := bson.D{{Key: "_id", Value: leslieThompkins.ID}} - leslieUpdate := bson.D{{Key: "$set", Value: bson.D{{Key: "city", Value: "Atlanta"}}}} - - updateOne(t, coll1, leslieFilter, leslieUpdate) - updateOne(t, coll2, leslieFilter, leslieUpdate) - updateOne(t, coll3, leslieFilter, leslieUpdate) - - jamesFilter := bson.D{{Key: "_id", Value: jamesGordon.ID}} - jamesUpdate := bson.D{{Key: "$set", Value: bson.D{{Key: "age", Value: 34}}}} - - updateOne(t, coll1, jamesFilter, jamesUpdate) - updateOne(t, coll2, jamesFilter, jamesUpdate) - updateOne(t, coll3, jamesFilter, jamesUpdate) - - //------------------------------------------------------------------------------------ - // check results - - require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection1, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection2, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection3, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mongo2mongo/snapshot/check_db_test.go b/tests/e2e/mongo2mongo/snapshot/check_db_test.go deleted file mode 100644 index c35c0244..00000000 --- a/tests/e2e/mongo2mongo/snapshot/check_db_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package snapshot - -import ( - "context" - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mongocommon "github.com/doublecloud/tross/pkg/providers/mongo" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -const databaseName string = "db" - -var ( - Source = *mongocommon.RecipeSource( - mongocommon.WithCollections( - mongocommon.MongoCollection{DatabaseName: databaseName, CollectionName: "timmyb32r_test"}, - mongocommon.MongoCollection{DatabaseName: databaseName, CollectionName: "empty"}, - ), - ) - Target = *mongocommon.RecipeTarget(mongocommon.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - - dpconfig.Cloud = &dpconfig.CloudConfig{} -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongocommon.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongocommon.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongocommon.MongoDestination) (*mongocommon.MongoClientWrapper, error) { - return mongocommon.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Snapshot", Snapshot) - }) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(&Source) - client, err := mongocommon.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - client, err := mongocommon.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // insert one record - - db := client.Database(databaseName) - err = db.CreateCollection(context.Background(), "timmyb32r_test") - require.NoError(t, err) - - coll := db.Collection("timmyb32r_test") - - type Trainer struct { - Name string - Age int - City string - } - - _, err = coll.InsertOne(context.Background(), Trainer{"a", 1, "aa"}) - require.NoError(t, err) - - err = db.CreateCollection(context.Background(), "empty") - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // upload snapshot - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - snapshotLoader := tasks.NewSnapshotLoader(client2.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // check results - - err = helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams()) - require.NoError(t, err) -} diff --git a/tests/e2e/mysql2ch/comparators.go b/tests/e2e/mysql2ch/comparators.go deleted file mode 100644 index 83ec190d..00000000 --- a/tests/e2e/mysql2ch/comparators.go +++ /dev/null @@ -1,69 +0,0 @@ -package mysql2ch - -import ( - "bytes" - "encoding/base64" - "fmt" - "strings" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/abstract" -) - -func RightStringToBase64BytesComparator(l interface{}, lCol abstract.ColSchema, r interface{}, rCol abstract.ColSchema, _ bool) (comparable bool, result bool, err error) { - lOriginalIsMySQL := strings.HasPrefix(lCol.OriginalType, "mysql:") - rOriginalIsClickHouse := strings.HasPrefix(rCol.OriginalType, "ch:") - if !lOriginalIsMySQL || !rOriginalIsClickHouse { - return false, false, nil - } - - lBytes, lIsBytes := l.([]byte) - if !lIsBytes { - return false, false, nil - } - - rString, rIsString := r.(string) - if !rIsString { - return false, false, nil - } - - if len(rString) < 2 { - return false, false, xerrors.Errorf("the right string '%s' does not have surrounding double quotes", rString) - } - rDecodedBytes, err := base64.StdEncoding.DecodeString(rString[1 : len(rString)-1]) // remove trailing double quotes - if err != nil { - return false, false, nil - } - - return true, bytes.Equal(lBytes, rDecodedBytes), nil -} - -func MySQLBytesToStringOptionalComparator(l interface{}, lCol abstract.ColSchema, r interface{}, rCol abstract.ColSchema, _ bool) (comparable bool, result bool, err error) { - lOriginalIsMySQL := strings.HasPrefix(lCol.OriginalType, "mysql:") - rOriginalIsMySQL := strings.HasPrefix(rCol.OriginalType, "mysql:") - - if !lOriginalIsMySQL && !rOriginalIsMySQL { - return false, false, nil - } - - if lOriginalIsMySQL { - l = maybeBytesToString(l, r) - } - if rOriginalIsMySQL { - r = maybeBytesToString(r, l) - } - - if fmt.Sprintf("%v", l) == fmt.Sprintf("%v", r) { - return true, true, nil - } - return false, false, nil -} - -func maybeBytesToString(v any, other any) any { - if _, otherIsString := other.(string); otherIsString { - if vB, vIsB := v.([]byte); vIsB { - return string(vB) - } - } - return v -} diff --git a/tests/e2e/mysql2ch/replication/check_db_test.go b/tests/e2e/mysql2ch/replication/check_db_test.go deleted file mode 100644 index d12896f7..00000000 --- a/tests/e2e/mysql2ch/replication/check_db_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package snapshot - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/e2e/mysql2ch" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = mysql.MysqlSource{ - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - NoTracking: false, // what is it? - ServerID: 1, // what is it? - } - Target = model.ChDestination{ - ShardsList: []model.ClickHouseShard{ - { - Name: "_", - Hosts: []string{ - "localhost", - }, - }, - }, - User: "default", - Password: "", - Database: "source", - HTTPPort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_HTTP_PORT"), - NativePort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_NATIVE_PORT"), - ProtocolUnspecified: true, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) -} - -func TestReplication(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //------------------------------------------------------------------------------------ - // insert/update/delete several record - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - client, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = client.Exec("INSERT INTO mysql_replication (id, val1, val2, b1, b8, b11) VALUES (3, 3, 'c', NULL, NULL, NULL), (4, 4, 'd', b'0', b'00000000', b'00000000000'), (5, 5, 'e', b'1', b'11111111', b'11111111111')") - require.NoError(t, err) - - _, err = client.Exec("UPDATE mysql_replication SET val2='ee' WHERE id=5;") - require.NoError(t, err) - - _, err = client.Query("DELETE FROM mysql_replication WHERE id=3;") - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - - require.NoError(t, helpers.WaitEqualRowsCount(t, Source.Database, "mysql_replication", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator).WithPriorityComparators(mysql2ch.MySQLBytesToStringOptionalComparator))) -} diff --git a/tests/e2e/mysql2ch/replication/dump/ch/dump.sql b/tests/e2e/mysql2ch/replication/dump/ch/dump.sql deleted file mode 100644 index 9bcf3484..00000000 --- a/tests/e2e/mysql2ch/replication/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE source; diff --git a/tests/e2e/mysql2ch/replication/dump/mysql/dump.sql b/tests/e2e/mysql2ch/replication/dump/mysql/dump.sql deleted file mode 100644 index 29cb51b2..00000000 --- a/tests/e2e/mysql2ch/replication/dump/mysql/dump.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE `mysql_replication` -( - `id` INT AUTO_INCREMENT PRIMARY KEY, - - `val1` INT, - `val2` VARCHAR(20), - - `b1` BIT(1), - `b8` BIT(8), - `b11` BIT(11) -) engine = innodb default charset = utf8; - -INSERT INTO mysql_replication (id, val1, val2, b1, b8, b11) VALUES -(1, 1, 'a', b'0', b'00000000', b'00000000000'), -(2, 2, 'b', b'1', b'10000000', b'10000000000'); diff --git a/tests/e2e/mysql2ch/replication_minimal/check_db_test.go b/tests/e2e/mysql2ch/replication_minimal/check_db_test.go deleted file mode 100644 index 147668f5..00000000 --- a/tests/e2e/mysql2ch/replication_minimal/check_db_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package snapshot - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "source" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = mysql.MysqlSource{ - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - NoTracking: false, // what is it? - ServerID: 1, // what is it? - } - Target = model.ChDestination{ - ShardsList: []model.ClickHouseShard{ - { - Name: "_", - Hosts: []string{ - "localhost", - }, - }, - }, - User: "default", - Password: "", - Database: databaseName, - HTTPPort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_HTTP_PORT"), - NativePort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_NATIVE_PORT"), - ProtocolUnspecified: true, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestReplication(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - - client, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = mysql.SyncBinlogPosition(&Source, "", cpclient.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // insert/update/delete several record - - rows, err := client.Query("INSERT INTO __test (id, val1, val2) VALUES (3, 3, 'c'), (4, 4, 'd'), (5, 5, 'e')") - require.NoError(t, err) - _ = rows.Close() - - rows, err = client.Query("UPDATE __test SET val2='ee' WHERE id=5;") - require.NoError(t, err) - _ = rows.Close() - - rows, err = client.Query("DELETE FROM __test WHERE id=3;") - require.NoError(t, err) - _ = rows.Close() - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(compareDataTypes))) -} - -func compareDataTypes(l, r string) bool { - if l == "utf8" && r == "string" { - return true - } - return l == r -} diff --git a/tests/e2e/mysql2ch/replication_minimal/dump/ch/dump.sql b/tests/e2e/mysql2ch/replication_minimal/dump/ch/dump.sql deleted file mode 100644 index 9bcf3484..00000000 --- a/tests/e2e/mysql2ch/replication_minimal/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE source; diff --git a/tests/e2e/mysql2ch/replication_minimal/dump/mysql/dump.sql b/tests/e2e/mysql2ch/replication_minimal/dump/mysql/dump.sql deleted file mode 100644 index e73a061b..00000000 --- a/tests/e2e/mysql2ch/replication_minimal/dump/mysql/dump.sql +++ /dev/null @@ -1,9 +0,0 @@ -set @@GLOBAL.binlog_row_image = 'minimal'; - -CREATE TABLE `__test` -( - `id` INT NOT NULL AUTO_INCREMENT PRIMARY KEY, - `val1` INT, - `val2` VARCHAR(20) -) engine = innodb - default charset = utf8; diff --git a/tests/e2e/mysql2ch/snapshot/check_db_test.go b/tests/e2e/mysql2ch/snapshot/check_db_test.go deleted file mode 100644 index 2e14f08f..00000000 --- a/tests/e2e/mysql2ch/snapshot/check_db_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package snapshot - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/e2e/mysql2ch" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = mysql.MysqlSource{ - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - NoTracking: true, - } - Target = model.ChDestination{ - ShardsList: []model.ClickHouseShard{ - { - Name: "_", - Hosts: []string{ - "localhost", - }, - }, - }, - User: "default", - Password: "", - Database: "source", - HTTPPort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_HTTP_PORT"), - NativePort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_NATIVE_PORT"), - ProtocolUnspecified: true, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "MySQL source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - _ = helpers.Activate(t, transfer) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator).WithPriorityComparators(mysql2ch.MySQLBytesToStringOptionalComparator))) -} diff --git a/tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql b/tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql deleted file mode 100644 index 9bcf3484..00000000 --- a/tests/e2e/mysql2ch/snapshot/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE source; diff --git a/tests/e2e/mysql2ch/snapshot/dump/mysql/dump.sql b/tests/e2e/mysql2ch/snapshot/dump/mysql/dump.sql deleted file mode 100644 index 3c58c497..00000000 --- a/tests/e2e/mysql2ch/snapshot/dump/mysql/dump.sql +++ /dev/null @@ -1,101 +0,0 @@ -DROP TABLE IF EXISTS `mysql_snapshot`; -CREATE TABLE `mysql_snapshot` ( - `i` INT AUTO_INCREMENT PRIMARY KEY, - `ti` TINYINT, - `si` SMALLINT, - `mi` MEDIUMINT, - `bi` BIGINT, - - `f` FLOAT, - `dp` DOUBLE PRECISION, - - `b1` BIT(1), - `b8` BIT(8), - `b11` BIT(11), - - `b` BOOL, - - `c10` CHAR(10), - `vc20` VARCHAR(20), - `tx` TEXT, - - `d` DATE, - `t` TIME, - `dt` DATETIME, - `ts` TIMESTAMP, - `y` YEAR -) engine=innodb default charset=utf8; - -INSERT INTO `mysql_snapshot` (ti, si, mi, bi, f, dp, b1, b8, b11, b, c10, vc20, tx, d, t, dt, ts, y) VALUES -( - 0, -- ti - 0, -- si - 0, -- mi - 0, -- bi - - 0.0, -- f - 0.0, -- dp - - b'0', -- b1 - b'00000000', -- b8 - b'00000000000', -- b11 - - false, -- b - - ' ', -- c10 - ' ', -- c20 - '', -- tx - '1970-01-01', -- d - '00:00:00.000000', -- t - '1900-01-01 01:00:00.000000', -- dt - '1970-01-01 01:00:00.000000', -- ts - '1901' -- y -), -( - 127, -- ti - 32767, -- si - 8388607, -- mi - 9223372036854775807, -- bi - - 1.1, -- f - 1.1, -- dp - - b'1', -- b1 - b'10000000', -- b8 - b'10000000000', -- b11 - - true, -- b - - 'char1char1', -- c10 - 'char1char1char1char1', -- c20 - 'text-text-text', -- tx - '1999-12-31', -- d - '01:02:03.456789', -- t - '1999-12-31 23:59:59.999999', -- dt - '1999-12-31 23:59:59.999999', -- ts - '1999' -- y -), -( - -128, -- ti - -32768, -- si - -8388608, -- mi - -9223372036854775808, -- bi - - 1.1, -- f - 1.1, -- dp - - b'1', -- b1 - b'11111111', -- b8 - b'11111111111', -- b11 - - true, -- b - - 'sant" '' CL', -- c10 - 'sant" '' CLAWS \\\\\\\\""', -- c20 - 'ho-ho-ho my name is "Santa" ''CLAWS\\', -- tx - '2038-12-31', -- d - '23:59:59.999999', -- t - '2106-02-07 06:28:15.999999', -- dt - '2038-01-19 04:14:06.999999', -- ts - '2155' -- y -); diff --git a/tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go b/tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go deleted file mode 100644 index badc676c..00000000 --- a/tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package snapshot - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = mysql.MysqlSource{ - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - NoTracking: true, - } - Target = model.ChDestination{ - ShardsList: []model.ClickHouseShard{ - { - Name: "_", - Hosts: []string{ - "localhost", - }, - }, - }, - User: "default", - Password: "", - Database: "source", - HTTPPort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_HTTP_PORT"), - NativePort: helpers.GetIntFromEnv("RECIPE_CLICKHOUSE_NATIVE_PORT"), - ProtocolUnspecified: true, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - _ = helpers.Activate(t, transfer) - - err := helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator)) - require.Error(t, err) - require.Contains(t, err.Error(), "Total Errors: 1") - require.Contains(t, err.Error(), "Total unmatched: 1") - require.Contains(t, err.Error(), `"source"."empty"`) - require.Contains(t, err.Error(), "table not found in target DB") -} diff --git a/tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql b/tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql deleted file mode 100644 index 9bcf3484..00000000 --- a/tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE source; diff --git a/tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql b/tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql deleted file mode 100644 index 35b48c3e..00000000 --- a/tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE `__test` ( - `int` INT, - `int_u` INT UNSIGNED, - - `bool` BOOL, - - `char` CHAR(10), - `varchar` VARCHAR(20), - - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY -- just to have a primary key -) engine=innodb default charset=utf8; - -INSERT INTO `__test` -(`int`, `int_u`, `bool`, `char`, `varchar`) -VALUES -(1, 2, true, 'text', 'test') -, -(-123, 234, false, 'magic', 'string') -; - -CREATE TABLE `empty` ( - `int` INT, - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY -- just to have a primary key -) engine=innodb default charset=utf8; diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted b/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted deleted file mode 100644 index 932d4048..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577902210000,"datetime1":1577902210100,"datetime2":1577902210120,"datetime3":1577902210123,"datetime4":1577902210123400,"datetime5":1577902210123450,"datetime6":1577902210123456,"datetime_":1577902210000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"c","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":5598,"query":null,"row":0,"server_id":0,"snapshot":"false","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0 b/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0 deleted file mode 100644 index 932d4048..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0 +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577902210000,"datetime1":1577902210100,"datetime2":1577902210120,"datetime3":1577902210123,"datetime4":1577902210123400,"datetime5":1577902210123450,"datetime6":1577902210123456,"datetime_":1577902210000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"c","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":5598,"query":null,"row":0,"server_id":0,"snapshot":"false","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1 b/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1 deleted file mode 100644 index 932d4048..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1 +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577902210000,"datetime1":1577902210100,"datetime2":1577902210120,"datetime3":1577902210123,"datetime4":1577902210123400,"datetime5":1577902210123450,"datetime6":1577902210123456,"datetime_":1577902210000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"c","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":5598,"query":null,"row":0,"server_id":0,"snapshot":"false","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.2 b/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.2 deleted file mode 100644 index 932d4048..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.2 +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577902210000,"datetime1":1577902210100,"datetime2":1577902210120,"datetime3":1577902210123,"datetime4":1577902210123400,"datetime5":1577902210123450,"datetime6":1577902210123456,"datetime_":1577902210000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"c","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":5598,"query":null,"row":0,"server_id":0,"snapshot":"false","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.3 b/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.3 deleted file mode 100644 index 932d4048..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.3 +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577902210000,"datetime1":1577902210100,"datetime2":1577902210120,"datetime3":1577902210123,"datetime4":1577902210123400,"datetime5":1577902210123450,"datetime6":1577902210123456,"datetime_":1577902210000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"c","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":5598,"query":null,"row":0,"server_id":0,"snapshot":"false","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.4 b/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.4 deleted file mode 100644 index 932d4048..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.4 +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577902210000,"datetime1":1577902210100,"datetime2":1577902210120,"datetime3":1577902210123,"datetime4":1577902210123400,"datetime5":1577902210123450,"datetime6":1577902210123456,"datetime_":1577902210000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"c","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":5598,"query":null,"row":0,"server_id":0,"snapshot":"false","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int32"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"integer5","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"char_","optional":true,"type":"string"},{"field":"char5","optional":true,"type":"string"},{"field":"varchar5","optional":true,"type":"string"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"blob_","optional":true,"type":"bytes"},{"field":"text_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/replication/canondata/result.json b/tests/e2e/mysql2kafka/debezium/replication/canondata/result.json deleted file mode 100644 index 21c935f8..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/canondata/result.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "replication.replication.TestReplication": [ - "", - "", - "", - "", - "", - "", - { - "uri": "file://replication.replication.TestReplication/extracted" - }, - { - "uri": "file://replication.replication.TestReplication/extracted.0" - }, - { - "uri": "file://replication.replication.TestReplication/extracted.1" - }, - { - "uri": "file://replication.replication.TestReplication/extracted.2" - }, - { - "uri": "file://replication.replication.TestReplication/extracted.3" - }, - { - "uri": "file://replication.replication.TestReplication/extracted.4" - } - ] -} diff --git a/tests/e2e/mysql2kafka/debezium/replication/check_db_test.go b/tests/e2e/mysql2kafka/debezium/replication/check_db_test.go deleted file mode 100644 index 02a4741f..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/check_db_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "context" - "os" - "regexp" - "testing" - "time" - - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/internal/test/yatest" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - kafka2 "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/util" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = helpers.RecipeMysqlSource() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Source.WithDefaults() -} - -func eraseMeta(in string) string { - result := in - tsmsRegexp := regexp.MustCompile(`"ts_ms":\d+`) - result = tsmsRegexp.ReplaceAllString(result, `"ts_ms":0`) - return result -} - -func TestReplication(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - )) - //------------------------------------------------------------------------------ - //initialize variables - // fill 't' by giant random string - insertStmt, err := os.ReadFile(yatest.SourcePath("transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql")) - require.NoError(t, err) - update1Stmt, err := os.ReadFile(yatest.SourcePath("transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql")) - require.NoError(t, err) - update2Stmt := `UPDATE customers3 SET bool1=true WHERE bool1=false;` - // update with pkey change - update3Stmt := `UPDATE customers3 SET pk=2 WHERE pk=1;` - deleteStmt := `DELETE FROM customers3 WHERE 1=1;` - - //------------------------------------------------------------------------------ - //prepare dst - - dst, err := kafka2.DestinationRecipe() - require.NoError(t, err) - dst.Topic = "dbserver1" - dst.FormatSettings = server.SerializationFormat{Name: server.SerializationFormatDebezium} - - // prepare additional transfer: from dst to mock - - result := make([]abstract.ChangeItem, 0) - mockSink := &helpers.MockSink{ - PushCallback: func(in []abstract.ChangeItem) { - abstract.Dump(in) - for _, el := range in { - if len(el.ColumnValues) > 0 { - result = append(result, el) - } - } - }, - } - mockTarget := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return mockSink }, - Cleanup: server.DisabledCleanup, - } - additionalTransfer := helpers.MakeTransfer("additional", &kafka2.KafkaSource{ - Connection: dst.Connection, - Auth: dst.Auth, - GroupTopics: []string{dst.Topic}, - IsHomo: true, - }, &mockTarget, abstract.TransferTypeIncrementOnly) - - // activate main transfer - - helpers.InitSrcDst(helpers.TransferID, Source, dst, abstract.TransferTypeIncrementOnly) - transfer := helpers.MakeTransfer(helpers.TransferID, Source, dst, abstract.TransferTypeIncrementOnly) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - go func() { - for { - // restart transfer if error - errCh := make(chan error, 1) - w, err := helpers.ActivateErr(additionalTransfer, func(err error) { - errCh <- err - }) - require.NoError(t, err) - _, ok := util.Receive(ctx, errCh) - if !ok { - return - } - w.Close(t) - } - }() - //----------------------------------------------------------------------------------------------------------------- - // execute SQL statements - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - srcConn, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(string(insertStmt)) - require.NoError(t, err) - _, err = srcConn.Exec(string(update1Stmt)) - require.NoError(t, err) - _, err = srcConn.Exec(update2Stmt) - require.NoError(t, err) - _, err = srcConn.Exec(update3Stmt) - require.NoError(t, err) - _, err = srcConn.Exec(deleteStmt) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - - for { - if len(result) == 6 { - canonData := make([]string, 6) - for i := 0; i < len(result); i += 1 { - canonVal := eraseMeta(string(kafka2.GetKafkaRawMessageData(&result[0]))) - canonData = append(canonData, canonVal) - } - canon.SaveJSON(t, canonData) - break - } - time.Sleep(time.Second) - } -} diff --git a/tests/e2e/mysql2kafka/debezium/replication/init_source/dump.sql b/tests/e2e/mysql2kafka/debezium/replication/init_source/dump.sql deleted file mode 100644 index ab4cde1d..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/init_source/dump.sql +++ /dev/null @@ -1,114 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - --- timestamp_ TIMESTAMP, -- uncomment after TM-4377 --- timestamp0 TIMESTAMP(0),-- uncomment after TM-4377 --- timestamp1 TIMESTAMP(1),-- uncomment after TM-4377 --- timestamp2 TIMESTAMP(2),-- uncomment after TM-4377 --- timestamp3 TIMESTAMP(3),-- uncomment after TM-4377 --- timestamp4 TIMESTAMP(4),-- uncomment after TM-4377 --- timestamp5 TIMESTAMP(5),-- uncomment after TM-4377 --- timestamp6 TIMESTAMP(6),-- uncomment after TM-4377 - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), - time1 TIME(1), - time2 TIME(2), - time3 TIME(3), - time4 TIME(4), - time5 TIME(5), - time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - --- NUMERIC_ NUMERIC, -- See TM-4581 --- NUMERIC_5 NUMERIC(5), -- See TM-4581 --- NUMERIC_5_2 NUMERIC(5,2), -- See TM-4581 - --- DECIMAL_ DECIMAL, -- See TM-4581 --- DECIMAL_5 DECIMAL(5), -- See TM-4581 --- DECIMAL_5_2 DECIMAL(5,2), -- See TM-4581 - - primary key (pk) -) engine=innodb default charset=utf8; diff --git a/tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql b/tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql deleted file mode 100644 index b619d234..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/testdata/insert.sql +++ /dev/null @@ -1,123 +0,0 @@ -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - --- '1999-01-01 00:00:01', -- TIMESTAMP -- uncomment after TM-4377 --- '1999-10-19 10:23:54', -- TIMESTAMP(0) -- uncomment after TM-4377 --- '2004-10-19 10:23:54.1', -- TIMESTAMP(1) -- uncomment after TM-4377 --- '2004-10-19 10:23:54.12', -- TIMESTAMP(2) -- uncomment after TM-4377 --- '2004-10-19 10:23:54.123', -- TIMESTAMP(3) -- uncomment after TM-4377 --- '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) -- uncomment after TM-4377 --- '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) -- uncomment after TM-4377 --- '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) -- uncomment after TM-4377 - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - '04:05:06.12345', -- TIME(5) - '04:05:06.123456', -- TIME(6) - - '2020-01-01 18:10:10', -- DATETIME - '2020-01-01 18:10:10', -- DATETIME(0) - '2020-01-01 18:10:10.1', -- DATETIME(1) - '2020-01-01 18:10:10.12', -- DATETIME(2) - '2020-01-01 18:10:10.123', -- DATETIME(3) - '2020-01-01 18:10:10.1234', -- DATETIME(4) - '2020-01-01 18:10:10.12345', -- DATETIME(5) - '2020-01-01 18:10:10.123456' -- DATETIME(6) - - -- DECIMAL TYPES - --- 1234567890, -- NUMERIC -- See TM-4581 --- 12345, -- NUMERIC(5) -- See TM-4581 --- 123.45, -- NUMERIC(5,2) -- See TM-4581 --- --- 2345678901, -- DECIMAL -- See TM-4581 --- 23451, -- DECIMAL(5) -- See TM-4581 --- 231.45 -- DECIMAL(5,2) -- See TM-4581 - - -- SPATIAL TYPES - - # ST_GeomFromText('LINESTRING(0 0,1 2,2 4)'), -- LINESTRING_ GEOMETRY, - # ST_GeomFromText('POLYGON((0 0,10 0,10 10,0 10,0 0),(5 5,7 5,7 7,5 7, 5 5))'), -- POLYGON_ GEOMETRY, - # ST_GeomFromText('MULTIPOINT(0 0, 15 25, 45 65)'), -- MULTIPOINT_ GEOMETRY, - # ST_GeomFromText('MULTILINESTRING((12 12, 22 22), (19 19, 32 18))'), -- MULTILINESTRING_ GEOMETRY, - # ST_GeomFromText('MULTIPOLYGON(((0 0,11 0,12 11,0 9,0 0)),((3 5,7 4,4 7,7 7,3 5)))'), -- MULTIPOLYGON_ GEOMETRY, - # ST_GeomFromText('GEOMETRYCOLLECTION(POINT(3 2),LINESTRING(0 0,1 3,2 5,3 5,4 7))') -- GEOMETRYCOLLECTION_ GEOMETRY, -); diff --git a/tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql b/tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql deleted file mode 100644 index 01dd70a9..00000000 --- a/tests/e2e/mysql2kafka/debezium/replication/testdata/update_string.sql +++ /dev/null @@ -1 +0,0 @@ -UPDATE customers3 SET text_ = 'LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\u003c4SaNJTHkL@1?6YcDf\u003eHI[862bUb4gT@k\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^\u003e\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4\u003cI_@d]\u003eF1e]hj_XJII862[N\u003cj=bYA\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\u003e0UDDBb8h]65C\u003efC\u003c[02jRT]bJ\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\u003eH2X\\]_\u003cEE3@?U2_L67UV8FNQecS2Y=@6\u003ehb1\\3F66UE[W9\u003c]?HH\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJ\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\u003e@A\u003e5\u003cK\\d4QM:7:41B^_c\\FCI=\u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[T\u003cIQI4S_g\u003e;gf[BF_EN\u003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\u003c\u003e[=0W3Of;6;RFY=Q\\OK\\7[\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\u003cOK6gV=EMGC?\\F\u003cXaa_\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;\u003eMZG\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\u003e7cU]M[\u003c72c\u003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:\u003ea5a;j\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\u003c84==_9FJbjbEhQeOV\u003eWDP4MV^W1_]=TeAa66jLObKG\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\u003ebGaJ2f;VB\u003eG\\3\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\u003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\u003ek\u003ePUHD6\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\u003e=\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@f\u003ciDV?6i0WVXj\u003c@ZPd5d\\5B]O?7h=C=8O:L:IR8I\u003e^6\u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLE\u003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\u003e7UBbR58G?[X_O1b\\:[65\u003eP9Z6\u003c]S8=a\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\u003e^GMUPS2]b\u003e]DN?aUKNL^@RV\u003cFTBh:Q[Q3E5VHbK?5=RTKI\u003eggZZ\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\u003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[H\u003cUb4EE^\u003ckWO7\u003eR8fD9JQHR\u003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5\u003cBA\\3IVT@gG\\4\u003cRRS459YROd=_H1OM=a_hd\u003cSMLOd=S6^:eG\u003ejPgQ4_^d\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\u003cDa;\\Ni[AC\u003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9\u003eT12E?FZ9cYCLQbH[2O\u003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihE\u003ehMVaDF\u003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\u003eHga5eW[E8\u003c9jdYO7\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Q\u003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NR\u003eTK07=]7Ecdej\u003cUj\u003cDe1H\u003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_I\u003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\u003cSV@b[GVEU3Xh;R7\u003cXeTNgN\u003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhR\u003e]@GIYf[L55g\u003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\u003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\u003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9He\u003e1L[3\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\u003eCLdAe]6L2AD0aYHc5\u003e=fM7h\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]\u003eKE\u003cea\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\u003eLW5VIfJL:eQ4K3a1^WN5T=\\X=\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1C\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\u003eTQQWYJ_@FAX\\]9jh\u003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6\u003eBgES\u003e5EaeOFeG:i\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\u003eRQ]5Z9jA@Y9V1ZI6TDkC\u003eNZ_f_DR\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\u003eFW\u003cJ6X?IiJ\u003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;f\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHI\u003eI]gBS\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34\u003eh_2@i3kd02G\u003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\u003c^U7Hk]7Q6P:QZS;Ge@:\u003c\u003cfT6PK7j4?;cdC@c5GI:gS[W\u003cf26;\u003cBG7fMXFTWJcbB\\9QT\u003eh3HdV8Pb3Rh\u003e^?Ue:7RP[=jT4AE\u003ebiL_1dYW1\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\u003cA9LXQbECIc2M\u003c^I\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;c\u003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\u003c_F9P\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\u003e=R4U3W1G;\u003chN\\WFO_=DD\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5T\u003eY?bFOMZO\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\u003efaaP8P4KDVSCiQ=2\u003c=Ef:\u003eP\u003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_\u003c@5Z8fDPJAE8DcGUIb8C\u003c_L7XhP=\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\u003e]LW\u003ee^b\u003e?0G9Ie\u003c\u003c@UT4e9\u003cGM_jME7[6TFEN:\u003c\\H\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\u003cL42d\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\u003eEJQi8j;]L5CILgXdR_\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\u003cKHA:\\[CW7SRYVhE1[MD\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8k\u003eQb]UVVZ:18fe_8M?\\?\u003e\u003eLf4QSG@jO@\u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\u003cR]Ofg:TNGW0L\u003ePOC_CP\u003e^PI[aZ:KY^V@Q;;ME_k\\K0\u003eYP]1D5QSc51SfZ]FIP1Y6\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\u003e0]RJGeQiJG5\\=O8TRG5U\u003eLGa\u003eRi2K\u003c3=1TVHN=FhTJYajbIP\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\u003e93HU2ig?7\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1J\u003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^G\u003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1a\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\u003cA\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\u003e@^\u003e[4\u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVde\u003cUVVNH2EJ^=ALOFKUX:^\u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2S\u003cX?9bC7Ebc5V5E]\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SC\u003e\u003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\u003ceM8?j]NZai4\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\u003eI=?@f\u003cG349NMId8[T^@Sf\u003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@d\u003cc\u003cMhS3K;b\u003eZbHAf[GKME9igTY7iVFba\u003e4D;WFVb=dQ4Abj2\u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\u003e@TM9eO\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\u003c@=gPHLhQFDC@:T\u003cREdY\u003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1ad\u003cIiK1O7fbD[7[\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\u003e=FFMHPSBf8:\\XRZ91D:2D[1Y\u003eX\\bfj4BEQZe:1A\u003cQj^@7SAK]C_NCM\\0\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\u003e2\u003e4X:9JYPXk\u003eX_?;DAfL\u003ec?HF\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\u003e1\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aE\u003cY^MPd\u003e6M^iNNe=P6i6Lf::P6ebjX;\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\u003c93\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\u003c8@j5e\u003eVA76=g2=gD4V1eYF0bZd0EZ\u003cMk2M4g[Z=baJ]cVY\u003c[D=U2RUdBNdW=69=8UB4E1@\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;D\u003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4ia\u003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fa\u003e:Vj=BR7EW0_hV4=]DaSeQ\u003c?8]?9X4GbZF41h;FS\u003c9Pa=^SQT\u003cL:GAIP3XX[\\4RKJVLFabj20Oc\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\W\u003cHg9FWd\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\u003eS\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\u003e67\u003eW\u003cQNZRKDH@]_j^M_AV9g4\u003chIF\u003eaSDhbj9GMdjh=F=j:\u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaS\u003eO]caAKi\u003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\u003cWkC\u003c^KSgbI7?aGVaRkbA2?_Raf^\u003e9DID]07\u003cS431;BaRhX:hNJj]\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\u003cN?J\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\u003e8]\u003eU2:HGATaUBPG\u003c\\c0aX@_D;_EOK=]Sjk=1:VGK\u003e=4P^K\\OD\\D008D\u003cgY[GfMjeM\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNf\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\u003ceaB7BV;\u003eIRKK' WHERE pk=1; diff --git a/tests/e2e/mysql2kafka/debezium/snapshot/canondata/result.json b/tests/e2e/mysql2kafka/debezium/snapshot/canondata/result.json deleted file mode 100644 index bc134030..00000000 --- a/tests/e2e/mysql2kafka/debezium/snapshot/canondata/result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "snapshot.snapshot.TestSnapshot": { - "uri": "file://snapshot.snapshot.TestSnapshot/extracted" - } -} diff --git a/tests/e2e/mysql2kafka/debezium/snapshot/canondata/snapshot.snapshot.TestSnapshot/extracted b/tests/e2e/mysql2kafka/debezium/snapshot/canondata/snapshot.snapshot.TestSnapshot/extracted deleted file mode 100644 index 83c75818..00000000 --- a/tests/e2e/mysql2kafka/debezium/snapshot/canondata/snapshot.snapshot.TestSnapshot/extracted +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"after":{"bigint5":88,"bigint_":8,"bigint_u":888,"binary5":"nwAAAAA=","binary_":"nw==","bit":true,"bit16":"nwA=","blob_":"/w==","bool1":false,"bool2":true,"char5":"abc","char_":"a","date_":-354285,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"datetime_":1577891410000,"double_":2.34,"double_precision":2.34,"enum_":"x-small","float_":1.23,"float_53":1.23,"int_":9,"int_u":9999,"integer5":999,"integer_":99,"json_":"{\"k1\":\"v1\"}","longblob_":"q80=","longtext_":"my-longtext","mediumblob_":"q80=","mediumint5":11,"mediumint_":1,"mediumint_u":111,"mediumtext_":"my-mediumtext","pk":1,"real_":123.45,"real_10_2":99999.99,"set_":"a","smallint5":100,"smallint_":1000,"smallint_u":10,"text_":"my-text","time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"time_":14706000000,"tinyblob_":"n5+f","tinyint1":true,"tinyint1u":1,"tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinytext_":"qwerty12345","varbinary5":"n58=","varchar5":"blab","year4":2155,"year_":1901},"before":null,"op":"r","source":{"connector":"mysql","db":"source","file":"mysql-log.000002","gtid":null,"name":"dbserver1","pos":4036,"query":null,"row":0,"server_id":0,"snapshot":"true","table":"customers3","thread":null,"ts_ms":0,"version":"1.1.2.Final"},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"pk","optional":false,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"blob_","optional":true,"type":"bytes"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"char5","optional":true,"type":"string"},{"field":"char_","optional":true,"type":"string"},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"integer5","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"text_","optional":true,"type":"string"},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"varchar5","optional":true,"type":"string"},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"pk","optional":false,"type":"int64"},{"field":"bigint5","optional":true,"type":"int64"},{"field":"bigint_","optional":true,"type":"int64"},{"field":"bigint_u","optional":true,"type":"int64"},{"field":"binary5","optional":true,"type":"bytes"},{"field":"binary_","optional":true,"type":"bytes"},{"field":"bit","optional":true,"type":"boolean"},{"field":"bit16","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1},{"field":"blob_","optional":true,"type":"bytes"},{"field":"bool1","optional":true,"type":"boolean"},{"field":"bool2","optional":true,"type":"boolean"},{"field":"char5","optional":true,"type":"string"},{"field":"char_","optional":true,"type":"string"},{"field":"datetime0","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime1","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime2","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime3","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"datetime4","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime5","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime6","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},{"field":"datetime_","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1},{"field":"date_","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},{"field":"double_","optional":true,"type":"double"},{"field":"double_precision","optional":true,"type":"double"},{"field":"enum_","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"type":"string","version":1},{"field":"float_","optional":true,"type":"double"},{"field":"float_53","optional":true,"type":"double"},{"field":"integer5","optional":true,"type":"int32"},{"field":"integer_","optional":true,"type":"int32"},{"field":"int_","optional":true,"type":"int32"},{"field":"int_u","optional":true,"type":"int64"},{"field":"json_","name":"io.debezium.data.Json","optional":true,"type":"string","version":1},{"field":"longblob_","optional":true,"type":"bytes"},{"field":"longtext_","optional":true,"type":"string"},{"field":"mediumblob_","optional":true,"type":"bytes"},{"field":"mediumint5","optional":true,"type":"int32"},{"field":"mediumint_","optional":true,"type":"int32"},{"field":"mediumint_u","optional":true,"type":"int32"},{"field":"mediumtext_","optional":true,"type":"string"},{"field":"real_","optional":true,"type":"double"},{"field":"real_10_2","optional":true,"type":"double"},{"field":"set_","name":"io.debezium.data.EnumSet","optional":true,"parameters":{"allowed":"a,b,c,d"},"type":"string","version":1},{"field":"smallint5","optional":true,"type":"int16"},{"field":"smallint_","optional":true,"type":"int16"},{"field":"smallint_u","optional":true,"type":"int32"},{"field":"text_","optional":true,"type":"string"},{"field":"time0","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time1","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time2","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time3","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time4","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time5","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time6","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"time_","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},{"field":"tinyblob_","optional":true,"type":"bytes"},{"field":"tinyint1","optional":true,"type":"boolean"},{"field":"tinyint1u","optional":true,"type":"int16"},{"field":"tinyint_","optional":true,"type":"int16"},{"field":"tinyint_def","optional":true,"type":"int16"},{"field":"tinyint_u","optional":true,"type":"int16"},{"field":"tinytext_","optional":true,"type":"string"},{"field":"varbinary5","optional":true,"type":"bytes"},{"field":"varchar5","optional":true,"type":"string"},{"field":"year4","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1},{"field":"year_","name":"io.debezium.time.Year","optional":true,"type":"int32","version":1}],"name":"dbserver1.source.customers3.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"file","optional":false,"type":"string"},{"field":"gtid","optional":true,"type":"string"},{"field":"pos","optional":false,"type":"int64"},{"field":"query","optional":true,"type":"string"},{"field":"row","optional":false,"type":"int32"},{"field":"server_id","optional":false,"type":"int64"},{"field":"thread","optional":true,"type":"int64"}],"name":"io.debezium.connector.mysql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"dbserver1.source.customers3.Envelope","optional":false,"type":"struct"}} \ No newline at end of file diff --git a/tests/e2e/mysql2kafka/debezium/snapshot/check_db_test.go b/tests/e2e/mysql2kafka/debezium/snapshot/check_db_test.go deleted file mode 100644 index c03d81eb..00000000 --- a/tests/e2e/mysql2kafka/debezium/snapshot/check_db_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "context" - "os" - "regexp" - "testing" - "time" - - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - kafka2 "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/util" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = helpers.RecipeMysqlSource() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Source.WithDefaults() -} - -func eraseMeta(in string) string { - result := in - tsmsRegexp := regexp.MustCompile(`"ts_ms":\d+`) - result = tsmsRegexp.ReplaceAllString(result, `"ts_ms":0`) - return result -} - -func TestSnapshot(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - )) - //------------------------------------------------------------------------------ - //prepare dst - - dst, err := kafka2.DestinationRecipe() - require.NoError(t, err) - dst.Topic = "dbserver1" - dst.FormatSettings = server.SerializationFormat{Name: server.SerializationFormatDebezium} - //------------------------------------------------------------------------------ - // prepare additional transfer: from dst to mock - - result := make([]abstract.ChangeItem, 0) - mockSink := &helpers.MockSink{ - PushCallback: func(in []abstract.ChangeItem) { - abstract.Dump(in) - result = append(result, in...) - }, - } - mockTarget := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return mockSink }, - Cleanup: server.DisabledCleanup, - } - additionalTransfer := helpers.MakeTransfer("additional", &kafka2.KafkaSource{ - Connection: dst.Connection, - Auth: dst.Auth, - GroupTopics: []string{dst.Topic}, - IsHomo: true, - }, &mockTarget, abstract.TransferTypeIncrementOnly) - //------------------------------------------------------------------------------ - // activate main transfer - - helpers.InitSrcDst(helpers.TransferID, Source, dst, abstract.TransferTypeSnapshotOnly) - transfer := helpers.MakeTransfer(helpers.TransferID, Source, dst, abstract.TransferTypeSnapshotOnly) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - go func() { - for { - // restart transfer if error - errCh := make(chan error, 1) - w, err := helpers.ActivateErr(additionalTransfer, func(err error) { - errCh <- err - }) - require.NoError(t, err) - _, ok := util.Receive(ctx, errCh) - if !ok { - return - } - w.Close(t) - } - }() - - for { - if len(result) == 1 { - canonVal := eraseMeta(string(kafka2.GetKafkaRawMessageData(&result[0]))) - canon.SaveJSON(t, canonVal) - break - } - time.Sleep(time.Second) - } -} diff --git a/tests/e2e/mysql2kafka/debezium/snapshot/init_source/dump.sql b/tests/e2e/mysql2kafka/debezium/snapshot/init_source/dump.sql deleted file mode 100644 index 085bb27f..00000000 --- a/tests/e2e/mysql2kafka/debezium/snapshot/init_source/dump.sql +++ /dev/null @@ -1,235 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), - time1 TIME(1), - time2 TIME(2), - time3 TIME(3), - time4 TIME(4), - time5 TIME(5), - time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - --- NUMERIC_ NUMERIC, -- See TM-4581 --- NUMERIC_5 NUMERIC(5), -- See TM-4581 --- NUMERIC_5_2 NUMERIC(5,2), -- See TM-4581 - --- DECIMAL_ DECIMAL, -- See TM-4581 --- DECIMAL_5 DECIMAL(5), -- See TM-4581 --- DECIMAL_5_2 DECIMAL(5,2), -- See TM-4581 - - -- SPATIAL TYPES - - # LINESTRING_ GEOMETRY, - # POLYGON_ GEOMETRY, - # MULTIPOINT_ GEOMETRY, - # MULTILINESTRING_ GEOMETRY, - # MULTIPOLYGON_ GEOMETRY, - # GEOMETRYCOLLECTION_ GEOMETRY, - - -- - - primary key (pk) - ) engine=innodb default charset=utf8; - - - - - - INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - '04:05:06.12345', -- TIME(5) - '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456' -- DATETIME(6) - - -- DECIMAL TYPES - --- 1234567890, -- NUMERIC -- See TM-4581 --- 12345, -- NUMERIC(5) -- See TM-4581 --- 123.45, -- NUMERIC(5,2) -- See TM-4581 - --- 2345678901, -- DECIMAL -- See TM-4581 --- 23451, -- DECIMAL(5) -- See TM-4581 --- 231.45 -- DECIMAL(5,2) -- See TM-4581 - - -- SPATIAL TYPES - - # ST_GeomFromText('LINESTRING(0 0,1 2,2 4)'), -- LINESTRING_ GEOMETRY, - # ST_GeomFromText('POLYGON((0 0,10 0,10 10,0 10,0 0),(5 5,7 5,7 7,5 7, 5 5))'), -- POLYGON_ GEOMETRY, - # ST_GeomFromText('MULTIPOINT(0 0, 15 25, 45 65)'), -- MULTIPOINT_ GEOMETRY, - # ST_GeomFromText('MULTILINESTRING((12 12, 22 22), (19 19, 32 18))'), -- MULTILINESTRING_ GEOMETRY, - # ST_GeomFromText('MULTIPOLYGON(((0 0,11 0,12 11,0 9,0 0)),((3 5,7 4,4 7,7 7,3 5)))'), -- MULTIPOLYGON_ GEOMETRY, - # ST_GeomFromText('GEOMETRYCOLLECTION(POINT(3 2),LINESTRING(0 0,1 3,2 5,3 5,4 7))') -- GEOMETRYCOLLECTION_ GEOMETRY, - ); diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go b/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go deleted file mode 100644 index ab8c475e..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go +++ /dev/null @@ -1,460 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - "os" - "sync" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/test/yatest" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - debeziumcommon "github.com/doublecloud/tross/pkg/debezium/common" - "github.com/doublecloud/tross/pkg/debezium/testutil" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = helpers.RecipeMysqlSource() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Source.WithDefaults() - Source.AllowDecimalAsFloat = true -} - -//--------------------------------------------------------------------------------------------------------------------- - -// fill 't' by giant random string -var update1Stmt = `UPDATE customers3 SET text_ = 'LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\u003c4SaNJTHkL@1?6YcDf\u003eHI[862bUb4gT@k\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^\u003e\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4\u003cI_@d]\u003eF1e]hj_XJII862[N\u003cj=bYA\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\u003e0UDDBb8h]65C\u003efC\u003c[02jRT]bJ\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\u003eH2X\\]_\u003cEE3@?U2_L67UV8FNQecS2Y=@6\u003ehb1\\3F66UE[W9\u003c]?HH\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJ\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\u003e@A\u003e5\u003cK\\d4QM:7:41B^_c\\FCI=\u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[T\u003cIQI4S_g\u003e;gf[BF_EN\u003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\u003c\u003e[=0W3Of;6;RFY=Q\\OK\\7[\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\u003cOK6gV=EMGC?\\F\u003cXaa_\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;\u003eMZG\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\u003e7cU]M[\u003c72c\u003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:\u003ea5a;j\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\u003c84==_9FJbjbEhQeOV\u003eWDP4MV^W1_]=TeAa66jLObKG\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\u003ebGaJ2f;VB\u003eG\\3\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\u003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\u003ek\u003ePUHD6\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\u003e=\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@f\u003ciDV?6i0WVXj\u003c@ZPd5d\\5B]O?7h=C=8O:L:IR8I\u003e^6\u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLE\u003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\u003e7UBbR58G?[X_O1b\\:[65\u003eP9Z6\u003c]S8=a\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\u003e^GMUPS2]b\u003e]DN?aUKNL^@RV\u003cFTBh:Q[Q3E5VHbK?5=RTKI\u003eggZZ\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\u003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[H\u003cUb4EE^\u003ckWO7\u003eR8fD9JQHR\u003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5\u003cBA\\3IVT@gG\\4\u003cRRS459YROd=_H1OM=a_hd\u003cSMLOd=S6^:eG\u003ejPgQ4_^d\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\u003cDa;\\Ni[AC\u003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9\u003eT12E?FZ9cYCLQbH[2O\u003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihE\u003ehMVaDF\u003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\u003eHga5eW[E8\u003c9jdYO7\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Q\u003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NR\u003eTK07=]7Ecdej\u003cUj\u003cDe1H\u003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_I\u003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\u003cSV@b[GVEU3Xh;R7\u003cXeTNgN\u003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhR\u003e]@GIYf[L55g\u003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\u003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\u003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9He\u003e1L[3\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\u003eCLdAe]6L2AD0aYHc5\u003e=fM7h\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]\u003eKE\u003cea\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\u003eLW5VIfJL:eQ4K3a1^WN5T=\\X=\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1C\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\u003eTQQWYJ_@FAX\\]9jh\u003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6\u003eBgES\u003e5EaeOFeG:i\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\u003eRQ]5Z9jA@Y9V1ZI6TDkC\u003eNZ_f_DR\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\u003eFW\u003cJ6X?IiJ\u003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;f\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHI\u003eI]gBS\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34\u003eh_2@i3kd02G\u003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\u003c^U7Hk]7Q6P:QZS;Ge@:\u003c\u003cfT6PK7j4?;cdC@c5GI:gS[W\u003cf26;\u003cBG7fMXFTWJcbB\\9QT\u003eh3HdV8Pb3Rh\u003e^?Ue:7RP[=jT4AE\u003ebiL_1dYW1\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\u003cA9LXQbECIc2M\u003c^I\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;c\u003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\u003c_F9P\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\u003e=R4U3W1G;\u003chN\\WFO_=DD\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5T\u003eY?bFOMZO\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\u003efaaP8P4KDVSCiQ=2\u003c=Ef:\u003eP\u003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_\u003c@5Z8fDPJAE8DcGUIb8C\u003c_L7XhP=\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\u003e]LW\u003ee^b\u003e?0G9Ie\u003c\u003c@UT4e9\u003cGM_jME7[6TFEN:\u003c\\H\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\u003cL42d\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\u003eEJQi8j;]L5CILgXdR_\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\u003cKHA:\\[CW7SRYVhE1[MD\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8k\u003eQb]UVVZ:18fe_8M?\\?\u003e\u003eLf4QSG@jO@\u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\u003cR]Ofg:TNGW0L\u003ePOC_CP\u003e^PI[aZ:KY^V@Q;;ME_k\\K0\u003eYP]1D5QSc51SfZ]FIP1Y6\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\u003e0]RJGeQiJG5\\=O8TRG5U\u003eLGa\u003eRi2K\u003c3=1TVHN=FhTJYajbIP\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\u003e93HU2ig?7\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1J\u003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^G\u003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1a\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\u003cA\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\u003e@^\u003e[4\u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVde\u003cUVVNH2EJ^=ALOFKUX:^\u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2S\u003cX?9bC7Ebc5V5E]\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SC\u003e\u003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\u003ceM8?j]NZai4\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\u003eI=?@f\u003cG349NMId8[T^@Sf\u003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@d\u003cc\u003cMhS3K;b\u003eZbHAf[GKME9igTY7iVFba\u003e4D;WFVb=dQ4Abj2\u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\u003e@TM9eO\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\u003c@=gPHLhQFDC@:T\u003cREdY\u003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1ad\u003cIiK1O7fbD[7[\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\u003e=FFMHPSBf8:\\XRZ91D:2D[1Y\u003eX\\bfj4BEQZe:1A\u003cQj^@7SAK]C_NCM\\0\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\u003e2\u003e4X:9JYPXk\u003eX_?;DAfL\u003ec?HF\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\u003e1\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aE\u003cY^MPd\u003e6M^iNNe=P6i6Lf::P6ebjX;\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\u003c93\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\u003c8@j5e\u003eVA76=g2=gD4V1eYF0bZd0EZ\u003cMk2M4g[Z=baJ]cVY\u003c[D=U2RUdBNdW=69=8UB4E1@\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;D\u003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4ia\u003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fa\u003e:Vj=BR7EW0_hV4=]DaSeQ\u003c?8]?9X4GbZF41h;FS\u003c9Pa=^SQT\u003cL:GAIP3XX[\\4RKJVLFabj20Oc\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\W\u003cHg9FWd\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\u003eS\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\u003e67\u003eW\u003cQNZRKDH@]_j^M_AV9g4\u003chIF\u003eaSDhbj9GMdjh=F=j:\u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaS\u003eO]caAKi\u003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\u003cWkC\u003c^KSgbI7?aGVaRkbA2?_Raf^\u003e9DID]07\u003cS431;BaRhX:hNJj]\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\u003cN?J\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\u003e8]\u003eU2:HGATaUBPG\u003c\\c0aX@_D;_EOK=]Sjk=1:VGK\u003e=4P^K\\OD\\D008D\u003cgY[GfMjeM\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNf\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\u003ceaB7BV;\u003eIRKK' WHERE pk=1;` - -// TOASTed update -var update2Stmt = `UPDATE customers3 SET bool1=true WHERE bool1=false;` - -// update with pkey change -var update3Stmt = `UPDATE customers3 SET pk=2 WHERE pk=1;` -var deleteStmt = `DELETE FROM customers3 WHERE 1=1;` -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - '04:05:06.12345', -- TIME(5) - '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) - - -- SPATIAL TYPES - -# ST_GeomFromText('LINESTRING(0 0,1 2,2 4)'), -- LINESTRING_ GEOMETRY, -# ST_GeomFromText('POLYGON((0 0,10 0,10 10,0 10,0 0),(5 5,7 5,7 7,5 7, 5 5))'), -- POLYGON_ GEOMETRY, -# ST_GeomFromText('MULTIPOINT(0 0, 15 25, 45 65)'), -- MULTIPOINT_ GEOMETRY, -# ST_GeomFromText('MULTILINESTRING((12 12, 22 22), (19 19, 32 18))'), -- MULTILINESTRING_ GEOMETRY, -# ST_GeomFromText('MULTIPOLYGON(((0 0,11 0,12 11,0 9,0 0)),((3 5,7 4,4 7,7 7,3 5)))'), -- MULTIPOLYGON_ GEOMETRY, -# ST_GeomFromText('GEOMETRYCOLLECTION(POINT(3 2),LINESTRING(0 0,1 3,2 5,3 5,4 7))') -- GEOMETRYCOLLECTION_ GEOMETRY, -); -` - -func ReadTextFiles(paths []string, out []*string) error { - for index, path := range paths { - valArr, err := ioutil.ReadFile(yatest.SourcePath(path)) - if err != nil { - return xerrors.Errorf("unable to read file %s: %w", path, err) - } - val := string(valArr) - *out[index] = val - } - return nil -} - -func TestReplication(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - - //------------------------------------------------------------------------------ - - var canonizedDebeziumInsertKey = `` - var canonizedDebeziumInsertVal = `` - - var canonizedDebeziumUpdate1Key = `` - var canonizedDebeziumUpdate1Val = `` - - var canonizedDebeziumUpdate2Key = `` - var canonizedDebeziumUpdate2Val = `` - - var canonizedDebeziumUpdate30Key = `` - var canonizedDebeziumUpdate30Val = `` - - var canonizedDebeziumUpdate31Key = `` - var canonizedDebeziumUpdate31Val *string = nil - - var canonizedDebeziumUpdate32Key = `` - var canonizedDebeziumUpdate32Val = `` - - var canonizedDebeziumDelete0Key = `` - var canonizedDebeziumDelete0Val = `` - - var canonizedDebeziumDelete1Key = `` - var canonizedDebeziumDelete1Val *string = nil - - err := ReadTextFiles( - []string{ - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt", - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt", - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt", - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt", - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt", - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt", - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt", - - "transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt", - }, - []*string{ - &canonizedDebeziumInsertKey, - &canonizedDebeziumInsertVal, - - &canonizedDebeziumUpdate1Key, - &canonizedDebeziumUpdate1Val, - - &canonizedDebeziumUpdate2Key, - &canonizedDebeziumUpdate2Val, - - &canonizedDebeziumUpdate30Key, - &canonizedDebeziumUpdate30Val, - - &canonizedDebeziumUpdate31Key, - - &canonizedDebeziumUpdate32Key, - &canonizedDebeziumUpdate32Val, - - &canonizedDebeziumDelete0Key, - &canonizedDebeziumDelete0Val, - - &canonizedDebeziumDelete1Key, - }, - ) - require.NoError(t, err) - - fmt.Printf("canonizedDebeziumInsertKey=%s\n", canonizedDebeziumInsertKey) - fmt.Printf("canonizedDebeziumInsertVal=%s\n", canonizedDebeziumInsertVal) - - fmt.Printf("canonizedDebeziumUpdate1Key=%s\n", canonizedDebeziumUpdate1Key) - fmt.Printf("canonizedDebeziumUpdate1Val=%s\n", canonizedDebeziumUpdate1Val) - - fmt.Printf("canonizedDebeziumUpdate2Key=%s\n", canonizedDebeziumUpdate2Key) - fmt.Printf("canonizedDebeziumUpdate2Val=%s\n", canonizedDebeziumUpdate2Val) - - fmt.Printf("canonizedDebeziumUpdate30Key=%s\n", canonizedDebeziumUpdate30Key) - fmt.Printf("canonizedDebeziumUpdate30Val=%s\n", canonizedDebeziumUpdate30Val) - - fmt.Printf("canonizedDebeziumUpdate31Key=%s\n", canonizedDebeziumUpdate31Key) - - fmt.Printf("canonizedDebeziumUpdate32Key=%s\n", canonizedDebeziumUpdate32Key) - fmt.Printf("canonizedDebeziumUpdate32Val=%s\n", canonizedDebeziumUpdate32Val) - - fmt.Printf("canonizedDebeziumDelete0Key=%s\n", canonizedDebeziumDelete0Key) - fmt.Printf("canonizedDebeziumDelete0Val=%s\n", canonizedDebeziumDelete0Val) - - fmt.Printf("canonizedDebeziumDelete1Key=%s\n", canonizedDebeziumDelete1Key) - - //------------------------------------------------------------------------------ - // start replication - - sinker := &helpers.MockSink{} - target := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return sinker }, - Cleanup: server.DisabledCleanup, - } - transfer := helpers.MakeTransfer("fake", Source, &target, abstract.TransferTypeSnapshotAndIncrement) - - mutex := sync.Mutex{} - var changeItems []abstract.ChangeItem - sinker.PushCallback = func(input []abstract.ChangeItem) { - found := false - for _, el := range input { - if el.Table == "customers3" { - found = true - } - } - if !found { - return - } - //--- - mutex.Lock() - defer mutex.Unlock() - - for _, el := range input { - if el.Table != "customers3" { - continue - } - changeItems = append(changeItems, el) - } - } - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //----------------------------------------------------------------------------------------------------------------- - // execute SQL statements - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - _, err = db.Exec(update1Stmt) - require.NoError(t, err) - _, err = db.Exec(update2Stmt) - require.NoError(t, err) - _, err = db.Exec(update3Stmt) - require.NoError(t, err) - _, err = db.Exec(deleteStmt) - require.NoError(t, err) - - for { - time.Sleep(time.Second) - - mutex.Lock() - if len(changeItems) == 9 { - break - } - mutex.Unlock() - } - - require.Equal(t, changeItems[0].Kind, abstract.InitShardedTableLoad) - require.Equal(t, changeItems[1].Kind, abstract.InitTableLoad) - require.Equal(t, changeItems[2].Kind, abstract.DoneTableLoad) - require.Equal(t, changeItems[3].Kind, abstract.DoneShardedTableLoad) - require.Equal(t, changeItems[4].Kind, abstract.InsertKind) - require.Equal(t, changeItems[5].Kind, abstract.UpdateKind) - require.Equal(t, changeItems[6].Kind, abstract.UpdateKind) - require.Equal(t, changeItems[7].Kind, abstract.UpdateKind) - require.Equal(t, changeItems[8].Kind, abstract.DeleteKind) - - for i := range changeItems { - fmt.Printf("changeItem dump: %s\n", changeItems[i].ToJSONString()) - } - - //----------------------------------------------------------------------------------------------------------------- - // TM-4377 - - parseTimestamp := func(t *testing.T, timeStr string) time.Time { - timeVal, err := time.Parse("2006-01-02T15:04:05Z", timeStr) - require.NoError(t, err) - return timeVal - } - changeItems[5].ColumnValues[48] = parseTimestamp(t, "1999-01-01T00:00:01Z") - - changeItems[6].ColumnValues[48] = parseTimestamp(t, "1999-01-01T00:00:01Z") - for i := range changeItems[6].OldKeys.KeyNames { - if changeItems[6].OldKeys.KeyNames[i] == "timestamp_" { - changeItems[6].OldKeys.KeyValues[i] = parseTimestamp(t, "1999-01-01T00:00:01Z") - break - } - } - - changeItems[7].ColumnValues[48] = parseTimestamp(t, "1999-01-01T00:00:01Z") - for i := range changeItems[7].OldKeys.KeyNames { - if changeItems[7].OldKeys.KeyNames[i] == "timestamp_" { - changeItems[7].OldKeys.KeyValues[i] = parseTimestamp(t, "1999-01-01T00:00:01Z") - break - } - } - - changeItems[8].ColumnValues[48] = parseTimestamp(t, "1999-01-01T00:00:01Z") - for i := range changeItems[8].OldKeys.KeyNames { - if changeItems[8].OldKeys.KeyNames[i] == "timestamp_" { - changeItems[8].OldKeys.KeyValues[i] = parseTimestamp(t, "1999-01-01T00:00:01Z") - break - } - } - - //----------------------------------------------------------------------------------------------------------------- - - testSuite := []debeziumcommon.ChangeItemCanon{ - { - ChangeItem: &changeItems[4], - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumInsertKey, - DebeziumVal: &canonizedDebeziumInsertVal, - }}, - }, - { - ChangeItem: &changeItems[5], - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumUpdate1Key, - DebeziumVal: &canonizedDebeziumUpdate1Val, - }}, - }, - { - ChangeItem: &changeItems[6], - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumUpdate2Key, - DebeziumVal: &canonizedDebeziumUpdate2Val, - }}, - }, - { - ChangeItem: &changeItems[7], - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumUpdate30Key, - DebeziumVal: &canonizedDebeziumUpdate30Val, - }, { - DebeziumKey: canonizedDebeziumUpdate31Key, - DebeziumVal: canonizedDebeziumUpdate31Val, - }, { - DebeziumKey: canonizedDebeziumUpdate32Key, - DebeziumVal: &canonizedDebeziumUpdate32Val, - }}, - }, - { - ChangeItem: &changeItems[8], - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumDelete0Key, - DebeziumVal: &canonizedDebeziumDelete0Val, - }, { - DebeziumKey: canonizedDebeziumDelete1Key, - DebeziumVal: canonizedDebeziumDelete1Val, - }}, - }, - } - - testSuite = testutil.FixTestSuite(t, testSuite, "dbserver1", "source", "mysql") - - for _, testCase := range testSuite { - testutil.CheckCanonizedDebeziumEvent(t, testCase.ChangeItem, "dbserver1", "source", "mysql", false, testCase.DebeziumEvents) - } - - for i := range testSuite { - testSuite[i].ChangeItem = helpers.UnmarshalChangeItemStr(t, testSuite[i].ChangeItem.ToJSONString()) - } - - for _, testCase := range testSuite { - testutil.CheckCanonizedDebeziumEvent(t, testCase.ChangeItem, "dbserver1", "source", "mysql", false, testCase.DebeziumEvents) - } -} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/dump/dump.sql b/tests/e2e/mysql2mock/debezium/debezium_replication/dump/dump.sql deleted file mode 100644 index 5f4e580d..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/dump/dump.sql +++ /dev/null @@ -1,125 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), - time1 TIME(1), - time2 TIME(2), - time3 TIME(3), - time4 TIME(4), - time5 TIME(5), - time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- SPATIAL TYPES - - -- LINESTRING_ GEOMETRY, - -- POLYGON_ GEOMETRY, - -- MULTIPOINT_ GEOMETRY, - -- MULTILINESTRING_ GEOMETRY, - -- MULTIPOLYGON_ GEOMETRY, - -- GEOMETRYCOLLECTION_ GEOMETRY, - - -- - - primary key (pk) -) engine=innodb default charset=utf8; diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt deleted file mode 100644 index 50a892ca..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":1}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt deleted file mode 100644 index 3624c20f..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_0_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":null,"after":{"pk":1,"bool1":false,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"my-text","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660748870000,"snapshot":"false","db":"source","sequence":null,"table":"customers3","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":3332,"row":0,"thread":12,"query":null},"op":"c","ts_ms":1660748870868,"transaction":null}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt deleted file mode 100644 index 50a892ca..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":1}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt deleted file mode 100644 index 1a647842..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_1_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":{"pk":1,"bool1":false,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"my-text","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"after":{"pk":1,"bool1":false,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]u003c4SaNJTHkL@1?6YcDfu003eHI[862bUb4gT@ku003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^u003eu003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4u003cI_@d]u003eF1e]hj_XJII862[Nu003cj=bYAu003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6jau003e0UDDBb8h]65Cu003efCu003c[02jRT]bJu003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28Uu003eH2X\\]_u003cEE3@?U2_L67UV8FNQecS2Y=@6u003ehb1\\3F66UE[W9u003c]?HHu003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJu003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZeu003e@Au003e5u003cK\\d4QM:7:41B^_c\\FCI=u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[Tu003cIQI4S_gu003e;gf[BF_ENu003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4u003ccXRAY4HNX_BXiX3XPYMAWhU?0u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZKu003cu003e[=0W3Of;6;RFY=Q\\OK\\7[\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODju003cOK6gV=EMGC?\\Fu003cXaa_u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;u003eMZGu003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6gu003e7cU]M[u003c72cu003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:u003ea5a;ju003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6Wu003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fXu003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26u003c84==_9FJbjbEhQeOVu003eWDP4MV^W1_]=TeAa66jLObKGu003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jFu003ebGaJ2f;VBu003eG\\3u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcddu003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Yu003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[Bu003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZu003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?u003eku003ePUHD6u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9u003e=u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@fu003ciDV?6i0WVXju003c@ZPd5d\\5B]O?7h=C=8O:L:IR8Iu003e^6u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLEu003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgKu003e7UBbR58G?[X_O1b\\:[65u003eP9Z6u003c]S8=au003eb96I==_LhM@LN7=XbC]5cfi7RQu003e^GMUPS2]bu003e]DN?aUKNL^@RVu003cFTBh:Q[Q3E5VHbK?5=RTKIu003eggZZu003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09ADu003eVd?f9iGZ3@g5b^@Zi9db_0b5Pu003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJu003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[Hu003cUb4EE^u003ckWO7u003eR8fD9JQHRu003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5u003cBA\\3IVT@gG\\4u003cRRS459YROd=_H1OM=a_hdu003cSMLOd=S6^:eGu003ejPgQ4_^du003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBWu003cDa;\\Ni[ACu003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9u003eT12E?FZ9cYCLQbH[2Ou003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihEu003ehMVaDFu003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWDu003eHga5eW[E8u003c9jdYO7u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Qu003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NRu003eTK07=]7Ecdeju003cUju003cDe1Hu003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_Iu003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagLu003cSV@b[GVEU3Xh;R7u003cXeTNgNu003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhRu003e]@GIYf[L55gu003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:Su003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZGu003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9Heu003e1L[3u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[u003eCLdAe]6L2AD0aYHc5u003e=fM7hu003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_du003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]u003eKEu003ceau003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feMu003eLW5VIfJL:eQ4K3a1^WN5T=\\X=u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1Cu003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52gu003eTQQWYJ_@FAX\\]9jhu003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jKu003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6u003eBgESu003e5EaeOFeG:iu003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDOu003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?u003eRQ]5Z9jA@Y9V1ZI6TDkCu003eNZ_f_DRu003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3Ou003eFWu003cJ6X?IiJu003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;fu003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHIu003eI]gBSu003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34u003eh_2@i3kd02Gu003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8u003c^U7Hk]7Q6P:QZS;Ge@:u003cu003cfT6PK7j4?;cdC@c5GI:gS[Wu003cf26;u003cBG7fMXFTWJcbB\\9QTu003eh3HdV8Pb3Rhu003e^?Ue:7RP[=jT4AEu003ebiL_1dYW1u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dEu003cA9LXQbECIc2Mu003c^Iu003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;cu003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_u003c_F9Pu003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2u003e=R4U3W1G;u003chN\\WFO_=DDu003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5Tu003eY?bFOMZOu003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcju003efaaP8P4KDVSCiQ=2u003c=Ef:u003ePu003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_u003c@5Z8fDPJAE8DcGUIb8Cu003c_L7XhP=u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8u003e]LWu003ee^bu003e?0G9Ieu003cu003c@UT4e9u003cGM_jME7[6TFEN:u003c\\Hu003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]u003cL42d\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[u003eEJQi8j;]L5CILgXdR_u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLAu003cKHA:\\[CW7SRYVhE1[MDu003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8ku003eQb]UVVZ:18fe_8M?\\?u003eu003eLf4QSG@jO@u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkUu003cR]Ofg:TNGW0Lu003ePOC_CPu003e^PI[aZ:KY^V@Q;;ME_k\\K0u003eYP]1D5QSc51SfZ]FIP1Y6u003cdRQXRC8RP7BaKGG2?L3bG]S];8_du003e0]RJGeQiJG5\\=O8TRG5Uu003eLGau003eRi2Ku003c3=1TVHN=FhTJYajbIPu003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQu003e93HU2ig?7u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1Ju003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^Gu003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1au003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZBu003cAu003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iKu003e@^u003e[4u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVdeu003cUVVNH2EJ^=ALOFKUX:^u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2Su003cX?9bC7Ebc5V5E]u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SCu003eu003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Yu003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^u003ceM8?j]NZai4u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3u003eI=?@fu003cG349NMId8[T^@Sfu003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@du003ccu003cMhS3K;bu003eZbHAf[GKME9igTY7iVFbau003e4D;WFVb=dQ4Abj2u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Zu003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[u003e@TM9eOu003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhcu003c@=gPHLhQFDC@:Tu003cREdYu003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1adu003cIiK1O7fbD[7[u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?u003e=FFMHPSBf8:\\XRZ91D:2D[1Yu003eX\\bfj4BEQZe:1Au003cQj^@7SAK]C_NCM\\0u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4u003e2u003e4X:9JYPXku003eX_?;DAfLu003ec?HFu003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_u003e1u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aEu003cY^MPdu003e6M^iNNe=P6i6Lf::P6ebjX;u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6Au003c93u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Uau003c8@j5eu003eVA76=g2=gD4V1eYF0bZd0EZu003cMk2M4g[Z=baJ]cVYu003c[D=U2RUdBNdW=69=8UB4E1@u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;fu003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;Du003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4iau003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fau003e:Vj=BR7EW0_hV4=]DaSeQu003c?8]?9X4GbZF41h;FSu003c9Pa=^SQTu003cL:GAIP3XX[\\4RKJVLFabj20Ocu003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\Wu003cHg9FWdu003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:Su003eSu003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BHu003e67u003eWu003cQNZRKDH@]_j^M_AV9g4u003chIFu003eaSDhbj9GMdjh=F=j:u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaSu003eO]caAKiu003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=Ru003cWkCu003c^KSgbI7?aGVaRkbA2?_Raf^u003e9DID]07u003cS431;BaRhX:hNJj]u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6u003cN?Ju003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWcu003e8]u003eU2:HGATaUBPGu003c\\c0aX@_D;_EOK=]Sjk=1:VGKu003e=4P^K\\OD\\D008Du003cgY[GfMjeMu003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNfu003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;u003ebKICA@L3VQ^BG2cZ;Vj@3Jjju003eFA6=LD4g]G=3c@YI305cO@ONPQhNPu003ceaB7BV;u003eIRKK","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660748886000,"snapshot":"false","db":"source","sequence":null,"table":"customers3","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":4207,"row":0,"thread":12,"query":null},"op":"u","ts_ms":1660748886329,"transaction":null}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt deleted file mode 100644 index 50a892ca..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":1}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt deleted file mode 100644 index e01cd5bd..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_2_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":{"pk":1,"bool1":false,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]u003c4SaNJTHkL@1?6YcDfu003eHI[862bUb4gT@ku003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^u003eu003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4u003cI_@d]u003eF1e]hj_XJII862[Nu003cj=bYAu003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6jau003e0UDDBb8h]65Cu003efCu003c[02jRT]bJu003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28Uu003eH2X\\]_u003cEE3@?U2_L67UV8FNQecS2Y=@6u003ehb1\\3F66UE[W9u003c]?HHu003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJu003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZeu003e@Au003e5u003cK\\d4QM:7:41B^_c\\FCI=u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[Tu003cIQI4S_gu003e;gf[BF_ENu003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4u003ccXRAY4HNX_BXiX3XPYMAWhU?0u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZKu003cu003e[=0W3Of;6;RFY=Q\\OK\\7[\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODju003cOK6gV=EMGC?\\Fu003cXaa_u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;u003eMZGu003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6gu003e7cU]M[u003c72cu003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:u003ea5a;ju003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6Wu003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fXu003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26u003c84==_9FJbjbEhQeOVu003eWDP4MV^W1_]=TeAa66jLObKGu003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jFu003ebGaJ2f;VBu003eG\\3u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcddu003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Yu003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[Bu003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZu003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?u003eku003ePUHD6u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9u003e=u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@fu003ciDV?6i0WVXju003c@ZPd5d\\5B]O?7h=C=8O:L:IR8Iu003e^6u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLEu003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgKu003e7UBbR58G?[X_O1b\\:[65u003eP9Z6u003c]S8=au003eb96I==_LhM@LN7=XbC]5cfi7RQu003e^GMUPS2]bu003e]DN?aUKNL^@RVu003cFTBh:Q[Q3E5VHbK?5=RTKIu003eggZZu003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09ADu003eVd?f9iGZ3@g5b^@Zi9db_0b5Pu003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJu003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[Hu003cUb4EE^u003ckWO7u003eR8fD9JQHRu003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5u003cBA\\3IVT@gG\\4u003cRRS459YROd=_H1OM=a_hdu003cSMLOd=S6^:eGu003ejPgQ4_^du003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBWu003cDa;\\Ni[ACu003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9u003eT12E?FZ9cYCLQbH[2Ou003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihEu003ehMVaDFu003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWDu003eHga5eW[E8u003c9jdYO7u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Qu003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NRu003eTK07=]7Ecdeju003cUju003cDe1Hu003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_Iu003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagLu003cSV@b[GVEU3Xh;R7u003cXeTNgNu003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhRu003e]@GIYf[L55gu003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:Su003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZGu003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9Heu003e1L[3u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[u003eCLdAe]6L2AD0aYHc5u003e=fM7hu003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_du003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]u003eKEu003ceau003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feMu003eLW5VIfJL:eQ4K3a1^WN5T=\\X=u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1Cu003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52gu003eTQQWYJ_@FAX\\]9jhu003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jKu003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6u003eBgESu003e5EaeOFeG:iu003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDOu003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?u003eRQ]5Z9jA@Y9V1ZI6TDkCu003eNZ_f_DRu003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3Ou003eFWu003cJ6X?IiJu003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;fu003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHIu003eI]gBSu003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34u003eh_2@i3kd02Gu003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8u003c^U7Hk]7Q6P:QZS;Ge@:u003cu003cfT6PK7j4?;cdC@c5GI:gS[Wu003cf26;u003cBG7fMXFTWJcbB\\9QTu003eh3HdV8Pb3Rhu003e^?Ue:7RP[=jT4AEu003ebiL_1dYW1u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dEu003cA9LXQbECIc2Mu003c^Iu003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;cu003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_u003c_F9Pu003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2u003e=R4U3W1G;u003chN\\WFO_=DDu003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5Tu003eY?bFOMZOu003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcju003efaaP8P4KDVSCiQ=2u003c=Ef:u003ePu003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_u003c@5Z8fDPJAE8DcGUIb8Cu003c_L7XhP=u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8u003e]LWu003ee^bu003e?0G9Ieu003cu003c@UT4e9u003cGM_jME7[6TFEN:u003c\\Hu003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]u003cL42d\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[u003eEJQi8j;]L5CILgXdR_u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLAu003cKHA:\\[CW7SRYVhE1[MDu003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8ku003eQb]UVVZ:18fe_8M?\\?u003eu003eLf4QSG@jO@u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkUu003cR]Ofg:TNGW0Lu003ePOC_CPu003e^PI[aZ:KY^V@Q;;ME_k\\K0u003eYP]1D5QSc51SfZ]FIP1Y6u003cdRQXRC8RP7BaKGG2?L3bG]S];8_du003e0]RJGeQiJG5\\=O8TRG5Uu003eLGau003eRi2Ku003c3=1TVHN=FhTJYajbIPu003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQu003e93HU2ig?7u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1Ju003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^Gu003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1au003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZBu003cAu003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iKu003e@^u003e[4u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVdeu003cUVVNH2EJ^=ALOFKUX:^u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2Su003cX?9bC7Ebc5V5E]u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SCu003eu003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Yu003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^u003ceM8?j]NZai4u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3u003eI=?@fu003cG349NMId8[T^@Sfu003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@du003ccu003cMhS3K;bu003eZbHAf[GKME9igTY7iVFbau003e4D;WFVb=dQ4Abj2u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Zu003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[u003e@TM9eOu003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhcu003c@=gPHLhQFDC@:Tu003cREdYu003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1adu003cIiK1O7fbD[7[u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?u003e=FFMHPSBf8:\\XRZ91D:2D[1Yu003eX\\bfj4BEQZe:1Au003cQj^@7SAK]C_NCM\\0u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4u003e2u003e4X:9JYPXku003eX_?;DAfLu003ec?HFu003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_u003e1u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aEu003cY^MPdu003e6M^iNNe=P6i6Lf::P6ebjX;u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6Au003c93u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Uau003c8@j5eu003eVA76=g2=gD4V1eYF0bZd0EZu003cMk2M4g[Z=baJ]cVYu003c[D=U2RUdBNdW=69=8UB4E1@u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;fu003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;Du003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4iau003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fau003e:Vj=BR7EW0_hV4=]DaSeQu003c?8]?9X4GbZF41h;FSu003c9Pa=^SQTu003cL:GAIP3XX[\\4RKJVLFabj20Ocu003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\Wu003cHg9FWdu003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:Su003eSu003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BHu003e67u003eWu003cQNZRKDH@]_j^M_AV9g4u003chIFu003eaSDhbj9GMdjh=F=j:u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaSu003eO]caAKiu003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=Ru003cWkCu003c^KSgbI7?aGVaRkbA2?_Raf^u003e9DID]07u003cS431;BaRhX:hNJj]u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6u003cN?Ju003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWcu003e8]u003eU2:HGATaUBPGu003c\\c0aX@_D;_EOK=]Sjk=1:VGKu003e=4P^K\\OD\\D008Du003cgY[GfMjeMu003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNfu003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;u003ebKICA@L3VQ^BG2cZ;Vj@3Jjju003eFA6=LD4g]G=3c@YI305cO@ONPQhNPu003ceaB7BV;u003eIRKK","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"after":{"pk":1,"bool1":true,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]u003c4SaNJTHkL@1?6YcDfu003eHI[862bUb4gT@ku003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^u003eu003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4u003cI_@d]u003eF1e]hj_XJII862[Nu003cj=bYAu003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6jau003e0UDDBb8h]65Cu003efCu003c[02jRT]bJu003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28Uu003eH2X\\]_u003cEE3@?U2_L67UV8FNQecS2Y=@6u003ehb1\\3F66UE[W9u003c]?HHu003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJu003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZeu003e@Au003e5u003cK\\d4QM:7:41B^_c\\FCI=u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[Tu003cIQI4S_gu003e;gf[BF_ENu003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4u003ccXRAY4HNX_BXiX3XPYMAWhU?0u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZKu003cu003e[=0W3Of;6;RFY=Q\\OK\\7[\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODju003cOK6gV=EMGC?\\Fu003cXaa_u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;u003eMZGu003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6gu003e7cU]M[u003c72cu003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:u003ea5a;ju003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6Wu003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fXu003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26u003c84==_9FJbjbEhQeOVu003eWDP4MV^W1_]=TeAa66jLObKGu003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jFu003ebGaJ2f;VBu003eG\\3u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcddu003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Yu003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[Bu003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZu003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?u003eku003ePUHD6u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9u003e=u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@fu003ciDV?6i0WVXju003c@ZPd5d\\5B]O?7h=C=8O:L:IR8Iu003e^6u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLEu003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgKu003e7UBbR58G?[X_O1b\\:[65u003eP9Z6u003c]S8=au003eb96I==_LhM@LN7=XbC]5cfi7RQu003e^GMUPS2]bu003e]DN?aUKNL^@RVu003cFTBh:Q[Q3E5VHbK?5=RTKIu003eggZZu003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09ADu003eVd?f9iGZ3@g5b^@Zi9db_0b5Pu003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJu003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[Hu003cUb4EE^u003ckWO7u003eR8fD9JQHRu003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5u003cBA\\3IVT@gG\\4u003cRRS459YROd=_H1OM=a_hdu003cSMLOd=S6^:eGu003ejPgQ4_^du003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBWu003cDa;\\Ni[ACu003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9u003eT12E?FZ9cYCLQbH[2Ou003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihEu003ehMVaDFu003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWDu003eHga5eW[E8u003c9jdYO7u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Qu003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NRu003eTK07=]7Ecdeju003cUju003cDe1Hu003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_Iu003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagLu003cSV@b[GVEU3Xh;R7u003cXeTNgNu003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhRu003e]@GIYf[L55gu003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:Su003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZGu003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9Heu003e1L[3u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[u003eCLdAe]6L2AD0aYHc5u003e=fM7hu003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_du003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]u003eKEu003ceau003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feMu003eLW5VIfJL:eQ4K3a1^WN5T=\\X=u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1Cu003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52gu003eTQQWYJ_@FAX\\]9jhu003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jKu003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6u003eBgESu003e5EaeOFeG:iu003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDOu003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?u003eRQ]5Z9jA@Y9V1ZI6TDkCu003eNZ_f_DRu003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3Ou003eFWu003cJ6X?IiJu003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;fu003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHIu003eI]gBSu003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34u003eh_2@i3kd02Gu003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8u003c^U7Hk]7Q6P:QZS;Ge@:u003cu003cfT6PK7j4?;cdC@c5GI:gS[Wu003cf26;u003cBG7fMXFTWJcbB\\9QTu003eh3HdV8Pb3Rhu003e^?Ue:7RP[=jT4AEu003ebiL_1dYW1u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dEu003cA9LXQbECIc2Mu003c^Iu003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;cu003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_u003c_F9Pu003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2u003e=R4U3W1G;u003chN\\WFO_=DDu003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5Tu003eY?bFOMZOu003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcju003efaaP8P4KDVSCiQ=2u003c=Ef:u003ePu003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_u003c@5Z8fDPJAE8DcGUIb8Cu003c_L7XhP=u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8u003e]LWu003ee^bu003e?0G9Ieu003cu003c@UT4e9u003cGM_jME7[6TFEN:u003c\\Hu003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]u003cL42d\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[u003eEJQi8j;]L5CILgXdR_u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLAu003cKHA:\\[CW7SRYVhE1[MDu003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8ku003eQb]UVVZ:18fe_8M?\\?u003eu003eLf4QSG@jO@u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkUu003cR]Ofg:TNGW0Lu003ePOC_CPu003e^PI[aZ:KY^V@Q;;ME_k\\K0u003eYP]1D5QSc51SfZ]FIP1Y6u003cdRQXRC8RP7BaKGG2?L3bG]S];8_du003e0]RJGeQiJG5\\=O8TRG5Uu003eLGau003eRi2Ku003c3=1TVHN=FhTJYajbIPu003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQu003e93HU2ig?7u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1Ju003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^Gu003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1au003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZBu003cAu003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iKu003e@^u003e[4u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVdeu003cUVVNH2EJ^=ALOFKUX:^u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2Su003cX?9bC7Ebc5V5E]u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SCu003eu003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Yu003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^u003ceM8?j]NZai4u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3u003eI=?@fu003cG349NMId8[T^@Sfu003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@du003ccu003cMhS3K;bu003eZbHAf[GKME9igTY7iVFbau003e4D;WFVb=dQ4Abj2u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Zu003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[u003e@TM9eOu003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhcu003c@=gPHLhQFDC@:Tu003cREdYu003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1adu003cIiK1O7fbD[7[u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?u003e=FFMHPSBf8:\\XRZ91D:2D[1Yu003eX\\bfj4BEQZe:1Au003cQj^@7SAK]C_NCM\\0u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4u003e2u003e4X:9JYPXku003eX_?;DAfLu003ec?HFu003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_u003e1u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aEu003cY^MPdu003e6M^iNNe=P6i6Lf::P6ebjX;u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6Au003c93u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Uau003c8@j5eu003eVA76=g2=gD4V1eYF0bZd0EZu003cMk2M4g[Z=baJ]cVYu003c[D=U2RUdBNdW=69=8UB4E1@u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;fu003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;Du003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4iau003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fau003e:Vj=BR7EW0_hV4=]DaSeQu003c?8]?9X4GbZF41h;FSu003c9Pa=^SQTu003cL:GAIP3XX[\\4RKJVLFabj20Ocu003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\Wu003cHg9FWdu003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:Su003eSu003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BHu003e67u003eWu003cQNZRKDH@]_j^M_AV9g4u003chIFu003eaSDhbj9GMdjh=F=j:u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaSu003eO]caAKiu003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=Ru003cWkCu003c^KSgbI7?aGVaRkbA2?_Raf^u003e9DID]07u003cS431;BaRhX:hNJj]u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6u003cN?Ju003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWcu003e8]u003eU2:HGATaUBPGu003c\\c0aX@_D;_EOK=]Sjk=1:VGKu003e=4P^K\\OD\\D008Du003cgY[GfMjeMu003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNfu003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;u003ebKICA@L3VQ^BG2cZ;Vj@3Jjju003eFA6=LD4g]G=3c@YI305cO@ONPQhNPu003ceaB7BV;u003eIRKK","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660748898000,"snapshot":"false","db":"source","sequence":null,"table":"customers3","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":16533,"row":0,"thread":12,"query":null},"op":"u","ts_ms":1660748898791,"transaction":null}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt deleted file mode 100644 index 50a892ca..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":1}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt deleted file mode 100644 index f20d6c1e..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_3_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":{"pk":1,"bool1":true,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]u003c4SaNJTHkL@1?6YcDfu003eHI[862bUb4gT@ku003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^u003eu003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4u003cI_@d]u003eF1e]hj_XJII862[Nu003cj=bYAu003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6jau003e0UDDBb8h]65Cu003efCu003c[02jRT]bJu003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28Uu003eH2X\\]_u003cEE3@?U2_L67UV8FNQecS2Y=@6u003ehb1\\3F66UE[W9u003c]?HHu003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJu003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZeu003e@Au003e5u003cK\\d4QM:7:41B^_c\\FCI=u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[Tu003cIQI4S_gu003e;gf[BF_ENu003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4u003ccXRAY4HNX_BXiX3XPYMAWhU?0u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZKu003cu003e[=0W3Of;6;RFY=Q\\OK\\7[\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODju003cOK6gV=EMGC?\\Fu003cXaa_u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;u003eMZGu003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6gu003e7cU]M[u003c72cu003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:u003ea5a;ju003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6Wu003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fXu003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26u003c84==_9FJbjbEhQeOVu003eWDP4MV^W1_]=TeAa66jLObKGu003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jFu003ebGaJ2f;VBu003eG\\3u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcddu003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Yu003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[Bu003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZu003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?u003eku003ePUHD6u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9u003e=u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@fu003ciDV?6i0WVXju003c@ZPd5d\\5B]O?7h=C=8O:L:IR8Iu003e^6u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLEu003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgKu003e7UBbR58G?[X_O1b\\:[65u003eP9Z6u003c]S8=au003eb96I==_LhM@LN7=XbC]5cfi7RQu003e^GMUPS2]bu003e]DN?aUKNL^@RVu003cFTBh:Q[Q3E5VHbK?5=RTKIu003eggZZu003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09ADu003eVd?f9iGZ3@g5b^@Zi9db_0b5Pu003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJu003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[Hu003cUb4EE^u003ckWO7u003eR8fD9JQHRu003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5u003cBA\\3IVT@gG\\4u003cRRS459YROd=_H1OM=a_hdu003cSMLOd=S6^:eGu003ejPgQ4_^du003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBWu003cDa;\\Ni[ACu003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9u003eT12E?FZ9cYCLQbH[2Ou003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihEu003ehMVaDFu003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWDu003eHga5eW[E8u003c9jdYO7u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Qu003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NRu003eTK07=]7Ecdeju003cUju003cDe1Hu003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_Iu003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagLu003cSV@b[GVEU3Xh;R7u003cXeTNgNu003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhRu003e]@GIYf[L55gu003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:Su003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZGu003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9Heu003e1L[3u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[u003eCLdAe]6L2AD0aYHc5u003e=fM7hu003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_du003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]u003eKEu003ceau003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feMu003eLW5VIfJL:eQ4K3a1^WN5T=\\X=u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1Cu003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52gu003eTQQWYJ_@FAX\\]9jhu003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jKu003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6u003eBgESu003e5EaeOFeG:iu003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDOu003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?u003eRQ]5Z9jA@Y9V1ZI6TDkCu003eNZ_f_DRu003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3Ou003eFWu003cJ6X?IiJu003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;fu003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHIu003eI]gBSu003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34u003eh_2@i3kd02Gu003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8u003c^U7Hk]7Q6P:QZS;Ge@:u003cu003cfT6PK7j4?;cdC@c5GI:gS[Wu003cf26;u003cBG7fMXFTWJcbB\\9QTu003eh3HdV8Pb3Rhu003e^?Ue:7RP[=jT4AEu003ebiL_1dYW1u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dEu003cA9LXQbECIc2Mu003c^Iu003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;cu003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_u003c_F9Pu003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2u003e=R4U3W1G;u003chN\\WFO_=DDu003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5Tu003eY?bFOMZOu003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcju003efaaP8P4KDVSCiQ=2u003c=Ef:u003ePu003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_u003c@5Z8fDPJAE8DcGUIb8Cu003c_L7XhP=u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8u003e]LWu003ee^bu003e?0G9Ieu003cu003c@UT4e9u003cGM_jME7[6TFEN:u003c\\Hu003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]u003cL42d\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[u003eEJQi8j;]L5CILgXdR_u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLAu003cKHA:\\[CW7SRYVhE1[MDu003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8ku003eQb]UVVZ:18fe_8M?\\?u003eu003eLf4QSG@jO@u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkUu003cR]Ofg:TNGW0Lu003ePOC_CPu003e^PI[aZ:KY^V@Q;;ME_k\\K0u003eYP]1D5QSc51SfZ]FIP1Y6u003cdRQXRC8RP7BaKGG2?L3bG]S];8_du003e0]RJGeQiJG5\\=O8TRG5Uu003eLGau003eRi2Ku003c3=1TVHN=FhTJYajbIPu003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQu003e93HU2ig?7u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1Ju003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^Gu003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1au003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZBu003cAu003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iKu003e@^u003e[4u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVdeu003cUVVNH2EJ^=ALOFKUX:^u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2Su003cX?9bC7Ebc5V5E]u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SCu003eu003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Yu003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^u003ceM8?j]NZai4u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3u003eI=?@fu003cG349NMId8[T^@Sfu003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@du003ccu003cMhS3K;bu003eZbHAf[GKME9igTY7iVFbau003e4D;WFVb=dQ4Abj2u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Zu003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[u003e@TM9eOu003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhcu003c@=gPHLhQFDC@:Tu003cREdYu003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1adu003cIiK1O7fbD[7[u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?u003e=FFMHPSBf8:\\XRZ91D:2D[1Yu003eX\\bfj4BEQZe:1Au003cQj^@7SAK]C_NCM\\0u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4u003e2u003e4X:9JYPXku003eX_?;DAfLu003ec?HFu003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_u003e1u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aEu003cY^MPdu003e6M^iNNe=P6i6Lf::P6ebjX;u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6Au003c93u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Uau003c8@j5eu003eVA76=g2=gD4V1eYF0bZd0EZu003cMk2M4g[Z=baJ]cVYu003c[D=U2RUdBNdW=69=8UB4E1@u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;fu003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;Du003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4iau003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fau003e:Vj=BR7EW0_hV4=]DaSeQu003c?8]?9X4GbZF41h;FSu003c9Pa=^SQTu003cL:GAIP3XX[\\4RKJVLFabj20Ocu003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\Wu003cHg9FWdu003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:Su003eSu003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BHu003e67u003eWu003cQNZRKDH@]_j^M_AV9g4u003chIFu003eaSDhbj9GMdjh=F=j:u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaSu003eO]caAKiu003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=Ru003cWkCu003c^KSgbI7?aGVaRkbA2?_Raf^u003e9DID]07u003cS431;BaRhX:hNJj]u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6u003cN?Ju003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWcu003e8]u003eU2:HGATaUBPGu003c\\c0aX@_D;_EOK=]Sjk=1:VGKu003e=4P^K\\OD\\D008Du003cgY[GfMjeMu003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNfu003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;u003ebKICA@L3VQ^BG2cZ;Vj@3Jjju003eFA6=LD4g]G=3c@YI305cO@ONPQhNPu003ceaB7BV;u003eIRKK","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"after":null,"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660748909000,"snapshot":"false","db":"source","sequence":null,"table":"customers3","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":39907,"row":0,"thread":12,"query":null},"op":"d","ts_ms":1660748909903,"transaction":null}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt deleted file mode 100644 index 50a892ca..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_4_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":1}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt deleted file mode 100644 index 79e8e5de..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":2}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt deleted file mode 100644 index 47a95201..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_5_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":null,"after":{"pk":2,"bool1":true,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]u003c4SaNJTHkL@1?6YcDfu003eHI[862bUb4gT@ku003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^u003eu003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4u003cI_@d]u003eF1e]hj_XJII862[Nu003cj=bYAu003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6jau003e0UDDBb8h]65Cu003efCu003c[02jRT]bJu003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28Uu003eH2X\\]_u003cEE3@?U2_L67UV8FNQecS2Y=@6u003ehb1\\3F66UE[W9u003c]?HHu003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJu003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZeu003e@Au003e5u003cK\\d4QM:7:41B^_c\\FCI=u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[Tu003cIQI4S_gu003e;gf[BF_ENu003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4u003ccXRAY4HNX_BXiX3XPYMAWhU?0u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZKu003cu003e[=0W3Of;6;RFY=Q\\OK\\7[\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODju003cOK6gV=EMGC?\\Fu003cXaa_u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;u003eMZGu003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6gu003e7cU]M[u003c72cu003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:u003ea5a;ju003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6Wu003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fXu003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26u003c84==_9FJbjbEhQeOVu003eWDP4MV^W1_]=TeAa66jLObKGu003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jFu003ebGaJ2f;VBu003eG\\3u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcddu003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Yu003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[Bu003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZu003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?u003eku003ePUHD6u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9u003e=u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@fu003ciDV?6i0WVXju003c@ZPd5d\\5B]O?7h=C=8O:L:IR8Iu003e^6u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLEu003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgKu003e7UBbR58G?[X_O1b\\:[65u003eP9Z6u003c]S8=au003eb96I==_LhM@LN7=XbC]5cfi7RQu003e^GMUPS2]bu003e]DN?aUKNL^@RVu003cFTBh:Q[Q3E5VHbK?5=RTKIu003eggZZu003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09ADu003eVd?f9iGZ3@g5b^@Zi9db_0b5Pu003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJu003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[Hu003cUb4EE^u003ckWO7u003eR8fD9JQHRu003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5u003cBA\\3IVT@gG\\4u003cRRS459YROd=_H1OM=a_hdu003cSMLOd=S6^:eGu003ejPgQ4_^du003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBWu003cDa;\\Ni[ACu003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9u003eT12E?FZ9cYCLQbH[2Ou003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihEu003ehMVaDFu003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWDu003eHga5eW[E8u003c9jdYO7u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Qu003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NRu003eTK07=]7Ecdeju003cUju003cDe1Hu003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_Iu003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagLu003cSV@b[GVEU3Xh;R7u003cXeTNgNu003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhRu003e]@GIYf[L55gu003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:Su003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZGu003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9Heu003e1L[3u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[u003eCLdAe]6L2AD0aYHc5u003e=fM7hu003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_du003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]u003eKEu003ceau003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feMu003eLW5VIfJL:eQ4K3a1^WN5T=\\X=u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1Cu003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52gu003eTQQWYJ_@FAX\\]9jhu003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jKu003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6u003eBgESu003e5EaeOFeG:iu003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDOu003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?u003eRQ]5Z9jA@Y9V1ZI6TDkCu003eNZ_f_DRu003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3Ou003eFWu003cJ6X?IiJu003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;fu003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHIu003eI]gBSu003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34u003eh_2@i3kd02Gu003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8u003c^U7Hk]7Q6P:QZS;Ge@:u003cu003cfT6PK7j4?;cdC@c5GI:gS[Wu003cf26;u003cBG7fMXFTWJcbB\\9QTu003eh3HdV8Pb3Rhu003e^?Ue:7RP[=jT4AEu003ebiL_1dYW1u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dEu003cA9LXQbECIc2Mu003c^Iu003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;cu003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_u003c_F9Pu003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2u003e=R4U3W1G;u003chN\\WFO_=DDu003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5Tu003eY?bFOMZOu003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcju003efaaP8P4KDVSCiQ=2u003c=Ef:u003ePu003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_u003c@5Z8fDPJAE8DcGUIb8Cu003c_L7XhP=u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8u003e]LWu003ee^bu003e?0G9Ieu003cu003c@UT4e9u003cGM_jME7[6TFEN:u003c\\Hu003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]u003cL42d\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[u003eEJQi8j;]L5CILgXdR_u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLAu003cKHA:\\[CW7SRYVhE1[MDu003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8ku003eQb]UVVZ:18fe_8M?\\?u003eu003eLf4QSG@jO@u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkUu003cR]Ofg:TNGW0Lu003ePOC_CPu003e^PI[aZ:KY^V@Q;;ME_k\\K0u003eYP]1D5QSc51SfZ]FIP1Y6u003cdRQXRC8RP7BaKGG2?L3bG]S];8_du003e0]RJGeQiJG5\\=O8TRG5Uu003eLGau003eRi2Ku003c3=1TVHN=FhTJYajbIPu003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQu003e93HU2ig?7u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1Ju003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^Gu003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1au003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZBu003cAu003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iKu003e@^u003e[4u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVdeu003cUVVNH2EJ^=ALOFKUX:^u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2Su003cX?9bC7Ebc5V5E]u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SCu003eu003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Yu003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^u003ceM8?j]NZai4u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3u003eI=?@fu003cG349NMId8[T^@Sfu003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@du003ccu003cMhS3K;bu003eZbHAf[GKME9igTY7iVFbau003e4D;WFVb=dQ4Abj2u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Zu003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[u003e@TM9eOu003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhcu003c@=gPHLhQFDC@:Tu003cREdYu003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1adu003cIiK1O7fbD[7[u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?u003e=FFMHPSBf8:\\XRZ91D:2D[1Yu003eX\\bfj4BEQZe:1Au003cQj^@7SAK]C_NCM\\0u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4u003e2u003e4X:9JYPXku003eX_?;DAfLu003ec?HFu003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_u003e1u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aEu003cY^MPdu003e6M^iNNe=P6i6Lf::P6ebjX;u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6Au003c93u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Uau003c8@j5eu003eVA76=g2=gD4V1eYF0bZd0EZu003cMk2M4g[Z=baJ]cVYu003c[D=U2RUdBNdW=69=8UB4E1@u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;fu003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;Du003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4iau003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fau003e:Vj=BR7EW0_hV4=]DaSeQu003c?8]?9X4GbZF41h;FSu003c9Pa=^SQTu003cL:GAIP3XX[\\4RKJVLFabj20Ocu003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\Wu003cHg9FWdu003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:Su003eSu003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BHu003e67u003eWu003cQNZRKDH@]_j^M_AV9g4u003chIFu003eaSDhbj9GMdjh=F=j:u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaSu003eO]caAKiu003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=Ru003cWkCu003c^KSgbI7?aGVaRkbA2?_Raf^u003e9DID]07u003cS431;BaRhX:hNJj]u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6u003cN?Ju003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWcu003e8]u003eU2:HGATaUBPGu003c\\c0aX@_D;_EOK=]Sjk=1:VGKu003e=4P^K\\OD\\D008Du003cgY[GfMjeMu003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNfu003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;u003ebKICA@L3VQ^BG2cZ;Vj@3Jjju003eFA6=LD4g]G=3c@YI305cO@ONPQhNPu003ceaB7BV;u003eIRKK","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660748909000,"snapshot":"false","db":"source","sequence":null,"table":"customers3","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":39907,"row":0,"thread":12,"query":null},"op":"c","ts_ms":1660748909903,"transaction":null}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt deleted file mode 100644 index 79e8e5de..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":2}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt deleted file mode 100644 index 9be260da..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_6_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"},{"type":"boolean","optional":true,"field":"bool1"},{"type":"boolean","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"float","optional":true,"field":"real_"},{"type":"float","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":{"pk":2,"bool1":true,"bool2":true,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]u003c4SaNJTHkL@1?6YcDfu003eHI[862bUb4gT@ku003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^u003eu003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4u003cI_@d]u003eF1e]hj_XJII862[Nu003cj=bYAu003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6jau003e0UDDBb8h]65Cu003efCu003c[02jRT]bJu003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28Uu003eH2X\\]_u003cEE3@?U2_L67UV8FNQecS2Y=@6u003ehb1\\3F66UE[W9u003c]?HHu003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJu003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZeu003e@Au003e5u003cK\\d4QM:7:41B^_c\\FCI=u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[Tu003cIQI4S_gu003e;gf[BF_ENu003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4u003ccXRAY4HNX_BXiX3XPYMAWhU?0u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZKu003cu003e[=0W3Of;6;RFY=Q\\OK\\7[\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODju003cOK6gV=EMGC?\\Fu003cXaa_u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;u003eMZGu003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6gu003e7cU]M[u003c72cu003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:u003ea5a;ju003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6Wu003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fXu003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26u003c84==_9FJbjbEhQeOVu003eWDP4MV^W1_]=TeAa66jLObKGu003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jFu003ebGaJ2f;VBu003eG\\3u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcddu003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Yu003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[Bu003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZu003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?u003eku003ePUHD6u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9u003e=u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@fu003ciDV?6i0WVXju003c@ZPd5d\\5B]O?7h=C=8O:L:IR8Iu003e^6u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLEu003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgKu003e7UBbR58G?[X_O1b\\:[65u003eP9Z6u003c]S8=au003eb96I==_LhM@LN7=XbC]5cfi7RQu003e^GMUPS2]bu003e]DN?aUKNL^@RVu003cFTBh:Q[Q3E5VHbK?5=RTKIu003eggZZu003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09ADu003eVd?f9iGZ3@g5b^@Zi9db_0b5Pu003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJu003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[Hu003cUb4EE^u003ckWO7u003eR8fD9JQHRu003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5u003cBA\\3IVT@gG\\4u003cRRS459YROd=_H1OM=a_hdu003cSMLOd=S6^:eGu003ejPgQ4_^du003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBWu003cDa;\\Ni[ACu003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9u003eT12E?FZ9cYCLQbH[2Ou003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihEu003ehMVaDFu003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWDu003eHga5eW[E8u003c9jdYO7u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Qu003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NRu003eTK07=]7Ecdeju003cUju003cDe1Hu003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_Iu003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagLu003cSV@b[GVEU3Xh;R7u003cXeTNgNu003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhRu003e]@GIYf[L55gu003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:Su003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZGu003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9Heu003e1L[3u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[u003eCLdAe]6L2AD0aYHc5u003e=fM7hu003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_du003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]u003eKEu003ceau003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feMu003eLW5VIfJL:eQ4K3a1^WN5T=\\X=u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1Cu003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52gu003eTQQWYJ_@FAX\\]9jhu003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jKu003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6u003eBgESu003e5EaeOFeG:iu003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDOu003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?u003eRQ]5Z9jA@Y9V1ZI6TDkCu003eNZ_f_DRu003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3Ou003eFWu003cJ6X?IiJu003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;fu003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHIu003eI]gBSu003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34u003eh_2@i3kd02Gu003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8u003c^U7Hk]7Q6P:QZS;Ge@:u003cu003cfT6PK7j4?;cdC@c5GI:gS[Wu003cf26;u003cBG7fMXFTWJcbB\\9QTu003eh3HdV8Pb3Rhu003e^?Ue:7RP[=jT4AEu003ebiL_1dYW1u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dEu003cA9LXQbECIc2Mu003c^Iu003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;cu003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_u003c_F9Pu003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2u003e=R4U3W1G;u003chN\\WFO_=DDu003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5Tu003eY?bFOMZOu003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcju003efaaP8P4KDVSCiQ=2u003c=Ef:u003ePu003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_u003c@5Z8fDPJAE8DcGUIb8Cu003c_L7XhP=u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8u003e]LWu003ee^bu003e?0G9Ieu003cu003c@UT4e9u003cGM_jME7[6TFEN:u003c\\Hu003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]u003cL42d\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[u003eEJQi8j;]L5CILgXdR_u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLAu003cKHA:\\[CW7SRYVhE1[MDu003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8ku003eQb]UVVZ:18fe_8M?\\?u003eu003eLf4QSG@jO@u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkUu003cR]Ofg:TNGW0Lu003ePOC_CPu003e^PI[aZ:KY^V@Q;;ME_k\\K0u003eYP]1D5QSc51SfZ]FIP1Y6u003cdRQXRC8RP7BaKGG2?L3bG]S];8_du003e0]RJGeQiJG5\\=O8TRG5Uu003eLGau003eRi2Ku003c3=1TVHN=FhTJYajbIPu003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQu003e93HU2ig?7u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1Ju003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^Gu003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1au003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZBu003cAu003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iKu003e@^u003e[4u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVdeu003cUVVNH2EJ^=ALOFKUX:^u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2Su003cX?9bC7Ebc5V5E]u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SCu003eu003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Yu003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^u003ceM8?j]NZai4u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3u003eI=?@fu003cG349NMId8[T^@Sfu003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@du003ccu003cMhS3K;bu003eZbHAf[GKME9igTY7iVFbau003e4D;WFVb=dQ4Abj2u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Zu003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[u003e@TM9eOu003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhcu003c@=gPHLhQFDC@:Tu003cREdYu003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1adu003cIiK1O7fbD[7[u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?u003e=FFMHPSBf8:\\XRZ91D:2D[1Yu003eX\\bfj4BEQZe:1Au003cQj^@7SAK]C_NCM\\0u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4u003e2u003e4X:9JYPXku003eX_?;DAfLu003ec?HFu003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_u003e1u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aEu003cY^MPdu003e6M^iNNe=P6i6Lf::P6ebjX;u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6Au003c93u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Uau003c8@j5eu003eVA76=g2=gD4V1eYF0bZd0EZu003cMk2M4g[Z=baJ]cVYu003c[D=U2RUdBNdW=69=8UB4E1@u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;fu003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;Du003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4iau003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fau003e:Vj=BR7EW0_hV4=]DaSeQu003c?8]?9X4GbZF41h;FSu003c9Pa=^SQTu003cL:GAIP3XX[\\4RKJVLFabj20Ocu003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\Wu003cHg9FWdu003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:Su003eSu003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BHu003e67u003eWu003cQNZRKDH@]_j^M_AV9g4u003chIFu003eaSDhbj9GMdjh=F=j:u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaSu003eO]caAKiu003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=Ru003cWkCu003c^KSgbI7?aGVaRkbA2?_Raf^u003e9DID]07u003cS431;BaRhX:hNJj]u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6u003cN?Ju003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWcu003e8]u003eU2:HGATaUBPGu003c\\c0aX@_D;_EOK=]Sjk=1:VGKu003e=4P^K\\OD\\D008Du003cgY[GfMjeMu003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNfu003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;u003ebKICA@L3VQ^BG2cZ;Vj@3Jjju003eFA6=LD4g]G=3c@YI305cO@ONPQhNPu003ceaB7BV;u003eIRKK","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\":\"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"after":null,"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660748917000,"snapshot":"false","db":"source","sequence":null,"table":"customers3","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":63272,"row":0,"thread":12,"query":null},"op":"d","ts_ms":1660748917917,"transaction":null}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt b/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt deleted file mode 100644 index 79e8e5de..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/testdata/debezium_msg_7_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":2}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_snapshot/check_db_test.go b/tests/e2e/mysql2mock/debezium/debezium_snapshot/check_db_test.go deleted file mode 100644 index 4a35bba3..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_snapshot/check_db_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "testing" - - "github.com/doublecloud/tross/internal/test/yatest" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - debeziumcommon "github.com/doublecloud/tross/pkg/debezium/common" - "github.com/doublecloud/tross/pkg/debezium/testutil" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = helpers.RecipeMysqlSource() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Source.WithDefaults() - Source.AllowDecimalAsFloat = true -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestSnapshot(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "mysql source", Port: Source.Port}, - )) - - canonizedDebeziumKeyBytes, err := ioutil.ReadFile(yatest.SourcePath("transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt")) - require.NoError(t, err) - canonizedDebeziumValBytes, err := ioutil.ReadFile(yatest.SourcePath("transfer_manager/go/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt")) - require.NoError(t, err) - canonizedDebeziumVal := string(canonizedDebeziumValBytes) - - //------------------------------------------------------------------------------ - - sinker := &helpers.MockSink{} - target := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return sinker }, - Cleanup: server.DisabledCleanup, - } - transfer := helpers.MakeTransfer("fake", Source, &target, abstract.TransferTypeSnapshotOnly) - - var changeItems []abstract.ChangeItem - sinker.PushCallback = func(input []abstract.ChangeItem) { - changeItems = append(changeItems, input...) - } - - helpers.Activate(t, transfer) - - require.Equal(t, 5, len(changeItems)) - require.Equal(t, changeItems[0].Kind, abstract.InitShardedTableLoad) - require.Equal(t, changeItems[1].Kind, abstract.InitTableLoad) - require.Equal(t, changeItems[2].Kind, abstract.InsertKind) - require.Equal(t, changeItems[3].Kind, abstract.DoneTableLoad) - require.Equal(t, changeItems[4].Kind, abstract.DoneShardedTableLoad) - - fmt.Printf("changeItem dump: %s\n", changeItems[2].ToJSONString()) - - testutil.CheckCanonizedDebeziumEvent(t, &changeItems[2], "dbserver1", "source", "mysql", true, []debeziumcommon.KeyValue{{DebeziumKey: string(canonizedDebeziumKeyBytes), DebeziumVal: &canonizedDebeziumVal}}) - - changeItemBuf, err := json.Marshal(changeItems[2]) - require.NoError(t, err) - changeItemDeserialized := helpers.UnmarshalChangeItem(t, changeItemBuf) - testutil.CheckCanonizedDebeziumEvent(t, changeItemDeserialized, "dbserver1", "source", "mysql", true, []debeziumcommon.KeyValue{{DebeziumKey: string(canonizedDebeziumKeyBytes), DebeziumVal: &canonizedDebeziumVal}}) -} diff --git a/tests/e2e/mysql2mock/debezium/debezium_snapshot/dump/dump.sql b/tests/e2e/mysql2mock/debezium/debezium_snapshot/dump/dump.sql deleted file mode 100644 index 57a99e73..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_snapshot/dump/dump.sql +++ /dev/null @@ -1,253 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), - time1 TIME(1), - time2 TIME(2), - time3 TIME(3), - time4 TIME(4), - time5 TIME(5), - time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- SPATIAL TYPES - -# LINESTRING_ GEOMETRY, -# POLYGON_ GEOMETRY, -# MULTIPOINT_ GEOMETRY, -# MULTILINESTRING_ GEOMETRY, -# MULTIPOLYGON_ GEOMETRY, -# GEOMETRYCOLLECTION_ GEOMETRY, - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - - - - - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - '04:05:06.12345', -- TIME(5) - '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) - - -- SPATIAL TYPES - -# ST_GeomFromText('LINESTRING(0 0,1 2,2 4)'), -- LINESTRING_ GEOMETRY, -# ST_GeomFromText('POLYGON((0 0,10 0,10 10,0 10,0 0),(5 5,7 5,7 7,5 7, 5 5))'), -- POLYGON_ GEOMETRY, -# ST_GeomFromText('MULTIPOINT(0 0, 15 25, 45 65)'), -- MULTIPOINT_ GEOMETRY, -# ST_GeomFromText('MULTILINESTRING((12 12, 22 22), (19 19, 32 18))'), -- MULTILINESTRING_ GEOMETRY, -# ST_GeomFromText('MULTIPOLYGON(((0 0,11 0,12 11,0 9,0 0)),((3 5,7 4,4 7,7 7,3 5)))'), -- MULTIPOLYGON_ GEOMETRY, -# ST_GeomFromText('GEOMETRYCOLLECTION(POINT(3 2),LINESTRING(0 0,1 3,2 5,3 5,4 7))') -- GEOMETRYCOLLECTION_ GEOMETRY, -); diff --git a/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt b/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt deleted file mode 100644 index 6e786a1e..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_key.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"int64","optional":false,"field":"pk"}],"optional":false,"name":"dbserver1.source.customers3.Key"},"payload":{"pk":1}} diff --git a/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt b/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt deleted file mode 100644 index 0f6e2163..00000000 --- a/tests/e2e/mysql2mock/debezium/debezium_snapshot/testdata/change_item_val.txt +++ /dev/null @@ -1 +0,0 @@ -{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int64","optional":false,"field":"pk"},{"type":"int16","optional":true,"field":"bool1"},{"type":"int16","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"double","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"before"},{"type":"struct","fields":[{"type":"int64","optional":false,"field":"pk"},{"type":"int16","optional":true,"field":"bool1"},{"type":"int16","optional":true,"field":"bool2"},{"type":"boolean","optional":true,"field":"bit"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"16"},"field":"bit16"},{"type":"int16","optional":true,"field":"tinyint_"},{"type":"int16","optional":true,"default":0,"field":"tinyint_def"},{"type":"int16","optional":true,"field":"tinyint_u"},{"type":"int16","optional":true,"field":"tinyint1"},{"type":"int16","optional":true,"field":"tinyint1u"},{"type":"int16","optional":true,"field":"smallint_"},{"type":"int16","optional":true,"field":"smallint5"},{"type":"int32","optional":true,"field":"smallint_u"},{"type":"int32","optional":true,"field":"mediumint_"},{"type":"int32","optional":true,"field":"mediumint5"},{"type":"int32","optional":true,"field":"mediumint_u"},{"type":"int32","optional":true,"field":"int_"},{"type":"int32","optional":true,"field":"integer_"},{"type":"int32","optional":true,"field":"integer5"},{"type":"int64","optional":true,"field":"int_u"},{"type":"int64","optional":true,"field":"bigint_"},{"type":"int64","optional":true,"field":"bigint5"},{"type":"int64","optional":true,"field":"bigint_u"},{"type":"double","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"real_10_2"},{"type":"double","optional":true,"field":"float_"},{"type":"double","optional":true,"field":"float_53"},{"type":"double","optional":true,"field":"double_"},{"type":"double","optional":true,"field":"double_precision"},{"type":"string","optional":true,"field":"char_"},{"type":"string","optional":true,"field":"char5"},{"type":"string","optional":true,"field":"varchar5"},{"type":"bytes","optional":true,"field":"binary_"},{"type":"bytes","optional":true,"field":"binary5"},{"type":"bytes","optional":true,"field":"varbinary5"},{"type":"bytes","optional":true,"field":"tinyblob_"},{"type":"string","optional":true,"field":"tinytext_"},{"type":"bytes","optional":true,"field":"blob_"},{"type":"string","optional":true,"field":"text_"},{"type":"bytes","optional":true,"field":"mediumblob_"},{"type":"string","optional":true,"field":"mediumtext_"},{"type":"bytes","optional":true,"field":"longblob_"},{"type":"string","optional":true,"field":"longtext_"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"json_"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"x-small,small,medium,large,x-large"},"field":"enum_"},{"type":"string","optional":true,"name":"io.debezium.data.EnumSet","version":1,"parameters":{"allowed":"a,b,c,d"},"field":"set_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year_"},{"type":"int32","optional":true,"name":"io.debezium.time.Year","version":1,"field":"year4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp0"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp2"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp3"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp4"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp5"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamp6"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time0"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time2"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime_"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime0"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime1"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime2"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"datetime3"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime4"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime5"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"datetime6"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"NUMERIC_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"NUMERIC_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"NUMERIC_5_2"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"10"},"field":"DECIMAL_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"DECIMAL_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"DECIMAL_5_2"}],"optional":true,"name":"dbserver1.source.customers3.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false,incremental"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":true,"field":"sequence"},{"type":"string","optional":true,"field":"table"},{"type":"int64","optional":false,"field":"server_id"},{"type":"string","optional":true,"field":"gtid"},{"type":"string","optional":false,"field":"file"},{"type":"int64","optional":false,"field":"pos"},{"type":"int32","optional":false,"field":"row"},{"type":"int64","optional":true,"field":"thread"},{"type":"string","optional":true,"field":"query"}],"optional":false,"name":"io.debezium.connector.mysql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"dbserver1.source.customers3.Envelope"},"payload":{"before":null,"after":{"pk":1,"bool1":0,"bool2":1,"bit":true,"bit16":"nwA=","tinyint_":1,"tinyint_def":22,"tinyint_u":255,"tinyint1":1,"tinyint1u":1,"smallint_":1000,"smallint5":100,"smallint_u":10,"mediumint_":1,"mediumint5":11,"mediumint_u":111,"int_":9,"integer_":99,"integer5":999,"int_u":9999,"bigint_":8,"bigint5":88,"bigint_u":888,"real_":123.45,"real_10_2":99999.99,"float_":1.2300000190734863,"float_53":1.23,"double_":2.34,"double_precision":2.34,"char_":"a","char5":"abc","varchar5":"blab","binary_":"nw==","binary5":"nwAAAAA=","varbinary5":"n58=","tinyblob_":"n5+f","tinytext_":"qwerty12345","blob_":"/w==","text_":"my-text","mediumblob_":"q80=","mediumtext_":"my-mediumtext","longblob_":"q80=","longtext_":"my-longtext","json_":"{\"k1\": \"v1\"}","enum_":"x-small","set_":"a","year_":1901,"year4":2155,"timestamp_":"1999-01-01T00:00:01Z","timestamp0":"1999-10-19T10:23:54Z","timestamp1":"2004-10-19T10:23:54.1Z","timestamp2":"2004-10-19T10:23:54.12Z","timestamp3":"2004-10-19T10:23:54.123Z","timestamp4":"2004-10-19T10:23:54.1234Z","timestamp5":"2004-10-19T10:23:54.12345Z","timestamp6":"2004-10-19T10:23:54.123456Z","date_":-354285,"time_":14706000000,"time0":14706000000,"time1":14706100000,"time2":14706120000,"time3":14706123000,"time4":14706123400,"time5":14706123450,"time6":14706123456,"datetime_":1577891410000,"datetime0":1577891410000,"datetime1":1577891410100,"datetime2":1577891410120,"datetime3":1577891410123,"datetime4":1577891410123400,"datetime5":1577891410123450,"datetime6":1577891410123456,"NUMERIC_":"SZYC0g==","NUMERIC_5":"MDk=","NUMERIC_5_2":"MDk=","DECIMAL_":"AIvQODU=","DECIMAL_5":"W5s=","DECIMAL_5_2":"Wmk="},"source":{"version":"1.9.5.Final","connector":"mysql","name":"dbserver1","ts_ms":1660399176110,"snapshot":"true","db":"source","sequence":null,"table":"customers3","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":3776,"row":0,"thread":null,"query":null},"op":"r","ts_ms":1660399176126,"transaction":null}} diff --git a/tests/e2e/mysql2mock/non_utf8_charset/check_db_test.go b/tests/e2e/mysql2mock/non_utf8_charset/check_db_test.go deleted file mode 100644 index 0dd57a03..00000000 --- a/tests/e2e/mysql2mock/non_utf8_charset/check_db_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package nonutf8charset - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mysql_storage "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - db = os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE") - source = helpers.WithMysqlInclude( - helpers.RecipeMysqlSource(), - []string{fmt.Sprintf("%s.kek", db)}, - ) -) - -func init() { - source.WithDefaults() -} - -type mockSinker struct { - pushCallback func(input []abstract.ChangeItem) error -} - -func (s *mockSinker) Push(input []abstract.ChangeItem) error { - return s.pushCallback(input) -} - -func (s *mockSinker) Close() error { - return nil -} - -func makeConnConfig() *mysql.Config { - cfg := mysql.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", source.Host, source.Port) - cfg.User = source.User - cfg.Passwd = string(source.Password) - cfg.DBName = source.Database - cfg.Net = "tcp" - return cfg -} - -func TestNonUtf8Charset(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: source.Port}, - )) - }() - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} - storage, err := mysql_storage.NewStorage(source.ToStorageParams()) - require.NoError(t, err) - - called := false - table := abstract.TableDescription{Name: "kek", Schema: source.Database} - err = storage.LoadTable(context.Background(), table, func(input []abstract.ChangeItem) error { - i := 0 - for _, item := range input { - if item.Kind != "insert" { - continue - } - require.Len(t, item.ColumnValues, 2) - if i == 0 { - require.EqualValues(t, 1, item.ColumnValues[0]) - require.EqualValues(t, "абыр", item.ColumnValues[1]) - } else { - require.EqualValues(t, 2, item.ColumnValues[0]) - require.EqualValues(t, "валг", item.ColumnValues[1]) - } - i++ - } - if i != 2 { - return nil - } - require.EqualValues(t, 2, i) - called = true - return nil - }) - require.NoError(t, err) - require.True(t, called) - - var sinker mockSinker - target := server.MockDestination{SinkerFactory: func() abstract.Sinker { - return &sinker - }} - transfer := server.Transfer{ - ID: "test", - Src: source, - Dst: &target, - } - - err = mysql_storage.SyncBinlogPosition(source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - wrk := local.NewLocalWorker(coordinator.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - - var haveBambarbia, haveKirgudu bool - sinker.pushCallback = func(input []abstract.ChangeItem) error { - logger.Log.Info("Got items:") - abstract.Dump(input) - for _, item := range input { - if item.Kind != "insert" { - continue - } - require.Len(t, item.ColumnValues, 2) - if item.ColumnValues[0].(int32) == 3 { - require.EqualValues(t, item.ColumnValues[1].(string), "бамбарбия") - haveBambarbia = true - } else { - require.EqualValues(t, item.ColumnValues[1].(string), "киргуду") - haveKirgudu = true - } - if haveBambarbia && haveKirgudu { - _ = wrk.Stop() - } - } - return nil - } - - errCh := make(chan error) - go func() { - errCh <- wrk.Run() - }() - - conn, err := mysql.NewConnector(makeConnConfig()) - require.NoError(t, err) - db := sql.OpenDB(conn) - _, err = db.Exec("INSERT INTO kek VALUES (3, 'бамбарбия')") - require.NoError(t, err) - _, err = db.Exec("INSERT INTO kek VALUES (4, 'киргуду')") - require.NoError(t, err) - - require.NoError(t, <-errCh) -} diff --git a/tests/e2e/mysql2mock/non_utf8_charset/dump/dump.sql b/tests/e2e/mysql2mock/non_utf8_charset/dump/dump.sql deleted file mode 100644 index 8884b5b7..00000000 --- a/tests/e2e/mysql2mock/non_utf8_charset/dump/dump.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE TABLE `kek` ( - `id` int PRIMARY KEY, - `value` text -) -ENGINE = InnoDB -DEFAULT CHARSET = cp1251 -; - -insert into `kek` values - (1, 'абыр'), - (2, 'валг'); diff --git a/tests/e2e/mysql2mock/timezone/canondata/result.json b/tests/e2e/mysql2mock/timezone/canondata/result.json deleted file mode 100644 index 41a4ae8b..00000000 --- a/tests/e2e/mysql2mock/timezone/canondata/result.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "timezone.timezone.TestTimeZone": [ - [ - 1, - "2020-12-23T10:11:12+03:00" - ], - [ - 2, - "2020-12-23T14:15:16+03:00" - ], - [ - 3, - "2020-12-23T13:11:12+03:00" - ], - [ - 4, - "2020-12-23T17:15:16+03:00" - ] - ] -} diff --git a/tests/e2e/mysql2mock/timezone/check_db_test.go b/tests/e2e/mysql2mock/timezone/check_db_test.go deleted file mode 100644 index 1ba062dd..00000000 --- a/tests/e2e/mysql2mock/timezone/check_db_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package nonutf8charset - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - mysql_storage "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -const tableName = "__test1" - -var ( - db = os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE") - source = helpers.WithMysqlInclude( - helpers.RecipeMysqlSource(), - []string{fmt.Sprintf("%s.%s", db, tableName)}, - ) -) - -func init() { - source.WithDefaults() - source.Timezone = "Europe/Moscow" -} - -type mockSinker struct { - pushCallback func(input []abstract.ChangeItem) error -} - -func (s *mockSinker) Push(input []abstract.ChangeItem) error { - return s.pushCallback(input) -} - -func (s *mockSinker) Close() error { - return nil -} - -func dummyProgress(current, progress, total uint64) {} - -func makeConnConfig() *mysql.Config { - cfg := mysql.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", source.Host, source.Port) - cfg.User = source.User - cfg.Passwd = string(source.Password) - cfg.DBName = source.Database - cfg.Net = "tcp" - return cfg -} - -func TestTimeZone(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: source.Port}, - )) - }() - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} - storage, err := mysql_storage.NewStorage(source.ToStorageParams()) - require.NoError(t, err) - - rowsValues := []any{} - - table := abstract.TableDescription{Name: tableName, Schema: source.Database} - err = storage.LoadTable(context.Background(), table, func(input []abstract.ChangeItem) error { - for _, item := range input { - if item.Kind != "insert" { - continue - } - rowsValues = append(rowsValues, item.ColumnValues) - } - return nil - }) - require.NoError(t, err) - - var sinker mockSinker - target := server.MockDestination{SinkerFactory: func() abstract.Sinker { - return &sinker - }} - transfer := server.Transfer{ - ID: "test", - Src: source, - Dst: &target, - } - - err = mysql_storage.SyncBinlogPosition(source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - wrk := local.NewLocalWorker(coordinator.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - - sinker.pushCallback = func(input []abstract.ChangeItem) error { - for _, item := range input { - if item.Kind != "insert" { - continue - } - rowsValues = append(rowsValues, item.ColumnValues) - } - - if len(rowsValues) >= 4 { - _ = wrk.Stop() - } - - return nil - } - - errCh := make(chan error) - go func() { - errCh <- wrk.Run() - }() - - conn, err := mysql.NewConnector(makeConnConfig()) - require.NoError(t, err) - db := sql.OpenDB(conn) - - tx, err := db.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelRepeatableRead, - }) - require.NoError(t, err) - - _, err = tx.Query("SET SESSION time_zone = '+00:00';") - require.NoError(t, err) - - _, err = tx.Query(fmt.Sprintf(` - INSERT INTO %s (ts) VALUES - ('2020-12-23 10:11:12'), - ('2020-12-23 14:15:16'); - `, tableName)) - require.NoError(t, err) - - err = tx.Commit() - require.NoError(t, err) - - require.NoError(t, <-errCh) - - canon.SaveJSON(t, rowsValues) -} diff --git a/tests/e2e/mysql2mock/timezone/dump/dump.sql b/tests/e2e/mysql2mock/timezone/dump/dump.sql deleted file mode 100644 index 7f440027..00000000 --- a/tests/e2e/mysql2mock/timezone/dump/dump.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE TABLE __test1 ( - id integer NOT NULL AUTO_INCREMENT PRIMARY KEY, - ts timestamp -) engine = innodb default charset = utf8; - -BEGIN; - SET SESSION time_zone = '+00:00'; - INSERT INTO __test1 (ts) VALUES - ('2020-12-23 10:11:12'), - ('2020-12-23 14:15:16'); -COMMIT; \ No newline at end of file diff --git a/tests/e2e/mysql2mock/views/check_db_test.go b/tests/e2e/mysql2mock/views/check_db_test.go deleted file mode 100644 index c53bec39..00000000 --- a/tests/e2e/mysql2mock/views/check_db_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "sync" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -type testCaseParams struct { - testCaseName string - tables []string - checkTableLength int - shouldBeError bool - transferType abstract.TransferType -} - -var requests = []string{ - "insert into test2(name, email, age) values ('name2', 'email2', 44);", - "insert into test(name, email, age) values ('name_test', 'email_test', 1);", -} - -func getCfg(source mysql.MysqlSource) *mysql_client.Config { - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", source.Host, source.Port) - cfg.User = source.User - cfg.Passwd = string(source.Password) - cfg.DBName = source.Database - cfg.Net = "tcp" - return cfg -} - -func TestMySQLHeteroViewsInteraction(t *testing.T) { - testCases := []testCaseParams{ - { - testCaseName: "SnapOnlyViewsStored", - checkTableLength: 4, - transferType: abstract.TransferTypeSnapshotOnly, - }, - { - testCaseName: "SnapAndReplicaViewsNotStored", - checkTableLength: 2, - transferType: abstract.TransferTypeSnapshotAndIncrement, - }, - { - testCaseName: "SnapAndReplicaOnlyViewsError", - tables: []string{"test_view", "test_view2"}, - checkTableLength: 2, - transferType: abstract.TransferTypeSnapshotAndIncrement, - shouldBeError: true, - }, - } - - for _, testCase := range testCases { - func(params testCaseParams) { - t.Run(params.testCaseName, func(t *testing.T) { - notesCounter := make(map[string]int) - mutex := sync.RWMutex{} - source := *helpers.RecipeMysqlSource() - source.IncludeTableRegex = params.tables - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: source.Port}, - )) - }() - sinker := &helpers.MockSink{PushCallback: func(items []abstract.ChangeItem) { - for _, item := range items { - if item.IsRowEvent() { - mutex.Lock() - notesCounter[item.Table]++ - mutex.Unlock() - } - } - }} - target := server.MockDestination{ - SinkerFactory: func() abstract.Sinker { return sinker }, - Cleanup: server.DisabledCleanup, - } - transfer := helpers.MakeTransfer("fake", &source, &target, params.transferType) - worker, err := helpers.ActivateErr(transfer) - if params.shouldBeError { - require.Error(t, err) - require.ErrorIs(t, err, tasks.NoTablesError) - return - } - require.NoError(t, err) - defer worker.Close(t) - var notesPerTable int - if params.transferType == abstract.TransferTypeSnapshotAndIncrement { - mysqlConnector, err := mysql_client.NewConnector(getCfg(source)) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - notesPerTable = 3 - } - notesPerTable = 2 - require.Equal(t, params.checkTableLength, func() int { - mutex.RLock() - defer mutex.RUnlock() - return len(notesCounter) - }()) - for table := range notesCounter { - require.Equal(t, notesPerTable, func(table string) int { - mutex.RLock() - defer mutex.RUnlock() - return notesCounter[table] - }(table)) - } - }) - }(testCase) - } - -} diff --git a/tests/e2e/mysql2mock/views/dump/dump.sql b/tests/e2e/mysql2mock/views/dump/dump.sql deleted file mode 100644 index 89172f6f..00000000 --- a/tests/e2e/mysql2mock/views/dump/dump.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE test ( - id INT PRIMARY KEY AUTO_INCREMENT, - name VARCHAR(50), - email VARCHAR(100), - age INT -); - -CREATE TABLE test2 ( - id INT PRIMARY KEY AUTO_INCREMENT, - name VARCHAR(50), - email VARCHAR(100), - age INT -); - -INSERT INTO test(name, email, age) VALUES ('Hideo Kojima', 'test', 69); -INSERT INTO test(name, email, age) VALUES ('Ya sjel deda', 'morgen', 20); -INSERT INTO test2(name, email, age) VALUES ('not deda', 'morgen2', 21); -INSERT INTO test2(name, email, age) VALUES ('Not Kojima', 'test2', 42); - -CREATE VIEW test_view (v_name, v_age, v_email) AS SELECT test.name, test.age, test.email FROM test; -CREATE VIEW test_view2 (v_name1, v_age1, v_email2) AS SELECT test2.name, test2.age, test2.email FROM test2; diff --git a/tests/e2e/mysql2mysql/alters/check_db_test.go b/tests/e2e/mysql2mysql/alters/check_db_test.go deleted file mode 100644 index 29513d68..00000000 --- a/tests/e2e/mysql2mysql/alters/check_db_test.go +++ /dev/null @@ -1,215 +0,0 @@ -package alters - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = mysql.MysqlSource{ - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - NoTracking: true, - } - Target = mysql.MysqlDestination{ - Host: os.Getenv("RECIPE_MYSQL_HOST"), - User: os.Getenv("RECIPE_MYSQL_USER"), - Password: server.SecretString(os.Getenv("RECIPE_MYSQL_PASSWORD")), - Database: os.Getenv("RECIPE_MYSQL_TARGET_DATABASE"), - Port: helpers.GetIntFromEnv("RECIPE_MYSQL_PORT"), - SkipKeyChecks: false, - } -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("Tables on source: %v", tables) - - sourceCfg := mysql_client.NewConfig() - sourceCfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - sourceCfg.User = Source.User - sourceCfg.Passwd = string(Source.Password) - sourceCfg.DBName = Source.Database - sourceCfg.Net = "tcp" - - sourceMysqlConnector, err := mysql_client.NewConnector(sourceCfg) - require.NoError(t, err) - sourceDB := sql.OpenDB(sourceMysqlConnector) - - sourceConn, err := sourceDB.Conn(context.Background()) - require.NoError(t, err) - - alterRequestA := "ALTER TABLE `__test_A` ADD `a_current_time` TIMESTAMP;" - _, err = sourceConn.ExecContext(context.Background(), alterRequestA) - require.NoError(t, err) - - alterRequestB := "ALTER TABLE `__test_B` DROP COLUMN `b_address`;" - _, err = sourceConn.ExecContext(context.Background(), alterRequestB) - require.NoError(t, err) - - alterRequestC := "ALTER TABLE `__test_C` DROP COLUMN `c_uid`;" - _, err = sourceConn.ExecContext(context.Background(), alterRequestC) - require.NoError(t, err) - - alterRequestExtensionD := "ALTER TABLE `__test_D` MODIFY `d_id` bigint NOT NULL;" - _, err = sourceConn.ExecContext(context.Background(), alterRequestExtensionD) - require.NoError(t, err) - - alterRequestNarrowingD := "ALTER TABLE `__test_D` MODIFY `d_uid` int;" - _, err = sourceConn.ExecContext(context.Background(), alterRequestNarrowingD) - require.NoError(t, err) - - var checkTypeD string - requestCheckTypeD := "SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '__test_D' AND COLUMN_NAME = 'd_uid'" - err = sourceConn.QueryRowContext(context.Background(), requestCheckTypeD).Scan(&checkTypeD) - require.NoError(t, err) - require.Equal(t, "int", checkTypeD) - - requestCorrectD := "INSERT INTO `__test_D` (`d_id`, `d_uid`, `d_name`) VALUES (2147483648, 0, 'Joseph');" - _, err = sourceConn.ExecContext(context.Background(), requestCorrectD) - require.NoError(t, err) - - // Enables strict SQL mode and an out of range error occurs while inserting bigger or smaller value than supported - changeOverflowBehaviour := "SET SESSION sql_mode = 'TRADITIONAL';" - _, err = sourceConn.ExecContext(context.Background(), changeOverflowBehaviour) - require.NoError(t, err) - - requestIncorrectD := "INSERT INTO `__test_D` (`d_id`, `d_uid`, `d_name`) VALUES (1337, 2147483648, 'Alex');" - _, err = sourceConn.ExecContext(context.Background(), requestIncorrectD) - require.Error(t, err) - - err = sourceConn.Close() - require.NoError(t, err) - - time.Sleep(10 * time.Second) - // timmyb32r: somewhy test fails if we change it to waiting-polling - - // --------------------------------------------------------------------- - - targetCfg := mysql_client.NewConfig() - targetCfg.Addr = fmt.Sprintf("%v:%v", Target.Host, Target.Port) - targetCfg.User = Target.User - targetCfg.Passwd = string(Target.Password) - targetCfg.DBName = Target.Database - targetCfg.Net = "tcp" - - targetMysqlConnector, err := mysql_client.NewConnector(targetCfg) - require.NoError(t, err) - targetDB := sql.OpenDB(targetMysqlConnector) - - targetConn, err := targetDB.Conn(context.Background()) - require.NoError(t, err) - - countA := 0 - requestA := "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = database() and TABLE_NAME = '__test_A' and COLUMN_NAME = 'a_current_time';" - err = targetConn.QueryRowContext(context.Background(), requestA).Scan(&countA) - require.NoError(t, err) - require.Equal(t, 1, countA) - - countB := 0 - requestB := "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = database() and TABLE_NAME = '__test_B' and COLUMN_NAME = 'b_address';" - err = targetConn.QueryRowContext(context.Background(), requestB).Scan(&countB) - require.NoError(t, err) - require.Equal(t, 0, countB) - - countC := 0 - requestC := "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = database() and TABLE_NAME = '__test_C' and COLUMN_NAME = 'c_uid';" - err = targetConn.QueryRowContext(context.Background(), requestC).Scan(&countC) - require.NoError(t, err) - require.Equal(t, 0, countC) - - var resultExtensionD int - requestExtensionD := "SELECT COUNT(*) FROM `__test_D` WHERE `d_id` = 2147483648;" - err = targetConn.QueryRowContext(context.Background(), requestExtensionD).Scan(&resultExtensionD) - require.NoError(t, err) - require.Equal(t, 1, resultExtensionD) - - var resultNarrowingD int - requestNarrowingD := "SELECT COUNT(*) FROM `__test_D` WHERE `d_id` = 1337;" - err = targetConn.QueryRowContext(context.Background(), requestNarrowingD).Scan(&resultNarrowingD) - require.NoError(t, err) - require.Equal(t, 0, resultNarrowingD) - - err = targetConn.Close() - require.NoError(t, err) -} diff --git a/tests/e2e/mysql2mysql/alters/dump/type_check.sql b/tests/e2e/mysql2mysql/alters/dump/type_check.sql deleted file mode 100644 index 93886ad7..00000000 --- a/tests/e2e/mysql2mysql/alters/dump/type_check.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE TABLE `__test_A` ( - `a_id` integer NOT NULL PRIMARY KEY, - `a_name` varchar(255) NOT NULL -) engine=innodb default charset=utf8; - -CREATE TABLE `__test_B` ( - `b_id` integer NOT NULL PRIMARY KEY, - `b_name` varchar(255) NOT NULL, - `b_address` varchar(255) NOT NULL -) engine=innodb default charset=utf8; - -CREATE TABLE `__test_C` ( - `c_id` integer NOT NULL, - `c_uid` integer NOT NULL, - `c_name` varchar(255) NOT NULL, - PRIMARY KEY(`c_id`, `c_uid`) -) engine=innodb default charset=utf8; - -CREATE TABLE `__test_D` ( - `d_id` int NOT NULL PRIMARY KEY, - `d_uid` bigint, - `d_name` varchar(255) -) engine=innodb default charset=utf8; diff --git a/tests/e2e/mysql2mysql/binary/check_db_test.go b/tests/e2e/mysql2mysql/binary/check_db_test.go deleted file mode 100644 index 34ce3214..00000000 --- a/tests/e2e/mysql2mysql/binary/check_db_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - wrkr := helpers.Activate(t, transfer) - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - "insert into __test values (X'11ECA452BAE6807D9FA707D7252F7EEA', 4, '{\"а\": \"3\"}');", - "insert into __test values (X'11ECA452BB571D439FA707D7252F7EEA', 5, '{\"а\": \"3\"}');", - "update __test set Data = '{\"updated\": \"da\"}' where Version in (1, 2, 3);", - "delete from __test where Version = 4", - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - defer wrkr.Close(t) - time.Sleep(20 * time.Second) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - - dstCfg := mysql_client.NewConfig() - dstCfg.Addr = fmt.Sprintf("%v:%v", Target.Host, Target.Port) - dstCfg.User = Target.User - dstCfg.Passwd = string(Target.Password) - dstCfg.DBName = Target.Database - dstCfg.Net = "tcp" - dstConnector, err := mysql_client.NewConnector(dstCfg) - require.NoError(t, err) - dstConn, err := sql.OpenDB(dstConnector).Conn(context.Background()) - require.NoError(t, err) - var dstSum int - require.NoError(t, dstConn.QueryRowContext(context.Background(), `select sum(Version) from __test;`).Scan(&dstSum)) - var srcSum int - require.NoError(t, conn.QueryRowContext(context.Background(), `select sum(Version) from __test;`).Scan(&srcSum)) - require.Equal(t, srcSum, dstSum) -} diff --git a/tests/e2e/mysql2mysql/binary/dump/type_check.sql b/tests/e2e/mysql2mysql/binary/dump/type_check.sql deleted file mode 100644 index db02f2b8..00000000 --- a/tests/e2e/mysql2mysql/binary/dump/type_check.sql +++ /dev/null @@ -1,12 +0,0 @@ --- needs to be sure there is db1 -create table __test ( - `Id` binary(16) NOT NULL, - `Version` int(11) NOT NULL, - `Data` json NOT NULL, - PRIMARY KEY (`Id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -insert into __test values (0x8E1CF5E9084080E811ECA1542DE42988, 1, '{"а": "1"}'); -insert into __test values (X'DAEBFCCC2D07B6B611ECA15454969110', 2, '{"а": "2"}'); -insert into __test values (X'DAEBFCCC2D07B6B611ECA15454969111', 3, '{"а": "3"}'); diff --git a/tests/e2e/mysql2mysql/cascade_deletes/common/test.go b/tests/e2e/mysql2mysql/cascade_deletes/common/test.go deleted file mode 100644 index afbadcef..00000000 --- a/tests/e2e/mysql2mysql/cascade_deletes/common/test.go +++ /dev/null @@ -1,120 +0,0 @@ -package cascadedeletescommon - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = helpers.RecipeMysqlSource() - Target = helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotOnly) - require.NoError(t, tasks.ActivateDelivery(context.TODO(), nil, coordinator.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from transfer.ID - err = mysql.SyncBinlogPosition(Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - // defer localWorker.Stop() // Uncommenting makes test crash - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("Tables on source: %v", tables) - - sourceCfg := mysql_client.NewConfig() - sourceCfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - sourceCfg.User = Source.User - sourceCfg.Passwd = string(Source.Password) - sourceCfg.DBName = Source.Database - sourceCfg.Net = "tcp" - - sourceMysqlConnector, err := mysql_client.NewConnector(sourceCfg) - require.NoError(t, err) - sourceDB := sql.OpenDB(sourceMysqlConnector) - - sourceConn, err := sourceDB.Conn(context.Background()) - require.NoError(t, err) - - tx, err := sourceConn.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelRepeatableRead, - }) - require.NoError(t, err) - - cascadeDeleteRequest := "DELETE FROM `__test_A` WHERE `a_id`=2;" - - _, err = tx.Exec(`CREATE TABLE test_create ( - id integer NOT NULL AUTO_INCREMENT PRIMARY KEY - ) engine=innodb default charset=utf8`) - require.NoError(t, err) - - _, err = tx.Query(cascadeDeleteRequest) - require.NoError(t, err) - - err = tx.Commit() - require.NoError(t, err) - err = sourceConn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "__test_A", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "__test_B", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/cascade_deletes/dump/type_check.sql b/tests/e2e/mysql2mysql/cascade_deletes/dump/type_check.sql deleted file mode 100644 index 4f22c134..00000000 --- a/tests/e2e/mysql2mysql/cascade_deletes/dump/type_check.sql +++ /dev/null @@ -1,39 +0,0 @@ -CREATE TABLE `__test_A` ( - `a_id` integer NOT NULL PRIMARY KEY, - `a_name` varchar(255) NOT NULL -) engine=innodb default charset=utf8; - -CREATE TABLE `__test_B` ( - `b_id` integer NOT NULL PRIMARY KEY, - `a_id` integer NOT NULL, - `b_name` varchar(255) NOT NULL, - FOREIGN KEY (`a_id`) REFERENCES `__test_A` (`a_id`) ON DELETE CASCADE -) engine=innodb default charset=utf8; - -INSERT INTO `__test_A` (`a_id`, `a_name`) VALUES -( - 1, 'John' -) -, -( - 2, 'Andrew' -) -, -( - 3, 'Kate' -) -; - -INSERT INTO `__test_B` (`b_id`, `a_id`, `b_name`) VALUES -( - 1, 1, 'just a random string' -) -, -( - 2, 1, 'another random string' -) -, -( - 3, 2, 'abracadabra' -) -; diff --git a/tests/e2e/mysql2mysql/cascade_deletes/test_per_table/check_db_test.go b/tests/e2e/mysql2mysql/cascade_deletes/test_per_table/check_db_test.go deleted file mode 100644 index 8352bfbe..00000000 --- a/tests/e2e/mysql2mysql/cascade_deletes/test_per_table/check_db_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package cascadedeletespertbl - -import ( - "testing" - - test "github.com/doublecloud/tross/tests/e2e/mysql2mysql/cascade_deletes/common" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: test.Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: test.Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", test.Existence) - t.Run("Snapshot", test.Snapshot) - t.Run("Replication", test.Load) - }) -} diff --git a/tests/e2e/mysql2mysql/cascade_deletes/test_per_transaction/check_db_test.go b/tests/e2e/mysql2mysql/cascade_deletes/test_per_transaction/check_db_test.go deleted file mode 100644 index 641a1ab7..00000000 --- a/tests/e2e/mysql2mysql/cascade_deletes/test_per_transaction/check_db_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package cascadedeletespertrans - -import ( - "testing" - - test "github.com/doublecloud/tross/tests/e2e/mysql2mysql/cascade_deletes/common" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: test.Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: test.Target.Port}, - )) - }() - - test.Target.PerTransactionPush = true - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", test.Existence) - t.Run("Snapshot", test.Snapshot) - t.Run("Replication", test.Load) - }) -} diff --git a/tests/e2e/mysql2mysql/cleanup_tables/cleanup_test.go b/tests/e2e/mysql2mysql/cleanup_tables/cleanup_test.go deleted file mode 100644 index 035f275a..00000000 --- a/tests/e2e/mysql2mysql/cleanup_tables/cleanup_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package light - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/middlewares" - "github.com/doublecloud/tross/pkg/sink" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/pkg/worker/tasks/cleanup" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - SourceWithBlackList = *helpers.WithMysqlInclude(helpers.RecipeMysqlSource(), []string{"items_.*"}) - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Drop by filter", TruncateAll) - t.Run("Drop by filter", DropFilter) - t.Run("Drop all tables", DropAll) - }) -} - -func DropAll(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - sink, err := sink.MakeAsyncSink(transfer, logger.Log, helpers.EmptyRegistry(), coordinator.NewFakeClient(), middlewares.MakeConfig(middlewares.WithNoData)) - require.NoError(t, err) - - err = cleanup.CleanupTables(sink, tables, server.Drop) - require.NoError(t, err) -} - -func DropFilter(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &SourceWithBlackList, &Target, abstract.TransferTypeSnapshotAndIncrement) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - sink, err := sink.MakeAsyncSink(transfer, logger.Log, helpers.EmptyRegistry(), coordinator.NewFakeClient(), middlewares.MakeConfig(middlewares.WithNoData)) - require.NoError(t, err) - - err = cleanup.CleanupTables(sink, tables, server.Drop) - require.NoError(t, err) -} - -func TruncateAll(t *testing.T) { - dstCopy := Target - dstCopy.Cleanup = server.Truncate - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &dstCopy, abstract.TransferTypeSnapshotAndIncrement) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - sink, err := sink.MakeAsyncSink(transfer, logger.Log, helpers.EmptyRegistry(), coordinator.NewFakeClient(), middlewares.MakeConfig(middlewares.WithNoData)) - require.NoError(t, err) - - err = cleanup.CleanupTables(sink, tables, server.Truncate) - require.NoError(t, err) -} diff --git a/tests/e2e/mysql2mysql/cleanup_tables/source/dump.sql b/tests/e2e/mysql2mysql/cleanup_tables/source/dump.sql deleted file mode 100644 index c17234e2..00000000 --- a/tests/e2e/mysql2mysql/cleanup_tables/source/dump.sql +++ /dev/null @@ -1,38 +0,0 @@ -create table ids_1 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -create table items_1 ( - id int not null primary key, - item_id int not null, - ts timestamp, - city varchar(100), - FOREIGN KEY (item_id) - REFERENCES ids_1(id) - ON DELETE CASCADE -); - -create table ids_2 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -create table items_2 ( - id int not null primary key, - item_id int not null, - city varchar(100), - FOREIGN KEY (item_id) - REFERENCES ids_2(id) - ON DELETE CASCADE -); - -create view spb_items_1_2020 as - select * - from items_1 - where city = 'spb' and ts >= timestamp '2020-01-01 00:00:00'; - diff --git a/tests/e2e/mysql2mysql/cleanup_tables/target/dump.sql b/tests/e2e/mysql2mysql/cleanup_tables/target/dump.sql deleted file mode 100644 index d7684a0e..00000000 --- a/tests/e2e/mysql2mysql/cleanup_tables/target/dump.sql +++ /dev/null @@ -1,46 +0,0 @@ -create table ids_1 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -insert into ids_1 (id, name) values (1, '1'); - -create table items_1 ( - id int not null primary key, - item_id int not null, - ts timestamp, - city varchar(100), - FOREIGN KEY (item_id) - REFERENCES ids_1(id) - ON DELETE CASCADE -); - -insert into items_1 (id, item_id) values (11, 1); - -create table ids_2 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -insert into ids_2 (id, name) values (2, '2'); - -create table items_2 ( - id int not null primary key, - item_id int not null, - city varchar(100), - FOREIGN KEY (item_id) - REFERENCES ids_2(id) - ON DELETE CASCADE -); - -insert into items_2 (id, item_id) values (22, 2); - -create view spb_items_1_2020 as - select * - from items_1 - where city = 'spb' and ts >= timestamp '2020-01-01 00:00:00'; - diff --git a/tests/e2e/mysql2mysql/comment/check_db_test.go b/tests/e2e/mysql2mysql/comment/check_db_test.go deleted file mode 100644 index 4be30e79..00000000 --- a/tests/e2e/mysql2mysql/comment/check_db_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package comment_test - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/comment/dump/comment.sql b/tests/e2e/mysql2mysql/comment/dump/comment.sql deleted file mode 100644 index 02436bcd..00000000 --- a/tests/e2e/mysql2mysql/comment/dump/comment.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE `comment_test` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'a;tricky\'com;ment;;\';x;x\'', - `txt` varchar(36) DEFAULT NULL COMMENT "do\"u;s;a;\"ble;d;;\"quotes\"\"\";\";s;\";", - PRIMARY KEY (`id`) -); - -insert into comment_test (txt) values ('\'b;\';;sd;\'l;'); -insert into comment_test (txt) values ('\";x\';;\"d;\";sd;d\"\'\"\"sdf;\";\"a;\';'); diff --git a/tests/e2e/mysql2mysql/consistent_snapshot/check_db_test.go b/tests/e2e/mysql2mysql/consistent_snapshot/check_db_test.go deleted file mode 100644 index 053c79c8..00000000 --- a/tests/e2e/mysql2mysql/consistent_snapshot/check_db_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package geometry_test - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/storage" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func dropData(t *testing.T) { - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - `delete from fruit`, - `delete from employee`, - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) -} - -func checkTarget(t *testing.T) { - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Target.Host, Target.Port) - cfg.User = Target.User - cfg.Passwd = string(Target.Password) - cfg.DBName = Target.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - var count int - - err = conn.QueryRowContext(context.Background(), "select count(*) from fruit").Scan(&count) - require.NoError(t, err) - require.EqualValues(t, 12, count) - - err = conn.QueryRowContext(context.Background(), "select count(*) from employee").Scan(&count) - require.NoError(t, err) - require.EqualValues(t, 8, count) - - err = conn.Close() - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - Source.ConsistentSnapshot = true - Source.SnapshotDegreeOfParallelism = 1 - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - transfer = helpers.WithLocalRuntime(transfer, 1, 1) - - storage, err := storage.NewStorage(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - defer storage.Close() - mysqlStorage, ok := storage.(*mysql.Storage) - require.True(t, ok) - tables, err := server.FilteredTableList(storage, transfer) - require.NoError(t, err) - - err = mysqlStorage.BeginSnapshot(context.TODO()) - require.NoError(t, err) - - dropData(t) - - operationID := "test-operation" - - operationTables := []*server.OperationTablePart{} - for _, table := range tables.ConvertToTableDescriptions() { - operationTables = append(operationTables, server.NewOperationTablePartFromDescription(operationID, &table)) - } - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), operationID, transfer, helpers.EmptyRegistry()) - - err = snapshotLoader.DoUploadTables(context.TODO(), storage, snapshotLoader.GetLocalTablePartProvider(operationTables...)) - require.NoError(t, err) - - err = mysqlStorage.EndSnapshot(context.TODO()) - require.NoError(t, err) - - checkTarget(t) -} diff --git a/tests/e2e/mysql2mysql/consistent_snapshot/dump/consistent_snapshot.sql b/tests/e2e/mysql2mysql/consistent_snapshot/dump/consistent_snapshot.sql deleted file mode 100644 index 3d49bd07..00000000 --- a/tests/e2e/mysql2mysql/consistent_snapshot/dump/consistent_snapshot.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE TABLE IF NOT EXISTS fruit ( - fruit_id INT(10) UNSIGNED NOT NULL auto_increment, - name VARCHAR(50) NOT NULL, - variety VARCHAR(50) NOT NULL, - PRIMARY KEY (fruit_id) -); - -INSERT INTO - fruit (fruit_id, name, variety) -VALUES - (1, 'Apple', 'Red Delicious'), - (2, 'Pear', 'Comice'), - (3, 'Orange', 'Navel'), - (4, 'Pear', 'Bartlett'), - (5, 'Orange', 'Blood'), - (6, 'Apple', 'Cox''s Orange Pippin'), - (7, 'Apple', 'Granny Smith'), - (8, 'Pear', 'Anjou'), - (9, 'Orange', 'Valencia'), - (10, 'Banana', 'Plantain'), - (11, 'Banana', 'Burro'), - (12, 'Banana', 'Cavendish'); - -CREATE TABLE employee ( - id INT NOT NULL AUTO_INCREMENT, - first_name VARCHAR(100) NOT NULL, - last_name VARCHAR(100) NOT NULL, - job_title VARCHAR(100) DEFAULT NULL, - salary DOUBLE DEFAULT NULL, - notes text, - PRIMARY KEY (id) -); - -INSERT INTO - employee (first_name, last_name, job_title, salary) -VALUES - ('Robin', 'Jackman', 'Software Engineer', 5500), - ('Taylor', 'Edward', 'Software Architect', 7200), - ('Vivian', 'Dickens', 'Database Administrator', 6000), - ('Harry', 'Clifford', 'Database Administrator', 6800), - ('Eliza', 'Clifford', 'Software Engineer', 4750), - ('Nancy', 'Newman', 'Software Engineer', 5100), - ('Melinda', 'Clifford', 'Project Manager', 8500), - ('Harley', 'Gilbert', 'Software Architect', 8000); diff --git a/tests/e2e/mysql2mysql/date_time/check_db_test.go b/tests/e2e/mysql2mysql/date_time/check_db_test.go deleted file mode 100644 index 24b68673..00000000 --- a/tests/e2e/mysql2mysql/date_time/check_db_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package datetime - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := &server.Transfer{ - ID: "test-id", - Src: &Source, - Dst: &Target, - } - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from transfer.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - modeRequest := `SET SESSION sql_mode=''` // drop strict mode - timeRequest := `SET SESSION time_zone = '+00:00'` // set UTC to check corner cases - insertRequest1 := `INSERT INTO __test1 (col_d, col_dt, col_ts) VALUES - ('0000-00-00', '0000-00-00 00:00:00', '0000-00-00 00:00:00'), - ('1000-01-01', '1000-01-01 00:00:00', '1970-01-01 00:00:01'), - ('9999-12-31', '9999-12-31 23:59:59', '2038-01-19 03:14:07'), - ('2020-12-23', '2020-12-23 14:15:16', '2020-12-23 14:15:16')` - insertRequest2 := `INSERT INTO __test2 (col_dt1, col_dt2, col_dt3, col_dt4, col_dt5, col_dt6, col_ts1, col_ts2, col_ts3, col_ts4, col_ts5, col_ts6) VALUES - ('2020-12-23 14:15:16.1', '2020-12-23 14:15:16.12', '2020-12-23 14:15:16.123', '2020-12-23 14:15:16.1234', '2020-12-23 14:15:16.12345', '2020-12-23 14:15:16.123456','2020-12-23 14:15:16.1', '2020-12-23 14:15:16.12', '2020-12-23 14:15:16.123', '2020-12-23 14:15:16.1234', '2020-12-23 14:15:16.12345', '2020-12-23 14:15:16.123456'), - ('2020-12-23 14:15:16.6', '2020-12-23 14:15:16.65', '2020-12-23 14:15:16.654', '2020-12-23 14:15:16.6543', '2020-12-23 14:15:16.65432', '2020-12-23 14:15:16.654321','2020-12-23 14:15:16.6', '2020-12-23 14:15:16.65', '2020-12-23 14:15:16.654', '2020-12-23 14:15:16.6543', '2020-12-23 14:15:16.65432', '2020-12-23 14:15:16.654321')` - - tx, err := conn.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelRepeatableRead, - }) - require.NoError(t, err) - - _, err = tx.Query(modeRequest) - require.NoError(t, err) - _, err = tx.Query(timeRequest) - require.NoError(t, err) - _, err = tx.Query(insertRequest1) - require.NoError(t, err) - _, err = tx.Query(insertRequest2) - require.NoError(t, err) - err = tx.Commit() - require.NoError(t, err) - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "__test1", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "__test2", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/date_time/dump/date_time.sql b/tests/e2e/mysql2mysql/date_time/dump/date_time.sql deleted file mode 100644 index d8bd575e..00000000 --- a/tests/e2e/mysql2mysql/date_time/dump/date_time.sql +++ /dev/null @@ -1,34 +0,0 @@ -SET SESSION sql_mode=''; - -CREATE TABLE `__test1` ( - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY, - `col_d` date, - `col_dt` datetime, - `col_ts` timestamp -) engine=innodb default charset=utf8; - -CREATE TABLE `__test2` ( - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY, - `col_dt1` datetime(1), - `col_dt2` datetime(2), - `col_dt3` datetime(3), - `col_dt4` datetime(4), - `col_dt5` datetime(5), - `col_dt6` datetime(6), - `col_ts1` timestamp(1), - `col_ts2` timestamp(2), - `col_ts3` timestamp(3), - `col_ts4` timestamp(4), - `col_ts5` timestamp(5), - `col_ts6` timestamp(6) -) engine=innodb default charset=utf8; - -INSERT INTO __test1 (col_d, col_dt, col_ts) VALUES - ('0000-00-00', '0000-00-00 00:00:00', '0000-00-00 00:00:00'), - ('1000-01-01', '1000-01-01 00:00:00', '1970-01-01 00:00:01'), - ('9999-12-31', '9999-12-31 23:59:59', '2038-01-19 03:14:07'), - ('2020-12-23', '2020-12-23 14:15:16', '2020-12-23 14:15:16'); - -INSERT INTO __test2 (col_dt1, col_dt2, col_dt3, col_dt4, col_dt5, col_dt6, col_ts1, col_ts2, col_ts3, col_ts4, col_ts5, col_ts6) VALUES - ('2020-12-23 14:15:16.1', '2020-12-23 14:15:16.12', '2020-12-23 14:15:16.123', '2020-12-23 14:15:16.1234', '2020-12-23 14:15:16.12345', '2020-12-23 14:15:16.123456','2020-12-23 14:15:16.1', '2020-12-23 14:15:16.12', '2020-12-23 14:15:16.123', '2020-12-23 14:15:16.1234', '2020-12-23 14:15:16.12345', '2020-12-23 14:15:16.123456'), - ('2020-12-23 14:15:16.6', '2020-12-23 14:15:16.65', '2020-12-23 14:15:16.654', '2020-12-23 14:15:16.6543', '2020-12-23 14:15:16.65432', '2020-12-23 14:15:16.654321','2020-12-23 14:15:16.6', '2020-12-23 14:15:16.65', '2020-12-23 14:15:16.654', '2020-12-23 14:15:16.6543', '2020-12-23 14:15:16.65432', '2020-12-23 14:15:16.654321'); \ No newline at end of file diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes/check_db_test.go deleted file mode 100644 index a946d773..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes/check_db_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 2, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - -- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - -- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - -- '04:05:06.12345', -- TIME(5) - -- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes/dump/type_check.sql deleted file mode 100644 index ec8beaef..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes/dump/type_check.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) --- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) --- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) --- '04:05:06.12345', -- TIME(5) --- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/check_db_test.go deleted file mode 100644 index 5487ab60..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/check_db_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 2, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - -- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - -- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - -- '04:05:06.12345', -- TIME(5) - -- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/dump/type_check.sql deleted file mode 100644 index ec8beaef..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_nohomo/dump/type_check.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) --- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) --- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) --- '04:05:06.12345', -- TIME(5) --- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/check_db_test.go deleted file mode 100644 index 78f3ba9b..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/check_db_test.go +++ /dev/null @@ -1,209 +0,0 @@ -package main - -import ( - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 2, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - -- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - -- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - -- '04:05:06.12345', -- TIME(5) - -- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -//--------------------------------------------------------------------------------------------------------------------- - -func serdeUdf(t *testing.T, items []abstract.ChangeItem) abstract.TransformerResult { - newChangeItems := make([]abstract.ChangeItem, 0) - errors := make([]abstract.TransformerError, 0) - for i := range items { - if items[i].IsSystemTable() { - continue - } - currJSON := items[i].ToJSONString() - fmt.Printf("changeItem dump:%s\n", currJSON) - outChangeItem, err := abstract.UnmarshalChangeItem([]byte(currJSON)) - if err != nil { - errors = append(errors, abstract.TransformerError{ - Input: items[i], - Error: err, - }) - } else { - newChangeItems = append(newChangeItems, *outChangeItem) - } - } - return abstract.TransformerResult{ - Transformed: newChangeItems, - Errors: errors, - } -} - -func anyTablesUdf(table abstract.TableID, schema abstract.TableColumns) bool { - return true -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - serdeTransformer := helpers.NewSimpleTransformer(t, serdeUdf, anyTablesUdf) - helpers.AddTransformer(t, transfer, serdeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/dump/type_check.sql deleted file mode 100644 index ec8beaef..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde/dump/type_check.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) --- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) --- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) --- '04:05:06.12345', -- TIME(5) --- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go deleted file mode 100644 index 80023bed..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go +++ /dev/null @@ -1,189 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 2, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - -- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - -- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - -- '04:05:06.12345', -- TIME(5) - -- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "mysql", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithoutCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/dump/type_check.sql deleted file mode 100644 index ec8beaef..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded/dump/type_check.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) --- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) --- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) --- '04:05:06.12345', -- TIME(5) --- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go deleted file mode 100644 index dc560e97..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "mysql", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithoutCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(`INSERT INTO customers3 (pk) VALUES (2);`) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/dump/type_check.sql deleted file mode 100644 index 98b2c02b..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_embedded_nulls/dump/type_check.sql +++ /dev/null @@ -1,118 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - --- timestamp_ TIMESTAMP, -- uncomment after TM-4377 --- timestamp0 TIMESTAMP(0),-- uncomment after TM-4377 --- timestamp1 TIMESTAMP(1),-- uncomment after TM-4377 --- timestamp2 TIMESTAMP(2),-- uncomment after TM-4377 --- timestamp3 TIMESTAMP(3),-- uncomment after TM-4377 --- timestamp4 TIMESTAMP(4),-- uncomment after TM-4377 --- timestamp5 TIMESTAMP(5),-- uncomment after TM-4377 --- timestamp6 TIMESTAMP(6),-- uncomment after TM-4377 - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 (pk) VALUES (1); diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go deleted file mode 100644 index 6ba4ac87..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go +++ /dev/null @@ -1,273 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumcommon "github.com/doublecloud/tross/pkg/debezium/common" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 2, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - -- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - -- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - -- '04:05:06.12345', -- TIME(5) - -- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "false", - debeziumparameters.SourceType: "mysql", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - originalTypes := map[abstract.TableID]map[string]*debeziumcommon.OriginalTypeInfo{ - abstract.TableID{Namespace: "", Name: "customers3"}: { - "pk": {OriginalType: "mysql:int(10) unsigned"}, - "bool1": {OriginalType: "mysql:tinyint(1)"}, - "bool2": {OriginalType: "mysql:tinyint(1)"}, - "bit": {OriginalType: "mysql:bit(1)"}, - "bit16": {OriginalType: "mysql:bit(16)"}, - "tinyint_": {OriginalType: "mysql:tinyint(4)"}, - "tinyint_def": {OriginalType: "mysql:tinyint(4)"}, - "tinyint_u": {OriginalType: "mysql:tinyint(3) unsigned"}, - "tinyint1": {OriginalType: "mysql:tinyint(1)"}, - "tinyint1u": {OriginalType: "mysql:tinyint(1) unsigned"}, - "smallint_": {OriginalType: "mysql:smallint(6)"}, - "smallint5": {OriginalType: "mysql:smallint(5)"}, - "smallint_u": {OriginalType: "mysql:smallint(5) unsigned"}, - "mediumint_": {OriginalType: "mysql:mediumint(9)"}, - "mediumint5": {OriginalType: "mysql:mediumint(5)"}, - "mediumint_u": {OriginalType: "mysql:mediumint(8) unsigned"}, - "int_": {OriginalType: "mysql:int(11)"}, - "integer_": {OriginalType: "mysql:int(11)"}, - "integer5": {OriginalType: "mysql:int(5)"}, - "int_u": {OriginalType: "mysql:int(10) unsigned"}, - "bigint_": {OriginalType: "mysql:bigint(20)"}, - "bigint5": {OriginalType: "mysql:bigint(5)"}, - "bigint_u": {OriginalType: "mysql:bigint(20) unsigned"}, - "real_": {OriginalType: "mysql:double"}, - "real_10_2": {OriginalType: "mysql:double(10,2)"}, - "float_": {OriginalType: "mysql:float"}, - "float_53": {OriginalType: "mysql:double"}, - "double_": {OriginalType: "mysql:double"}, - "double_precision": {OriginalType: "mysql:double"}, - "char_": {OriginalType: "mysql:char(1)"}, - "char5": {OriginalType: "mysql:char(5)"}, - "varchar5": {OriginalType: "mysql:varchar(5)"}, - "binary_": {OriginalType: "mysql:binary(1)"}, - "binary5": {OriginalType: "mysql:binary(5)"}, - "varbinary5": {OriginalType: "mysql:varbinary(5)"}, - "tinyblob_": {OriginalType: "mysql:tinyblob"}, - "tinytext_": {OriginalType: "mysql:tinytext"}, - "blob_": {OriginalType: "mysql:blob"}, - "text_": {OriginalType: "mysql:text"}, - "mediumblob_": {OriginalType: "mysql:mediumblob"}, - "mediumtext_": {OriginalType: "mysql:mediumtext"}, - "longblob_": {OriginalType: "mysql:longblob"}, - "longtext_": {OriginalType: "mysql:longtext"}, - "json_": {OriginalType: "mysql:json"}, - "enum_": {OriginalType: "mysql:enum('x-small','small','medium','large','x-large')"}, - "set_": {OriginalType: "mysql:set('a','b','c','d')"}, - "year_": {OriginalType: "mysql:year(4)"}, - "year4": {OriginalType: "mysql:year(4)"}, - "timestamp_": {OriginalType: "mysql:timestamp"}, - "timestamp0": {OriginalType: "mysql:timestamp"}, - "timestamp1": {OriginalType: "mysql:timestamp(1)"}, - "timestamp2": {OriginalType: "mysql:timestamp(2)"}, - "timestamp3": {OriginalType: "mysql:timestamp(3)"}, - "timestamp4": {OriginalType: "mysql:timestamp(4)"}, - "timestamp5": {OriginalType: "mysql:timestamp(5)"}, - "timestamp6": {OriginalType: "mysql:timestamp(6)"}, - "date_": {OriginalType: "mysql:date"}, - "time_": {OriginalType: "mysql:time"}, - "time0": {OriginalType: "mysql:time"}, - "time1": {OriginalType: "mysql:time(1)"}, - "time2": {OriginalType: "mysql:time(2)"}, - "time3": {OriginalType: "mysql:time(3)"}, - "time4": {OriginalType: "mysql:time(4)"}, - "time5": {OriginalType: "mysql:time(5)"}, - "time6": {OriginalType: "mysql:time(6)"}, - "datetime_": {OriginalType: "mysql:datetime"}, - "datetime0": {OriginalType: "mysql:datetime"}, - "datetime1": {OriginalType: "mysql:datetime(1)"}, - "datetime2": {OriginalType: "mysql:datetime(2)"}, - "datetime3": {OriginalType: "mysql:datetime(3)"}, - "datetime4": {OriginalType: "mysql:datetime(4)"}, - "datetime5": {OriginalType: "mysql:datetime(5)"}, - "datetime6": {OriginalType: "mysql:datetime(6)"}, - "NUMERIC_": {OriginalType: "mysql:decimal(10,0)"}, - "NUMERIC_5": {OriginalType: "mysql:decimal(5,0)"}, - "NUMERIC_5_2": {OriginalType: "mysql:decimal(5,2)"}, - "DECIMAL_": {OriginalType: "mysql:decimal(10,0)"}, - "DECIMAL_5": {OriginalType: "mysql:decimal(5,0)"}, - "DECIMAL_5_2": {OriginalType: "mysql:decimal(5,2)"}, - }, - } - receiver := debezium.NewReceiver(originalTypes, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithoutCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/dump/type_check.sql deleted file mode 100644 index ec8beaef..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_external/dump/type_check.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) --- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) --- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) --- '04:05:06.12345', -- TIME(5) --- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go deleted file mode 100644 index 2907740c..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go +++ /dev/null @@ -1,189 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -var insertStmt = ` -INSERT INTO customers3 VALUES ( - 2, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) - -- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) - -- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) - -- '04:05:06.12345', -- TIME(5) - -- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "false", - debeziumparameters.SourceType: "mysql", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithoutCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.Equal(t, 2, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/dump/type_check.sql deleted file mode 100644 index ec8beaef..00000000 --- a/tests/e2e/mysql2mysql/debezium/all_datatypes_serde_via_debezium_not_enriched/dump/type_check.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - bool1 BOOLEAN, - bool2 BOOL, - bit BIT(1), - bit16 BIT(16), - - tinyint_ TINYINT, - tinyint_def TINYINT DEFAULT 0, - tinyint_u TINYINT UNSIGNED, - - tinyint1 TINYINT(1), - tinyint1u TINYINT(1) UNSIGNED, - - smallint_ SMALLINT, - smallint5 SMALLINT(5), - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint5 MEDIUMINT(5), - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - integer_ INTEGER, - integer5 INTEGER(5), - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint5 BIGINT(5), - bigint_u BIGINT UNSIGNED, - - -- --- - - real_ REAL, - real_10_2 REAL(10, 2), - - float_ FLOAT, - float_53 FLOAT(53), - - double_ DOUBLE, - double_precision DOUBLE PRECISION, - - -- --- - - char_ CHAR, - char5 CHAR(5), - - varchar5 VARCHAR(5), - - binary_ BINARY, - binary5 BINARY(5), - - varbinary5 VARBINARY(5), - - tinyblob_ TINYBLOB, - tinytext_ TINYTEXT, - - blob_ BLOB, - text_ TEXT, - mediumblob_ MEDIUMBLOB, - mediumtext_ MEDIUMTEXT, - longblob_ LONGBLOB, - longtext_ LONGTEXT, - json_ JSON, - enum_ ENUM('x-small', 'small', 'medium', 'large', 'x-large'), - set_ SET('a', 'b', 'c', 'd'), - - year_ YEAR, - year4 YEAR(4), - - timestamp_ TIMESTAMP, - timestamp0 TIMESTAMP(0), - timestamp1 TIMESTAMP(1), - timestamp2 TIMESTAMP(2), - timestamp3 TIMESTAMP(3), - timestamp4 TIMESTAMP(4), - timestamp5 TIMESTAMP(5), - timestamp6 TIMESTAMP(6), - - -- TEMPORAL TYPES - - date_ DATE, - - time_ TIME, - time0 TIME(0), --- time1 TIME(1), - time2 TIME(2), --- time3 TIME(3), - time4 TIME(4), --- time5 TIME(5), --- time6 TIME(6), - - datetime_ DATETIME, - datetime0 DATETIME(0), - datetime1 DATETIME(1), - datetime2 DATETIME(2), - datetime3 DATETIME(3), - datetime4 DATETIME(4), - datetime5 DATETIME(5), - datetime6 DATETIME(6), - - -- DECIMAL TYPES - - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - -- - - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 VALUES ( - 1, - - 0, -- BOOLEAN - 1, -- BOOL - 1, -- BIT(1) - X'9f', -- BIT(16) - - 1, -- TINYINT - 22, -- TINYINT DEFAULT 0 - 255, -- TINYINT UNSIGNED - - 1, -- TINYINT(1) - 1, -- TINYINT(1) UNSIGNED - - 1000, -- SMALLINT - 100, -- SMALLINT(5) - 10, -- SMALLINT UNSIGNED - - 1, -- MEDIUMINT - 11, -- MEDIUMINT(5) - 111, -- MEDIUMINT UNSIGNED - - 9, -- INT - 99, -- INTEGER - 999, -- INTEGER(5) - 9999, -- INT UNSIGNED - - 8, -- BIGINT - 88, -- BIGINT(5) - 888, -- BIGINT UNSIGNED - - -- REAL - - 123.45, -- REAL - 99999.99, -- REAL(10, 2) - - 1.23, -- FLOAT - 1.23, -- FLOAT(53) - - 2.34, -- DOUBLE - 2.34, -- DOUBLE PRECISION - - -- CHAR - - 'a', -- CHAR - 'abc', -- CHAR(5) - - 'blab', -- VARCHAR(5) - - X'9f', -- BINARY - X'9f', -- BINARY(5) - - X'9f9f', -- VARBINARY(5) - - X'9f9f9f', -- TINYBLOB - 'qwerty12345', -- TINYTEXT - - X'ff', -- BLOB - 'my-text', -- TEXT - X'abcd', -- MEDIUMBLOB - 'my-mediumtext', -- MEDIUMTEXT - X'abcd', -- LONGBLOB - 'my-longtext', -- LONGTEXT - '{"k1": "v1"}', -- JSON - 'x-small', -- ENUM('x-small', 'small', 'medium', 'large', 'x-large') - 'a', -- SET('a', 'b', 'c', 'd') - - -- TEMPORAL DATA TYPES - - 1901, -- YEAR - 2155, -- YEAR(4) - - '1999-01-01 00:00:01', -- TIMESTAMP - '1999-10-19 10:23:54', -- TIMESTAMP(0) - '2004-10-19 10:23:54.1', -- TIMESTAMP(1) - '2004-10-19 10:23:54.12', -- TIMESTAMP(2) - '2004-10-19 10:23:54.123', -- TIMESTAMP(3) - '2004-10-19 10:23:54.1234', -- TIMESTAMP(4) - '2004-10-19 10:23:54.12345', -- TIMESTAMP(5) - '2004-10-19 10:23:54.123456', -- TIMESTAMP(6) - - -- TEMPORAL TYPES - - '1000-01-01', -- DATE - - '04:05:06', -- TIME - '04:05:06', -- TIME(0) --- '04:05:06.1', -- TIME(1) - '04:05:06.12', -- TIME(2) --- '04:05:06.123', -- TIME(3) - '04:05:06.1234', -- TIME(4) --- '04:05:06.12345', -- TIME(5) --- '04:05:06.123456', -- TIME(6) - - '2020-01-01 15:10:10', -- DATETIME - '2020-01-01 15:10:10', -- DATETIME(0) - '2020-01-01 15:10:10.1', -- DATETIME(1) - '2020-01-01 15:10:10.12', -- DATETIME(2) - '2020-01-01 15:10:10.123', -- DATETIME(3) - '2020-01-01 15:10:10.1234', -- DATETIME(4) - '2020-01-01 15:10:10.12345', -- DATETIME(5) - '2020-01-01 15:10:10.123456', -- DATETIME(6) - - -- DECIMAL TYPES - - 1234567890, -- NUMERIC - 12345, -- NUMERIC(5) - 123.45, -- NUMERIC(5,2) - - 2345678901, -- DECIMAL - 23451, -- DECIMAL(5) - 231.45 -- DECIMAL(5,2) -); diff --git a/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go b/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go deleted file mode 100644 index df51908e..00000000 --- a/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package main - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "mysql", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*mysql.MysqlSource).PlzNoHomo = true - transfer.Src.(*mysql.MysqlSource).AllowDecimalAsFloat = true - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithoutCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) - require.NoError(t, err) - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - - _, err = db.Exec(` - INSERT INTO customers3 (pk,tinyint_,tinyint_u,smallint_,smallint_u,mediumint_,mediumint_u,int_,int_u,bigint_,bigint_u) VALUES ( - 3, - - -128, - 0, - - -32768, - 0, - - -8388608, - 0, - - -2147483648, - 0, - - -9223372036854775808, - 0 - ); - `) - require.NoError(t, err) - - _, err = db.Exec(` - INSERT INTO customers3 (pk,tinyint_,tinyint_u,smallint_,smallint_u,mediumint_,mediumint_u,int_,int_u,bigint_,bigint_u) VALUES ( - 4, - - 127, - 255, - - 32767, - 65535, - - 8388607, - 16777215, - - 2147483647, - 4294967295, - - 9223372036854775807, - 18446744073709551615 - ); - `) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers3", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/dump/type_check.sql b/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/dump/type_check.sql deleted file mode 100644 index d0c8d479..00000000 --- a/tests/e2e/mysql2mysql/debezium/num_limits_serde_via_debezium_embedded/dump/type_check.sql +++ /dev/null @@ -1,59 +0,0 @@ -CREATE TABLE customers3 ( - pk integer unsigned auto_increment, - - tinyint_ TINYINT, - tinyint_u TINYINT UNSIGNED, - - smallint_ SMALLINT, - smallint_u SMALLINT UNSIGNED, - - mediumint_ MEDIUMINT, - mediumint_u MEDIUMINT UNSIGNED, - - int_ INT, - int_u INT UNSIGNED, - - bigint_ BIGINT, - bigint_u BIGINT UNSIGNED, - - -- - primary key (pk) -) engine=innodb default charset=utf8; - -INSERT INTO customers3 (pk,tinyint_,tinyint_u,smallint_,smallint_u,mediumint_,mediumint_u,int_,int_u,bigint_,bigint_u) VALUES ( - 1, - - -128, -- tinyint_ - 0, -- tinyint_u - - -32768, -- smallint_ - 0, -- smallint_u - - -8388608, -- mediumint_ - 0, -- mediumint_u - - -2147483648, -- int_ - 0, -- int_u - - -9223372036854775808, -- bigint_ - 0 -- bigint_u -); - -INSERT INTO customers3 (pk,tinyint_,tinyint_u,smallint_,smallint_u,mediumint_,mediumint_u,int_,int_u,bigint_,bigint_u) VALUES ( - 2, - - 127, -- tinyint_ - 255, -- tinyint_u - - 32767, -- smallint_ - 65535, -- smallint_u - - 8388607, -- mediumint_ - 16777215, -- mediumint_u - - 2147483647, -- int_ - 4294967295, -- int_u - - 9223372036854775807, -- bigint_ - 18446744073709551615 -- bigint_u -); diff --git a/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted b/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted deleted file mode 100644 index a2c69513..00000000 --- a/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted +++ /dev/null @@ -1,181 +0,0 @@ --- MySQL dump 10.13 Distrib 5.7.40, for linux-glibc2.12 (x86_64) --- --- Host: 127.0.0.1 Database: target --- ------------------------------------------------------ --- Server version 5.7.40-log - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8mb4 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Current Database: `target` --- - -CREATE DATABASE /*!32312 IF NOT EXISTS*/ `target` /*!40100 DEFAULT CHARACTER SET latin1 */; - -USE `target`; - --- --- Table structure for table `test` --- - -DROP TABLE IF EXISTS `test`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `test` ( - `id` int(11) NOT NULL, - `f` float DEFAULT NULL, - `f10` float DEFAULT NULL, - `f22` float DEFAULT NULL, - `f24` float DEFAULT NULL, - `f30` double DEFAULT NULL, - `f10_5` float(10,5) DEFAULT NULL, - `f20_10` float(20,10) DEFAULT NULL, - `f10_2` float(10,2) DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `test` --- - -LOCK TABLES `test` WRITE; -/*!40000 ALTER TABLE `test` DISABLE KEYS */; -INSERT INTO `test` VALUES (1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (2,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (3,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (4,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (5,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (6,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (7,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (8,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (9,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (10,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (11,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (12,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (13,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (14,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (15,NULL,NULL,0,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (16,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (17,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (18,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (19,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (20,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (21,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (22,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (23,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (24,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (25,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (26,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (27,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (28,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (29,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL); -INSERT INTO `test` VALUES (30,NULL,NULL,NULL,NULL,0.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (31,NULL,NULL,NULL,NULL,1.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (32,NULL,NULL,NULL,NULL,0.000000119209291,NULL,NULL,NULL); -INSERT INTO `test` VALUES (33,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (34,NULL,NULL,NULL,NULL,1.0000000000000002,NULL,NULL,NULL); -INSERT INTO `test` VALUES (35,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (36,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (37,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (38,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (39,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (40,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (41,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (42,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (43,NULL,NULL,NULL,NULL,NULL,12345.12305,NULL,NULL); -INSERT INTO `test` VALUES (44,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (45,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (46,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (47,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (48,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (49,NULL,NULL,NULL,NULL,NULL,NULL,1.0000001192,NULL); -INSERT INTO `test` VALUES (50,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (51,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (52,NULL,NULL,NULL,NULL,NULL,NULL,1.0000000000,NULL); -INSERT INTO `test` VALUES (53,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (54,NULL,NULL,NULL,NULL,NULL,NULL,12345.1230468750,NULL); -INSERT INTO `test` VALUES (55,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (56,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (57,NULL,NULL,NULL,NULL,NULL,NULL,10000000000.0000000000,NULL); -INSERT INTO `test` VALUES (58,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1.23); -INSERT INTO `test` VALUES (101,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (102,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (103,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (104,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (105,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (106,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (107,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (108,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (109,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (110,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (111,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (112,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (113,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (114,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (115,NULL,NULL,0,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (116,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (117,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (118,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (119,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (120,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (121,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (122,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (123,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (124,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (125,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (126,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (127,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (128,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (129,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL); -INSERT INTO `test` VALUES (130,NULL,NULL,NULL,NULL,0.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (131,NULL,NULL,NULL,NULL,1.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (132,NULL,NULL,NULL,NULL,0.000000119209291,NULL,NULL,NULL); -INSERT INTO `test` VALUES (133,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (134,NULL,NULL,NULL,NULL,1.0000000000000002,NULL,NULL,NULL); -INSERT INTO `test` VALUES (135,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (136,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (137,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (138,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (139,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (140,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (141,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (142,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (143,NULL,NULL,NULL,NULL,NULL,12345.12305,NULL,NULL); -INSERT INTO `test` VALUES (144,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (145,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (146,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (147,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (148,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (149,NULL,NULL,NULL,NULL,NULL,NULL,1.0000001192,NULL); -INSERT INTO `test` VALUES (150,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (151,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (152,NULL,NULL,NULL,NULL,NULL,NULL,1.0000000000,NULL); -INSERT INTO `test` VALUES (153,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (154,NULL,NULL,NULL,NULL,NULL,NULL,12345.1230468750,NULL); -INSERT INTO `test` VALUES (155,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (156,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (157,NULL,NULL,NULL,NULL,NULL,NULL,10000000000.0000000000,NULL); -INSERT INTO `test` VALUES (158,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1.23); -/*!40000 ALTER TABLE `test` ENABLE KEYS */; -UNLOCK TABLES; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed diff --git a/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted.0 b/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted.0 deleted file mode 100644 index 9f111c47..00000000 --- a/tests/e2e/mysql2mysql/float/canondata/float.float.TestFloat/extracted.0 +++ /dev/null @@ -1,181 +0,0 @@ --- MySQL dump 10.13 Distrib 5.7.40, for linux-glibc2.12 (x86_64) --- --- Host: 127.0.0.1 Database: source --- ------------------------------------------------------ --- Server version 5.7.40-log - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8mb4 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Current Database: `source` --- - -CREATE DATABASE /*!32312 IF NOT EXISTS*/ `source` /*!40100 DEFAULT CHARACTER SET latin1 */; - -USE `source`; - --- --- Table structure for table `test` --- - -DROP TABLE IF EXISTS `test`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `test` ( - `id` int(11) NOT NULL, - `f` float DEFAULT NULL, - `f10` float DEFAULT NULL, - `f22` float DEFAULT NULL, - `f24` float DEFAULT NULL, - `f30` double DEFAULT NULL, - `f10_5` float(10,5) DEFAULT NULL, - `f20_10` float(20,10) DEFAULT NULL, - `f10_2` float(10,2) DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `test` --- - -LOCK TABLES `test` WRITE; -/*!40000 ALTER TABLE `test` DISABLE KEYS */; -INSERT INTO `test` VALUES (1,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (2,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (3,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (4,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (5,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (6,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (7,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (8,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (9,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (10,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (11,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (12,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (13,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (14,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (15,NULL,NULL,0,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (16,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (17,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (18,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (19,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (20,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (21,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (22,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (23,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (24,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (25,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (26,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (27,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (28,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (29,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL); -INSERT INTO `test` VALUES (30,NULL,NULL,NULL,NULL,0.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (31,NULL,NULL,NULL,NULL,1.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (32,NULL,NULL,NULL,NULL,0.000000119209291,NULL,NULL,NULL); -INSERT INTO `test` VALUES (33,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (34,NULL,NULL,NULL,NULL,1.0000000000000002,NULL,NULL,NULL); -INSERT INTO `test` VALUES (35,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (36,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (37,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (38,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (39,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (40,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (41,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (42,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (43,NULL,NULL,NULL,NULL,NULL,12345.12305,NULL,NULL); -INSERT INTO `test` VALUES (44,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (45,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (46,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (47,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (48,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (49,NULL,NULL,NULL,NULL,NULL,NULL,1.0000001192,NULL); -INSERT INTO `test` VALUES (50,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (51,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (52,NULL,NULL,NULL,NULL,NULL,NULL,1.0000000000,NULL); -INSERT INTO `test` VALUES (53,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (54,NULL,NULL,NULL,NULL,NULL,NULL,12345.1230468750,NULL); -INSERT INTO `test` VALUES (55,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (56,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (57,NULL,NULL,NULL,NULL,NULL,NULL,10000000000.0000000000,NULL); -INSERT INTO `test` VALUES (58,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1.23); -INSERT INTO `test` VALUES (101,0,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (102,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (103,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (104,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (105,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (106,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (107,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (108,NULL,0,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (109,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (110,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (111,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (112,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (113,NULL,1,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (114,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (115,NULL,NULL,0,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (116,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (117,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (118,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (119,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (120,NULL,NULL,1,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (121,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (122,NULL,NULL,NULL,0,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (123,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (124,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (125,NULL,NULL,NULL,0.000000119209,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (126,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (127,NULL,NULL,NULL,1,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (128,NULL,NULL,NULL,2.22045e-16,NULL,NULL,NULL,NULL); -INSERT INTO `test` VALUES (129,NULL,NULL,NULL,NULL,0,NULL,NULL,NULL); -INSERT INTO `test` VALUES (130,NULL,NULL,NULL,NULL,0.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (131,NULL,NULL,NULL,NULL,1.00000011920929,NULL,NULL,NULL); -INSERT INTO `test` VALUES (132,NULL,NULL,NULL,NULL,0.000000119209291,NULL,NULL,NULL); -INSERT INTO `test` VALUES (133,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (134,NULL,NULL,NULL,NULL,1.0000000000000002,NULL,NULL,NULL); -INSERT INTO `test` VALUES (135,NULL,NULL,NULL,NULL,2.220446049250313e-16,NULL,NULL,NULL); -INSERT INTO `test` VALUES (136,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (137,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (138,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (139,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (140,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (141,NULL,NULL,NULL,NULL,NULL,1.00000,NULL,NULL); -INSERT INTO `test` VALUES (142,NULL,NULL,NULL,NULL,NULL,0.00000,NULL,NULL); -INSERT INTO `test` VALUES (143,NULL,NULL,NULL,NULL,NULL,12345.12305,NULL,NULL); -INSERT INTO `test` VALUES (144,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (145,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (146,NULL,NULL,NULL,NULL,NULL,100000.00000,NULL,NULL); -INSERT INTO `test` VALUES (147,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (148,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (149,NULL,NULL,NULL,NULL,NULL,NULL,1.0000001192,NULL); -INSERT INTO `test` VALUES (150,NULL,NULL,NULL,NULL,NULL,NULL,0.0000001192,NULL); -INSERT INTO `test` VALUES (151,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (152,NULL,NULL,NULL,NULL,NULL,NULL,1.0000000000,NULL); -INSERT INTO `test` VALUES (153,NULL,NULL,NULL,NULL,NULL,NULL,0.0000000000,NULL); -INSERT INTO `test` VALUES (154,NULL,NULL,NULL,NULL,NULL,NULL,12345.1230468750,NULL); -INSERT INTO `test` VALUES (155,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (156,NULL,NULL,NULL,NULL,NULL,NULL,1234567936.0000000000,NULL); -INSERT INTO `test` VALUES (157,NULL,NULL,NULL,NULL,NULL,NULL,10000000000.0000000000,NULL); -INSERT INTO `test` VALUES (158,NULL,NULL,NULL,NULL,NULL,NULL,NULL,1.23); -/*!40000 ALTER TABLE `test` ENABLE KEYS */; -UNLOCK TABLES; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed diff --git a/tests/e2e/mysql2mysql/float/canondata/result.json b/tests/e2e/mysql2mysql/float/canondata/result.json deleted file mode 100644 index 3f49d323..00000000 --- a/tests/e2e/mysql2mysql/float/canondata/result.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "float.float.TestFloat": { - "dst": { - "uri": "file://float.float.TestFloat/extracted" - }, - "src": { - "uri": "file://float.float.TestFloat/extracted.0" - } - } -} diff --git a/tests/e2e/mysql2mysql/float/check_db_test.go b/tests/e2e/mysql2mysql/float/check_db_test.go deleted file mode 100644 index 7971c42c..00000000 --- a/tests/e2e/mysql2mysql/float/check_db_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package light - -import ( - _ "embed" - "testing" - "time" - - "github.com/doublecloud/tross/internal/test/canon" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = helpers.RecipeMysqlSource() - Target = helpers.RecipeMysqlTarget() - - //go:embed increment.sql - IncrementStatements string -) - -func init() { - helpers.InitSrcDst(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestFloat(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) - worker := helpers.Activate(t, transfer, nil) - defer worker.Close(t) - - helpers.ExecuteMySQLStatementsLineByLine(t, IncrementStatements, helpers.NewMySQLConnectionParams(t, Source.ToStorageParams())) - - srcStorage, dstStorage := helpers.NewMySQLStorageFromSource(t, Source), helpers.NewMySQLStorageFromTarget(t, Target) - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, Source.Database, Target.Database, "test", srcStorage, dstStorage, 30*time.Second)) - dumpSrc := helpers.MySQLDump(t, Source.ToStorageParams()) - dumpDst := helpers.MySQLDump(t, Target.ToStorageParams()) - canon.SaveJSON(t, map[string]interface{}{"src": dumpSrc, "dst": dumpDst}) -} diff --git a/tests/e2e/mysql2mysql/float/dump/dump.sql b/tests/e2e/mysql2mysql/float/dump/dump.sql deleted file mode 100644 index c06fb606..00000000 --- a/tests/e2e/mysql2mysql/float/dump/dump.sql +++ /dev/null @@ -1,73 +0,0 @@ -CREATE TABLE `test` ( - id INTEGER PRIMARY KEY, - f FLOAT, - f10 FLOAT(10), - f22 FLOAT(20), - f24 FLOAT(22), - f30 FLOAT(30), - f10_5 FLOAT(10,5), - f20_10 FLOAT(20,10), - f10_2 FLOAT(10,2) -); - --- epsilon for float32: 1.1920929e-07 --- epsilon for float64: 2.220446049250313e-16 - -INSERT INTO `test` (id, f) VALUES (1, 0); -INSERT INTO `test` (id, f) VALUES (2, 1.1920929e-07); -INSERT INTO `test` (id, f) VALUES (3, 1+1.1920929e-07); -INSERT INTO `test` (id, f) VALUES (4, 1.19209291e-07); -INSERT INTO `test` (id, f) VALUES (5, 2.220446049250313e-16); -INSERT INTO `test` (id, f) VALUES (6, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f) VALUES (7, 2.2204460492503131e-16); -INSERT INTO `test` (id, f10) VALUES (8, 0); -INSERT INTO `test` (id, f10) VALUES (9, 1.1920929e-07); -INSERT INTO `test` (id, f10) VALUES (10, 1+1.1920929e-07); -INSERT INTO `test` (id, f10) VALUES (11, 1.19209291e-07); -INSERT INTO `test` (id, f10) VALUES (12, 2.220446049250313e-16); -INSERT INTO `test` (id, f10) VALUES (13, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f10) VALUES (14, 2.2204460492503131e-16); -INSERT INTO `test` (id, f22) VALUES (15, 0); -INSERT INTO `test` (id, f22) VALUES (16, 1.1920929e-07); -INSERT INTO `test` (id, f22) VALUES (17, 1+1.1920929e-07); -INSERT INTO `test` (id, f22) VALUES (18, 1.19209291e-07); -INSERT INTO `test` (id, f22) VALUES (19, 2.220446049250313e-16); -INSERT INTO `test` (id, f22) VALUES (20, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f22) VALUES (21, 2.2204460492503131e-16); -INSERT INTO `test` (id, f24) VALUES (22, 0); -INSERT INTO `test` (id, f24) VALUES (23, 1.1920929e-07); -INSERT INTO `test` (id, f24) VALUES (24, 1+1.1920929e-07); -INSERT INTO `test` (id, f24) VALUES (25, 1.19209291e-07); -INSERT INTO `test` (id, f24) VALUES (26, 2.220446049250313e-16); -INSERT INTO `test` (id, f24) VALUES (27, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f24) VALUES (28, 2.2204460492503131e-16); -INSERT INTO `test` (id, f30) VALUES (29, 0); -INSERT INTO `test` (id, f30) VALUES (30, 1.1920929e-07); -INSERT INTO `test` (id, f30) VALUES (31, 1+1.1920929e-07); -INSERT INTO `test` (id, f30) VALUES (32, 1.19209291e-07); -INSERT INTO `test` (id, f30) VALUES (33, 2.220446049250313e-16); -INSERT INTO `test` (id, f30) VALUES (34, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f30) VALUES (35, 2.2204460492503131e-16); -INSERT INTO `test` (id, f10_5) VALUES (36, 0); -INSERT INTO `test` (id, f10_5) VALUES (37, 1.1920929e-07); -INSERT INTO `test` (id, f10_5) VALUES (38, 1+1.1920929e-07); -INSERT INTO `test` (id, f10_5) VALUES (39, 1.19209291e-07); -INSERT INTO `test` (id, f10_5) VALUES (40, 2.220446049250313e-16); -INSERT INTO `test` (id, f10_5) VALUES (41, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f10_5) VALUES (42, 2.2204460492503131e-16); -INSERT INTO `test` (id, f10_5) VALUES (43, 12345.12345); -INSERT INTO `test` (id, f10_5) VALUES (44, 1234567890.12345); -INSERT INTO `test` (id, f10_5) VALUES (45, 1234567890.0123456789); -INSERT INTO `test` (id, f10_5) VALUES (46, 12345678901234567890.01234567890123456789); -INSERT INTO `test` (id, f20_10) VALUES (47, 0); -INSERT INTO `test` (id, f20_10) VALUES (48, 1.1920929e-07); -INSERT INTO `test` (id, f20_10) VALUES (49, 1+1.1920929e-07); -INSERT INTO `test` (id, f20_10) VALUES (50, 1.19209291e-07); -INSERT INTO `test` (id, f20_10) VALUES (51, 2.220446049250313e-16); -INSERT INTO `test` (id, f20_10) VALUES (52, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f20_10) VALUES (53, 2.2204460492503131e-16); -INSERT INTO `test` (id, f20_10) VALUES (54, 12345.12345); -INSERT INTO `test` (id, f20_10) VALUES (55, 1234567890.12345); -INSERT INTO `test` (id, f20_10) VALUES (56, 1234567890.0123456789); -INSERT INTO `test` (id, f20_10) VALUES (57, 12345678901234567890.01234567890123456789); -INSERT INTO `test` (id, f10_2) VALUES (58, 1.23); diff --git a/tests/e2e/mysql2mysql/float/increment.sql b/tests/e2e/mysql2mysql/float/increment.sql deleted file mode 100644 index 2db401a2..00000000 --- a/tests/e2e/mysql2mysql/float/increment.sql +++ /dev/null @@ -1,61 +0,0 @@ --- epsilon for float32: 1.1920929e-07 --- epsilon for float64: 2.220446049250313e-16 - -INSERT INTO `test` (id, f) VALUES (101, 0); -INSERT INTO `test` (id, f) VALUES (102, 1.1920929e-07); -INSERT INTO `test` (id, f) VALUES (103, 1+1.1920929e-07); -INSERT INTO `test` (id, f) VALUES (104, 1.19209291e-07); -INSERT INTO `test` (id, f) VALUES (105, 2.220446049250313e-16); -INSERT INTO `test` (id, f) VALUES (106, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f) VALUES (107, 2.2204460492503131e-16); -INSERT INTO `test` (id, f10) VALUES (108, 0); -INSERT INTO `test` (id, f10) VALUES (109, 1.1920929e-07); -INSERT INTO `test` (id, f10) VALUES (110, 1+1.1920929e-07); -INSERT INTO `test` (id, f10) VALUES (111, 1.19209291e-07); -INSERT INTO `test` (id, f10) VALUES (112, 2.220446049250313e-16); -INSERT INTO `test` (id, f10) VALUES (113, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f10) VALUES (114, 2.2204460492503131e-16); -INSERT INTO `test` (id, f22) VALUES (115, 0); -INSERT INTO `test` (id, f22) VALUES (116, 1.1920929e-07); -INSERT INTO `test` (id, f22) VALUES (117, 1+1.1920929e-07); -INSERT INTO `test` (id, f22) VALUES (118, 1.19209291e-07); -INSERT INTO `test` (id, f22) VALUES (119, 2.220446049250313e-16); -INSERT INTO `test` (id, f22) VALUES (120, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f22) VALUES (121, 2.2204460492503131e-16); -INSERT INTO `test` (id, f24) VALUES (122, 0); -INSERT INTO `test` (id, f24) VALUES (123, 1.1920929e-07); -INSERT INTO `test` (id, f24) VALUES (124, 1+1.1920929e-07); -INSERT INTO `test` (id, f24) VALUES (125, 1.19209291e-07); -INSERT INTO `test` (id, f24) VALUES (126, 2.220446049250313e-16); -INSERT INTO `test` (id, f24) VALUES (127, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f24) VALUES (128, 2.2204460492503131e-16); -INSERT INTO `test` (id, f30) VALUES (129, 0); -INSERT INTO `test` (id, f30) VALUES (130, 1.1920929e-07); -INSERT INTO `test` (id, f30) VALUES (131, 1+1.1920929e-07); -INSERT INTO `test` (id, f30) VALUES (132, 1.19209291e-07); -INSERT INTO `test` (id, f30) VALUES (133, 2.220446049250313e-16); -INSERT INTO `test` (id, f30) VALUES (134, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f30) VALUES (135, 2.2204460492503131e-16); -INSERT INTO `test` (id, f10_5) VALUES (136, 0); -INSERT INTO `test` (id, f10_5) VALUES (137, 1.1920929e-07); -INSERT INTO `test` (id, f10_5) VALUES (138, 1+1.1920929e-07); -INSERT INTO `test` (id, f10_5) VALUES (139, 1.19209291e-07); -INSERT INTO `test` (id, f10_5) VALUES (140, 2.220446049250313e-16); -INSERT INTO `test` (id, f10_5) VALUES (141, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f10_5) VALUES (142, 2.2204460492503131e-16); -INSERT INTO `test` (id, f10_5) VALUES (143, 12345.12345); -INSERT INTO `test` (id, f10_5) VALUES (144, 1234567890.12345); -INSERT INTO `test` (id, f10_5) VALUES (145, 1234567890.0123456789); -INSERT INTO `test` (id, f10_5) VALUES (146, 12345678901234567890.01234567890123456789); -INSERT INTO `test` (id, f20_10) VALUES (147, 0); -INSERT INTO `test` (id, f20_10) VALUES (148, 1.1920929e-07); -INSERT INTO `test` (id, f20_10) VALUES (149, 1+1.1920929e-07); -INSERT INTO `test` (id, f20_10) VALUES (150, 1.19209291e-07); -INSERT INTO `test` (id, f20_10) VALUES (151, 2.220446049250313e-16); -INSERT INTO `test` (id, f20_10) VALUES (152, 1+2.220446049250313e-16); -INSERT INTO `test` (id, f20_10) VALUES (153, 2.2204460492503131e-16); -INSERT INTO `test` (id, f20_10) VALUES (154, 12345.12345); -INSERT INTO `test` (id, f20_10) VALUES (155, 1234567890.12345); -INSERT INTO `test` (id, f20_10) VALUES (156, 1234567890.0123456789); -INSERT INTO `test` (id, f20_10) VALUES (157, 12345678901234567890.01234567890123456789); -INSERT INTO `test` (id, f10_2) VALUES (158, 1.23); diff --git a/tests/e2e/mysql2mysql/geometry/check_db_test.go b/tests/e2e/mysql2mysql/geometry/check_db_test.go deleted file mode 100644 index 8b24bd13..00000000 --- a/tests/e2e/mysql2mysql/geometry/check_db_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package geometry_test - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Replication) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Replication(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := &server.Transfer{ - ID: "test-id", - Src: &Source, - Dst: &Target, - } - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - `insert into geo_test(g, p, l, poly, mp, ml, mpoly, gs) values - ( - ST_GeomFromText('LINESTRING(15 15, 20 20)', 4326), - ST_GeomFromText('POINT(15 20)', 4326), - ST_GeomFromText('LINESTRING(0 0, 10 10, 20 25, 50 60)', 4326), - ST_GeomFromText('POLYGON((0 0, 10 0, 10 10, 0 10, 0 0),(5 5, 7 5, 7 7, 5 7, 5 5))', 4326), - ST_GeomFromText('MULTIPOINT(0 0, 20 20, 60 60)', 4326), - ST_GeomFromText('MULTILINESTRING((10 10, 20 20), (15 15, 30 15))', 4326), - ST_GeomFromText('MULTIPOLYGON(((0 0, 10 0, 10 10, 0 10, 0 0)),((5 5, 7 5, 7 7, 5 7, 5 5)))', 4326), - ST_GeomFromText('GEOMETRYCOLLECTION(POINT(10 10), POINT(30 30), LINESTRING(15 15, 20 20))', 4326) - );`, - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "geo_test", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/geometry/dump/geometry.sql b/tests/e2e/mysql2mysql/geometry/dump/geometry.sql deleted file mode 100644 index cb9d3aa2..00000000 --- a/tests/e2e/mysql2mysql/geometry/dump/geometry.sql +++ /dev/null @@ -1,23 +0,0 @@ -create table geo_test ( - id integer not null auto_increment primary key, - g geometry /*!80003 SRID 4326 */, - p point /*!80003 SRID 4326 */, - l linestring /*!80003 SRID 4326 */, - poly polygon /*!80003 SRID 4326 */, - mp multipoint /*!80003 SRID 4326 */, - ml multilinestring /*!80003 SRID 4326 */, - mpoly multipolygon /*!80003 SRID 4326 */, - gs geometrycollection /*!80003 SRID 4326 */ -); - -insert into geo_test(g, p, l, poly, mp, ml, mpoly, gs) values -( - ST_GeomFromText('LINESTRING(15 15, 20 20)', 4326), - ST_GeomFromText('POINT(15 20)', 4326), - ST_GeomFromText('LINESTRING(0 0, 10 10, 20 25, 50 60)', 4326), - ST_GeomFromText('POLYGON((0 0, 10 0, 10 10, 0 10, 0 0),(5 5, 7 5, 7 7, 5 7, 5 5))', 4326), - ST_GeomFromText('MULTIPOINT(0 0, 20 20, 60 60)', 4326), - ST_GeomFromText('MULTILINESTRING((10 10, 20 20), (15 15, 30 15))', 4326), - ST_GeomFromText('MULTIPOLYGON(((0 0, 10 0, 10 10, 0 10, 0 0)),((5 5, 7 5, 7 7, 5 7, 5 5)))', 4326), - ST_GeomFromText('GEOMETRYCOLLECTION(POINT(10 10), POINT(30 30), LINESTRING(15 15, 20 20))', 4326) -); diff --git a/tests/e2e/mysql2mysql/light/check_db_test.go b/tests/e2e/mysql2mysql/light/check_db_test.go deleted file mode 100644 index 8415a536..00000000 --- a/tests/e2e/mysql2mysql/light/check_db_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package light - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - require.NoError(t, tasks.ActivateDelivery(context.TODO(), nil, coordinator.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/light/dump/type_check.sql b/tests/e2e/mysql2mysql/light/dump/type_check.sql deleted file mode 100644 index fd4310c5..00000000 --- a/tests/e2e/mysql2mysql/light/dump/type_check.sql +++ /dev/null @@ -1,155 +0,0 @@ --- needs to be sure there is db1 -create table __test ( - id bigint(64) unsigned not null, - aid integer unsigned auto_increment, - - -- numeric - f float, - d double, - de decimal(10,2), - ti tinyint, - mi mediumint, - i int, - bi bigint, - biu bigint unsigned, - b bit(8), - - -- date time - da date, - ts timestamp, - dt datetime, - tm time, - y year, - - -- strings - c char, - str varchar(256), - t text, - bb blob, - - -- binary - bin binary(10), - vbin varbinary(100), - - -- other - e enum ("e1", "e2"), - se set('a', 'b', 'c'), - j json, - primary key (aid, str, id) -- test multi pk and reverse order keys -) engine=innodb default charset=utf8; - -insert into __test values ( - 1, - 0, - 1.45e-10, - 3.14e-100, - 2.5, - -124, - 32765, - -8388605, - 2147483642, - 9223372036854775804, - - b'10101111', - - '2005-03-04', - now(), - now(), - now(), - '2099', - - '1', - 'hello, friend of mine', - 'okay, now bye-bye', - 'this it actually text but blob', - 'a\0deadbeef', - 'cafebabe', - "e1", - 'a', - '{"yandex is the best place to work at": ["wish i", "would stay", 4.15, {"here after":"the "}, ["i", ["n", ["t", "e r n s h i"], "p"]]]}' -), ( - 2, - 1, - 1.34e-10, - null, - null, - -12, - 1123, - -1294129412, - 112412412421941041, - 129491244912401240, - - b'10000001', - - '1999-03-04', - now(), - null, - now(), - '1971', - - '2', - 'another hello', - 'okay, another bye', - 'another blob', - 'cafebabeda', - '\0\0\0\0\1', - "e2", - 'b', - '{"simpler": ["than", 13e-10, {"it": {"could": "be"}}]}' -), ( - 3, - 4, - 5.34e-10, - null, - 123, - -122, - -1123, - 294129412, - -784124124219410491, - 129491098649360240, - - b'10000010', - - '1999-03-05', - null, - now(), - now(), - '1972', - - 'c', - 'another another hello', - 'okay, another another bye', - 'another another blob but looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo' - 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' - 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' - 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' - 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' - 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' - 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' - 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' - 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' - 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' - 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg', - 'caafebabee', - '\0\0\0\0\1abcd124edb', - "e1", - 'c', - '{"simpler": ["than", 13e-10, {"it": {"could": ["be", "no", "ideas ", " again"], "sorry": null}}]}' -); - -insert into __test (str, id) values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) values ('nvaapsijfapfn', 201, now()), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But I\'ll be back, this is public promise', 422, now()), - ('Remember me, my name is hazzus', 333, now()); - -insert into __test (id, str, mi) values (2020, 'thanks for everything, my team', 5), - (2019, 'and other guys I worked with', 5); - -insert into __test (id, j) values (3000, '{"\\"": "\\\\", "''": []}'); -- JSON: {"\"": "\\", "'": []} diff --git a/tests/e2e/mysql2mysql/light_all_datatypes/check_db_test.go b/tests/e2e/mysql2mysql/light_all_datatypes/check_db_test.go deleted file mode 100644 index 344f7ec9..00000000 --- a/tests/e2e/mysql2mysql/light_all_datatypes/check_db_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("Tables on source: %v", tables) - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - insertRequest := strings.ReplaceAll(` - INSERT INTO __test - (%stinyint%s, %stinyint_def%s, %stinyint_u%s, %stinyint_z%s, %ssmallint%s, %ssmallint_u%s, %ssmallint_z%s, %smediumint%s, %smediumint_u%s, %smediumint_z%s, %sint%s , %sint_u%s , %sint_z%s , %sbigint%s , %sbigint_u%s , %sbigint_z%s , %sbool%s, %sdecimal_10_2%s ,%sdecimal_65_30%s, %sdecimal_65_0%s, %sdec%s , %snumeric%s , %sfloat%s , %sfloat_z%s, %sfloat_53%s, %sreal%s, %sdouble%s , %sdouble_precision%s, %sbit%s, %sbit_5%s, %sbit_9%s, %sbit_64%s, %sdate%s , %sdatetime%s , %sdatetime_6%s , %stimestamp%s , %stimestamp_2%s , %stime%s , %stime_2%s , %syear%s, %schar%s, %svarchar%s, %svarchar_def%s, %sbinary%s, %svarbinary%s, %stinyblob%s, %sblob%s, %smediumblob%s, %slongblob%s, %stinytext%s, %stext%s, %smediumtext%s, %slongtext%s, %senum%s , %sset%s, %sjson%s ) - VALUES - (-128 , -128 , 0 , 0 , -32768 , 0 , 0 , -8388608 , 0 , 0 , -2147483648 , 0 , 0 , -9223372036854775808 , 0 , 0 , 0 , '3.50' , NULL , NULL , '3.50' , '3.50' , 1.175494351E-38 , NULL , NULL , NULL , -1.7976931348623157E+308 , NULL , 0 , 0 , NULL , NULL , '1970-01-01' , '1000-01-01 00:00:00' , '1000-01-01 00:00:00' , '1970-01-01 03:00:01' , '1970-01-01 03:00:01' , '-838:59:59' , '-838:59:59' , '1901' , 0 , '' , '' , '' , '' , '' , '' , '' , '' , '' , '' , '' , '' , '1' , '1' , '{}' ) - , - (127 , 127 , 255 , 255 , 32767 , 65535 , 65535 , 8388607 , 16777215 , 16777215 , 2147483647 , 4294967295 , 4294967295 , 9223372036854775807 , 18446744073709551615 , 18446744073709551615 , 1 , '12345678.1' , NULL , NULL , '12345678.1' , '12345678.1' , 3.402823466E+7 , NULL , NULL , NULL , -2.2250738585072014E-308 , NULL , 1 , 31 , NULL , NULL , '2038-01-19' , '9999-12-31 23:59:59' , '9999-12-31 23:59:59' , '2038-01-19 03:14:07' , '2038-01-19 03:14:07' , '838:59:59' , '838:59:59' , '2155' , 255 , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , '3' , '3' , '{"a":"b" , "c":1 , "d":{} , "e":[]}') - ; - `, "%s", "`") - - tx, err := conn.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelRepeatableRead, - }) - require.NoError(t, err) - - _, err = tx.Query(insertRequest) - require.NoError(t, err) - err = tx.Commit() - require.NoError(t, err) - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "__test", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/light_all_datatypes/dump/type_check.sql b/tests/e2e/mysql2mysql/light_all_datatypes/dump/type_check.sql deleted file mode 100644 index 2c014fe0..00000000 --- a/tests/e2e/mysql2mysql/light_all_datatypes/dump/type_check.sql +++ /dev/null @@ -1,78 +0,0 @@ -CREATE TABLE `__test` ( - -- If you specify ZEROFILL for a numeric column, MySQL automatically adds the UNSIGNED attribute to the column. - `tinyint` TINYINT, - `tinyint_def` TINYINT DEFAULT 0, - `tinyint_u` TINYINT UNSIGNED, - `tinyint_z` TINYINT ZEROFILL, - `smallint` SMALLINT, - `smallint_u` SMALLINT UNSIGNED, - `smallint_z` SMALLINT ZEROFILL, - `mediumint` MEDIUMINT, - `mediumint_u` MEDIUMINT UNSIGNED, - `mediumint_z` MEDIUMINT ZEROFILL, - `int` INT, - `int_u` INT UNSIGNED, - `int_z` INT ZEROFILL, - `bigint` BIGINT, - `bigint_u` BIGINT UNSIGNED, - `bigint_z` BIGINT ZEROFILL, - - `bool` BOOL, -- synonym to TINYINT(1) - - `decimal_10_2` DECIMAL(10, 2), -- synonyms: decimal, dec, numeric, fixed - `decimal_65_30` DECIMAL(65, 30), - `decimal_65_0` DECIMAL(65, 0), - `dec` DEC, - `numeric` NUMERIC(11, 3), - `fixed` FIXED, - - -- "As of MySQL 8.0.17, the UNSIGNED attribute is deprecated for columns of type FLOAT, DOUBLE, and DECIMAL (and any synonyms); you should expect support for it to be removed in a future version of MySQL." - `float` FLOAT(10, 2), -- "As of MySQL 8.0.17, the nonstandard FLOAT(M,D) and DOUBLE(M,D) syntax is deprecated and you should expect support for it to be removed in a future version of MySQL." - `float_z` FLOAT(10, 2) ZEROFILL, -- same - `float_53` FLOAT(53), -- same - `real` REAL(10, 2), -- same && synonym to FLOAT - `double` DOUBLE, - `double_precision` DOUBLE PRECISION, - - `bit` BIT, - `bit_5` BIT(5), - `bit_9` BIT(9), - `bit_64` BIT(64), - - `date` DATE, - `datetime` DATETIME, - `datetime_6` DATETIME(6), - `timestamp` TIMESTAMP, - `timestamp_2` TIMESTAMP(2), - - `time` TIME, - `time_2` TIME(2), - `year` YEAR, - - `char` CHAR(10), - `varchar` VARCHAR(20), - `varchar_def` VARCHAR(20) DEFAULT 'default_value', - - `binary` BINARY(20), - `varbinary` VARBINARY(20), - - `tinyblob` TINYBLOB, - `blob` BLOB, - `mediumblob` MEDIUMBLOB, - `longblob` LONGBLOB, - - `tinytext` TINYTEXT , - `text` TEXT, - `mediumtext` MEDIUMTEXT , - `longtext` LONGTEXT , - - `enum` ENUM('1', '2', '3'), - `set` SET ('1', '2', '3'), - - -- json - - `json` JSON, - - - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY -- just to have a primary key -) engine=innodb default charset=utf8; diff --git a/tests/e2e/mysql2mysql/medium/check_db_test.go b/tests/e2e/mysql2mysql/medium/check_db_test.go deleted file mode 100644 index 98bb3c5e..00000000 --- a/tests/e2e/mysql2mysql/medium/check_db_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package light - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - require.NoError(t, tasks.ActivateDelivery(context.TODO(), nil, client2.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/no_auto_value_on_zero/check_db_test.go b/tests/e2e/mysql2mysql/no_auto_value_on_zero/check_db_test.go deleted file mode 100644 index 82ca9f1b..00000000 --- a/tests/e2e/mysql2mysql/no_auto_value_on_zero/check_db_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - require.NoError(t, snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true)) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - modeRequest := `SET SESSION sql_mode='NO_AUTO_VALUE_ON_ZERO'` - insertRequest := `INSERT INTO __test_special_values (id, data) VALUES - (0, 1), - (NULL, 2), - (NULL, 3)` - - tx, err := conn.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelRepeatableRead, - }) - require.NoError(t, err) - - _, err = tx.Query(modeRequest) - require.NoError(t, err) - _, err = tx.Query(insertRequest) - require.NoError(t, err) - err = tx.Commit() - require.NoError(t, err) - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "__test_special_values", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/no_auto_value_on_zero/dump/no_auto_value_on_zero.sql b/tests/e2e/mysql2mysql/no_auto_value_on_zero/dump/no_auto_value_on_zero.sql deleted file mode 100644 index 0b9e9c8a..00000000 --- a/tests/e2e/mysql2mysql/no_auto_value_on_zero/dump/no_auto_value_on_zero.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE `__test_special_values` ( - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY, - `data` int NOT NULL -) engine=innodb default charset=utf8; diff --git a/tests/e2e/mysql2mysql/partitioned_table/check_db_test.go b/tests/e2e/mysql2mysql/partitioned_table/check_db_test.go deleted file mode 100644 index 215d4d86..00000000 --- a/tests/e2e/mysql2mysql/partitioned_table/check_db_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package partitionedtable - -import ( - "database/sql" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - mysql "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID -} - -type TesttableRow struct { - ID int - Value string -} - -func TestPartitionedTable(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - sourceDB := connectToMysql(t, Source.ToStorageParams()) - defer sourceDB.Close() - targetDB := connectToMysql(t, Target.ToStorageParams()) - defer targetDB.Close() - - checkTableIsEmpty(t, targetDB) - - testRow := TesttableRow{ID: 1, Value: "kek"} - insertOneRow(t, sourceDB, testRow) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "testtable", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func connectToMysql(t *testing.T, storageParams *mysql.MysqlStorageParams) *sql.DB { - connParams, err := mysql.NewConnectionParams(storageParams) - require.NoError(t, err) - - db, err := mysql.Connect(connParams, nil) - require.NoError(t, err) - return db -} - -func checkTableIsEmpty(t *testing.T, db *sql.DB) { - var count int - require.NoError(t, db.QueryRow(`select count(*) from testtable`).Scan(&count)) - require.Equal(t, 0, count) -} - -func insertOneRow(t *testing.T, db *sql.DB, testRow TesttableRow) { - _, err := db.Exec(`insert into testtable (id, value) values (?, ?)`, testRow.ID, testRow.Value) - require.NoError(t, err) -} diff --git a/tests/e2e/mysql2mysql/partitioned_table/dump/dump.sql b/tests/e2e/mysql2mysql/partitioned_table/dump/dump.sql deleted file mode 100644 index df0d3f37..00000000 --- a/tests/e2e/mysql2mysql/partitioned_table/dump/dump.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE testtable ( - id INTEGER PRIMARY KEY, - value text -) ENGINE=InnoDB -/*!50100 PARTITION BY RANGE (id) ( - PARTITION testtable_2009_05 VALUES LESS THAN (733893) ENGINE = InnoDB, - PARTITION testtable_now VALUES LESS THAN MAXVALUE ENGINE = InnoDB -)*/ -; diff --git a/tests/e2e/mysql2mysql/pkeychanges/check_db_test.go b/tests/e2e/mysql2mysql/pkeychanges/check_db_test.go deleted file mode 100644 index ffbf45e9..00000000 --- a/tests/e2e/mysql2mysql/pkeychanges/check_db_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err := mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - queries := []string{ - "update api_reports_offline set taskid = 10, ClientID = 1, ReportName = 'test' where taskid = 1", - "update api_reports_offline set taskid = 20, ClientID = 2, ReportName = 'test' where taskid = 2", - "update api_reports_offline set taskid = 30, ClientID = 3, ReportName = 'test' where taskid = 3", - "delete from api_reports_offline where taskid = 10", - } - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - transaction, err := conn.BeginTx(context.Background(), nil) - require.NoError(t, err) - - for _, query := range queries { - _, err = transaction.Exec(query) - require.NoError(t, err) - } - - err = transaction.Commit() - require.NoError(t, err) - - err = conn.Close() - require.NoError(t, err) - - sourceStorage := helpers.GetSampleableStorageByModel(t, Source) - targetStorage := helpers.GetSampleableStorageByModel(t, Target) - testTableName := "api_reports_offline" - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, testTableName, - sourceStorage, - targetStorage, - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/pkeychanges/dump/type_check.sql b/tests/e2e/mysql2mysql/pkeychanges/dump/type_check.sql deleted file mode 100644 index 298bd5e0..00000000 --- a/tests/e2e/mysql2mysql/pkeychanges/dump/type_check.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE `api_reports_offline` -( - `taskid` bigint(20) NOT NULL AUTO_INCREMENT, - `ClientID` int(10) unsigned NOT NULL, - `ReportName` varchar(255) NOT NULL, - PRIMARY KEY (`taskid`), - UNIQUE KEY `i_ClientID_ReportName` (`ClientID`, `ReportName`) - ) ENGINE = InnoDB - AUTO_INCREMENT = 31793242 - DEFAULT CHARSET = utf8; - -insert into `api_reports_offline` (taskid, ClientID, ReportName) -values (1, 1, 'test'), - (2, 2, 'test'), - (3, 3, 'test'); diff --git a/tests/e2e/mysql2mysql/replace_fkey/common/test.go b/tests/e2e/mysql2mysql/replace_fkey/common/test.go deleted file mode 100644 index 2d611d62..00000000 --- a/tests/e2e/mysql2mysql/replace_fkey/common/test.go +++ /dev/null @@ -1,159 +0,0 @@ -package replacefkeycommon - -import ( - "context" - "database/sql" - "errors" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/util" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not to go to vanga - Source.WithDefaults() - Target.WithDefaults() - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - require.NoError(t, tasks.ActivateDelivery(context.TODO(), nil, coordinator.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func defaultChecksumParams() *tasks.ChecksumParameters { - return &tasks.ChecksumParameters{ - TableSizeThreshold: 0, - Tables: []abstract.TableDescription{ - {Name: "test_src", Schema: Target.Database, Filter: "", EtaRow: uint64(0), Offset: uint64(0)}, - {Name: "test_dst", Schema: Target.Database, Filter: "", EtaRow: uint64(0), Offset: uint64(0)}, - }, - PriorityComparators: nil, - } -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - connector := &server.Transfer{ - ID: "test-id", - Src: &Source, - Dst: &Target, - } - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), connector, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("Tables on source: %v", tables) - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - defer func() { - require.NoError(t, conn.Close()) - }() - - var rollbacks util.Rollbacks - defer rollbacks.Do() - tx, err := conn.BeginTx(context.Background(), &sql.TxOptions{ - Isolation: sql.LevelRepeatableRead, - }) - require.NoError(t, err) - rollbacks.Add(func() { - _ = tx.Rollback() - }) - _, err = tx.Exec("INSERT `test_src` VALUES (1, 'test2') ON DUPLICATE KEY UPDATE `name` = 'test2'") - require.NoError(t, err) - require.NoError(t, tx.Commit()) - rollbacks.Cancel() - - require.NoError(t, waitForSync(t)) - - require.NoError(t, tasks.Checksum(*transfer, logger.Log, helpers.EmptyRegistry(), defaultChecksumParams())) -} - -func waitForSync(t *testing.T) error { - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Target.Host, Target.Port) - cfg.User = Target.User - cfg.Passwd = string(Target.Password) - cfg.DBName = Target.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - const wait = 1 * time.Second - const maxWait = 120 * wait // 2 min - for passed := 0 * time.Second; passed < maxWait; passed += wait { - time.Sleep(wait) - - var name string - err := conn.QueryRowContext(context.Background(), "SELECT name FROM test_src WHERE id = 1").Scan(&name) - if err == nil && name == "test2" { - return nil - } - } - - return errors.New("incorrect rows count or sync timeout") -} diff --git a/tests/e2e/mysql2mysql/replace_fkey/dump/fkey.sql b/tests/e2e/mysql2mysql/replace_fkey/dump/fkey.sql deleted file mode 100644 index 4db744d3..00000000 --- a/tests/e2e/mysql2mysql/replace_fkey/dump/fkey.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TABLE `test_src` ( - `id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY, - `name` varchar(10), - UNIQUE (`name`) -) engine=innodb default charset=utf8; - -CREATE TABLE `test_dst` ( - `src_id` integer NOT NULL AUTO_INCREMENT PRIMARY KEY, - FOREIGN KEY (`src_id`) REFERENCES `test_src` (`id`) -) engine=innodb default charset=utf8; - -INSERT INTO `test_src` VALUES (1, 'test'); -INSERT INTO `test_dst` VALUES (1); diff --git a/tests/e2e/mysql2mysql/replace_fkey/test_per_table/check_db_test.go b/tests/e2e/mysql2mysql/replace_fkey/test_per_table/check_db_test.go deleted file mode 100644 index 495b3404..00000000 --- a/tests/e2e/mysql2mysql/replace_fkey/test_per_table/check_db_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package replacefkeypertable - -import ( - "testing" - - test "github.com/doublecloud/tross/tests/e2e/mysql2mysql/replace_fkey/common" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: test.Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: test.Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", test.Existence) - t.Run("Snapshot", test.Snapshot) - t.Run("Replication", test.Load) - }) -} diff --git a/tests/e2e/mysql2mysql/replace_fkey/test_per_transaction/check_db_test.go b/tests/e2e/mysql2mysql/replace_fkey/test_per_transaction/check_db_test.go deleted file mode 100644 index 358f733a..00000000 --- a/tests/e2e/mysql2mysql/replace_fkey/test_per_transaction/check_db_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package replacefkeypertrans - -import ( - "testing" - - test "github.com/doublecloud/tross/tests/e2e/mysql2mysql/replace_fkey/common" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: test.Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: test.Target.Port}, - )) - }() - - test.Target.PerTransactionPush = true - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", test.Existence) - t.Run("Snapshot", test.Snapshot) - t.Run("Replication", test.Load) - }) -} diff --git a/tests/e2e/mysql2mysql/scheme/check_db_test.go b/tests/e2e/mysql2mysql/scheme/check_db_test.go deleted file mode 100644 index 686a9124..00000000 --- a/tests/e2e/mysql2mysql/scheme/check_db_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package light - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - Source.PreSteps.View = true - Source.PreSteps.Routine = true - Source.PreSteps.Trigger = false - Source.PostSteps.View = false - Source.PostSteps.Routine = false - Source.PostSteps.Trigger = true -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Schema) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Schema(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - require.NoError(t, tasks.ActivateDelivery(context.TODO(), nil, client2.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/scheme/dump/scheme.sql b/tests/e2e/mysql2mysql/scheme/dump/scheme.sql deleted file mode 100644 index 5e55f5b3..00000000 --- a/tests/e2e/mysql2mysql/scheme/dump/scheme.sql +++ /dev/null @@ -1,711 +0,0 @@ -SET NAMES utf8mb4; -SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; -SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; -SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL'; - --- --- Table structure for table `actor` --- - -CREATE TABLE actor ( - actor_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - first_name VARCHAR(45) NOT NULL, - last_name VARCHAR(45) NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (actor_id), - KEY idx_actor_last_name (last_name) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `address` --- - -CREATE TABLE address ( - address_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - address VARCHAR(50) NOT NULL, - address2 VARCHAR(50) DEFAULT NULL, - district VARCHAR(20) NOT NULL, - city_id SMALLINT UNSIGNED NOT NULL, - postal_code VARCHAR(10) DEFAULT NULL, - phone VARCHAR(20) NOT NULL, - -- Add GEOMETRY column for MySQL 5.7.5 and higher - -- Also include SRID attribute for MySQL 8.0.3 and higher - /*!50705 location GEOMETRY */ /*!80003 SRID 0 */ /*!50705 NOT NULL,*/ - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (address_id), - KEY idx_fk_city_id (city_id), - /*!50705 SPATIAL KEY `idx_location` (location),*/ - CONSTRAINT `fk_address_city` FOREIGN KEY (city_id) REFERENCES city (city_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `category` --- - -CREATE TABLE category ( - category_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, - name VARCHAR(25) NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (category_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `city` --- - -CREATE TABLE city ( - city_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - city VARCHAR(50) NOT NULL, - country_id SMALLINT UNSIGNED NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (city_id), - KEY idx_fk_country_id (country_id), - CONSTRAINT `fk_city_country` FOREIGN KEY (country_id) REFERENCES country (country_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `country` --- - -CREATE TABLE country ( - country_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - country VARCHAR(50) NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (country_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `customer` --- - -CREATE TABLE customer ( - customer_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - store_id TINYINT UNSIGNED NOT NULL, - first_name VARCHAR(45) NOT NULL, - last_name VARCHAR(45) NOT NULL, - email VARCHAR(50) DEFAULT NULL, - address_id SMALLINT UNSIGNED NOT NULL, - active BOOLEAN NOT NULL DEFAULT TRUE, - create_date DATETIME NOT NULL, - last_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (customer_id), - KEY idx_fk_store_id (store_id), - KEY idx_fk_address_id (address_id), - KEY idx_last_name (last_name), - CONSTRAINT fk_customer_address FOREIGN KEY (address_id) REFERENCES address (address_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_customer_store FOREIGN KEY (store_id) REFERENCES store (store_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `film` --- - -CREATE TABLE film ( - film_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - title VARCHAR(128) NOT NULL, - description TEXT DEFAULT NULL, - release_year YEAR DEFAULT NULL, - language_id TINYINT UNSIGNED NOT NULL, - original_language_id TINYINT UNSIGNED DEFAULT NULL, - rental_duration TINYINT UNSIGNED NOT NULL DEFAULT 3, - rental_rate DECIMAL(4,2) NOT NULL DEFAULT 4.99, - length SMALLINT UNSIGNED DEFAULT NULL, - replacement_cost DECIMAL(5,2) NOT NULL DEFAULT 19.99, - rating ENUM('G','PG','PG-13','R','NC-17') DEFAULT 'G', - special_features SET('Trailers','Commentaries','Deleted Scenes','Behind the Scenes') DEFAULT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (film_id), - KEY idx_title (title), - KEY idx_fk_language_id (language_id), - KEY idx_fk_original_language_id (original_language_id), - CONSTRAINT fk_film_language FOREIGN KEY (language_id) REFERENCES language (language_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_film_language_original FOREIGN KEY (original_language_id) REFERENCES language (language_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `film_actor` --- - -CREATE TABLE film_actor ( - actor_id SMALLINT UNSIGNED NOT NULL, - film_id SMALLINT UNSIGNED NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (actor_id,film_id), - KEY idx_fk_film_id (`film_id`), - CONSTRAINT fk_film_actor_actor FOREIGN KEY (actor_id) REFERENCES actor (actor_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_film_actor_film FOREIGN KEY (film_id) REFERENCES film (film_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `film_category` --- - -CREATE TABLE film_category ( - film_id SMALLINT UNSIGNED NOT NULL, - category_id TINYINT UNSIGNED NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (film_id, category_id), - CONSTRAINT fk_film_category_film FOREIGN KEY (film_id) REFERENCES film (film_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_film_category_category FOREIGN KEY (category_id) REFERENCES category (category_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `film_text` --- --- InnoDB added FULLTEXT support in 5.6.10. If you use an --- earlier version, then consider upgrading (recommended) or --- changing InnoDB to MyISAM as the film_text engine --- - --- Use InnoDB for film_text as of 5.6.10, MyISAM prior to 5.6.10. -SET @old_default_storage_engine = @@default_storage_engine; -SET @@default_storage_engine = 'MyISAM'; -/*!50610 SET @@default_storage_engine = 'InnoDB'*/; - -CREATE TABLE film_text ( - film_id SMALLINT NOT NULL, - title VARCHAR(255) NOT NULL, - description TEXT, - PRIMARY KEY (film_id), - FULLTEXT KEY idx_title_description (title,description) -) DEFAULT CHARSET=utf8mb4; - -SET @@default_storage_engine = @old_default_storage_engine; - --- --- Triggers for loading film_text from film --- - -DELIMITER ;; -CREATE TRIGGER `ins_film` AFTER INSERT ON `film` FOR EACH ROW BEGIN - INSERT INTO film_text (film_id, title, description) - VALUES (new.film_id, new.title, new.description); - END;; - - -CREATE TRIGGER `upd_film` AFTER UPDATE ON `film` FOR EACH ROW BEGIN - IF (old.title != new.title) OR (old.description != new.description) OR (old.film_id != new.film_id) - THEN - UPDATE film_text - SET title=new.title, - description=new.description, - film_id=new.film_id - WHERE film_id=old.film_id; - END IF; - END;; - - -CREATE TRIGGER `del_film` AFTER DELETE ON `film` FOR EACH ROW BEGIN - DELETE FROM film_text WHERE film_id = old.film_id; - END;; - -DELIMITER ; - --- --- Table structure for table `inventory` --- - -CREATE TABLE inventory ( - inventory_id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT, - film_id SMALLINT UNSIGNED NOT NULL, - store_id TINYINT UNSIGNED NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (inventory_id), - KEY idx_fk_film_id (film_id), - KEY idx_store_id_film_id (store_id,film_id), - CONSTRAINT fk_inventory_store FOREIGN KEY (store_id) REFERENCES store (store_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_inventory_film FOREIGN KEY (film_id) REFERENCES film (film_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `language` --- - -CREATE TABLE language ( - language_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, - name CHAR(20) NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (language_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `payment` --- - -CREATE TABLE payment ( - payment_id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, - customer_id SMALLINT UNSIGNED NOT NULL, - staff_id TINYINT UNSIGNED NOT NULL, - rental_id INT DEFAULT NULL, - amount DECIMAL(5,2) NOT NULL, - payment_date DATETIME NOT NULL, - last_update TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (payment_id), - KEY idx_fk_staff_id (staff_id), - KEY idx_fk_customer_id (customer_id), - CONSTRAINT fk_payment_rental FOREIGN KEY (rental_id) REFERENCES rental (rental_id) ON DELETE SET NULL ON UPDATE CASCADE, - CONSTRAINT fk_payment_customer FOREIGN KEY (customer_id) REFERENCES customer (customer_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_payment_staff FOREIGN KEY (staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - --- --- Table structure for table `rental` --- - -CREATE TABLE rental ( - rental_id INT NOT NULL AUTO_INCREMENT, - rental_date DATETIME NOT NULL, - inventory_id MEDIUMINT UNSIGNED NOT NULL, - customer_id SMALLINT UNSIGNED NOT NULL, - return_date DATETIME DEFAULT NULL, - staff_id TINYINT UNSIGNED NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (rental_id), - UNIQUE KEY (rental_date,inventory_id,customer_id), - KEY idx_fk_inventory_id (inventory_id), - KEY idx_fk_customer_id (customer_id), - KEY idx_fk_staff_id (staff_id), - CONSTRAINT fk_rental_staff FOREIGN KEY (staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_rental_inventory FOREIGN KEY (inventory_id) REFERENCES inventory (inventory_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_rental_customer FOREIGN KEY (customer_id) REFERENCES customer (customer_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `staff` --- - -CREATE TABLE staff ( - staff_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, - first_name VARCHAR(45) NOT NULL, - last_name VARCHAR(45) NOT NULL, - address_id SMALLINT UNSIGNED NOT NULL, - picture BLOB DEFAULT NULL, - email VARCHAR(50) DEFAULT NULL, - store_id TINYINT UNSIGNED NOT NULL, - active BOOLEAN NOT NULL DEFAULT TRUE, - username VARCHAR(16) NOT NULL, - password VARCHAR(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (staff_id), - KEY idx_fk_store_id (store_id), - KEY idx_fk_address_id (address_id), - CONSTRAINT fk_staff_store FOREIGN KEY (store_id) REFERENCES store (store_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_staff_address FOREIGN KEY (address_id) REFERENCES address (address_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- Table structure for table `store` --- - -CREATE TABLE store ( - store_id TINYINT UNSIGNED NOT NULL AUTO_INCREMENT, - manager_staff_id TINYINT UNSIGNED NOT NULL, - address_id SMALLINT UNSIGNED NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (store_id), - UNIQUE KEY idx_unique_manager (manager_staff_id), - KEY idx_fk_address_id (address_id), - CONSTRAINT fk_store_staff FOREIGN KEY (manager_staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT fk_store_address FOREIGN KEY (address_id) REFERENCES address (address_id) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- --- View structure for view `customer_list` --- - -CREATE VIEW customer_list -AS -SELECT cu.customer_id AS ID, CONCAT(cu.first_name, _utf8mb4' ', cu.last_name) AS name, a.address AS address, a.postal_code AS `zip code`, - a.phone AS phone, city.city AS city, country.country AS country, IF(cu.active, _utf8mb4'active',_utf8mb4'') AS notes, cu.store_id AS SID -FROM customer AS cu JOIN address AS a ON cu.address_id = a.address_id JOIN city ON a.city_id = city.city_id - JOIN country ON city.country_id = country.country_id; - --- --- View structure for view `film_list` --- - -CREATE VIEW film_list -AS -SELECT film.film_id AS FID, film.title AS title, film.description AS description, category.name AS category, film.rental_rate AS price, - film.length AS length, film.rating AS rating, GROUP_CONCAT(CONCAT(actor.first_name, _utf8mb4' ', actor.last_name) SEPARATOR ', ') AS actors -FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id - JOIN film_actor ON film.film_id = film_actor.film_id - JOIN actor ON film_actor.actor_id = actor.actor_id -GROUP BY film.film_id, category.name; - --- --- View structure for view `nicer_but_slower_film_list` --- - -CREATE VIEW nicer_but_slower_film_list -AS -SELECT film.film_id AS FID, film.title AS title, film.description AS description, category.name AS category, film.rental_rate AS price, - film.length AS length, film.rating AS rating, GROUP_CONCAT(CONCAT(CONCAT(UCASE(SUBSTR(actor.first_name,1,1)), - LCASE(SUBSTR(actor.first_name,2,LENGTH(actor.first_name))),_utf8mb4' ',CONCAT(UCASE(SUBSTR(actor.last_name,1,1)), - LCASE(SUBSTR(actor.last_name,2,LENGTH(actor.last_name)))))) SEPARATOR ', ') AS actors -FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id - JOIN film_actor ON film.film_id = film_actor.film_id - JOIN actor ON film_actor.actor_id = actor.actor_id -GROUP BY film.film_id, category.name; - --- --- View structure for view `staff_list` --- - -CREATE VIEW staff_list -AS -SELECT s.staff_id AS ID, CONCAT(s.first_name, _utf8mb4' ', s.last_name) AS name, a.address AS address, a.postal_code AS `zip code`, a.phone AS phone, - city.city AS city, country.country AS country, s.store_id AS SID -FROM staff AS s JOIN address AS a ON s.address_id = a.address_id JOIN city ON a.city_id = city.city_id - JOIN country ON city.country_id = country.country_id; - --- --- View structure for view `sales_by_store` --- - -CREATE VIEW sales_by_store -AS -SELECT -CONCAT(c.city, _utf8mb4',', cy.country) AS store -, CONCAT(m.first_name, _utf8mb4' ', m.last_name) AS manager -, SUM(p.amount) AS total_sales -FROM payment AS p -INNER JOIN rental AS r ON p.rental_id = r.rental_id -INNER JOIN inventory AS i ON r.inventory_id = i.inventory_id -INNER JOIN store AS s ON i.store_id = s.store_id -INNER JOIN address AS a ON s.address_id = a.address_id -INNER JOIN city AS c ON a.city_id = c.city_id -INNER JOIN country AS cy ON c.country_id = cy.country_id -INNER JOIN staff AS m ON s.manager_staff_id = m.staff_id -GROUP BY s.store_id -ORDER BY cy.country, c.city; - --- --- View structure for view `sales_by_film_category` --- --- Note that total sales will add up to >100% because --- some titles belong to more than 1 category --- - -CREATE VIEW sales_by_film_category -AS -SELECT -c.name AS category -, SUM(p.amount) AS total_sales -FROM payment AS p -INNER JOIN rental AS r ON p.rental_id = r.rental_id -INNER JOIN inventory AS i ON r.inventory_id = i.inventory_id -INNER JOIN film AS f ON i.film_id = f.film_id -INNER JOIN film_category AS fc ON f.film_id = fc.film_id -INNER JOIN category AS c ON fc.category_id = c.category_id -GROUP BY c.name -ORDER BY total_sales DESC; - --- --- View structure for view `actor_info` --- - -CREATE DEFINER=CURRENT_USER SQL SECURITY INVOKER VIEW actor_info -AS -SELECT -a.actor_id, -a.first_name, -a.last_name, -GROUP_CONCAT(DISTINCT CONCAT(c.name, ': ', - (SELECT GROUP_CONCAT(f.title ORDER BY f.title SEPARATOR ', ') - FROM film f - INNER JOIN film_category fc - ON f.film_id = fc.film_id - INNER JOIN film_actor fa - ON f.film_id = fa.film_id - WHERE fc.category_id = c.category_id - AND fa.actor_id = a.actor_id - ) - ) - ORDER BY c.name SEPARATOR '; ') -AS film_info -FROM actor a -LEFT JOIN film_actor fa - ON a.actor_id = fa.actor_id -LEFT JOIN film_category fc - ON fa.film_id = fc.film_id -LEFT JOIN category c - ON fc.category_id = c.category_id -GROUP BY a.actor_id, a.first_name, a.last_name; - --- --- Procedure structure for procedure `rewards_report` --- - -DELIMITER // - -CREATE PROCEDURE rewards_report ( - IN min_monthly_purchases TINYINT UNSIGNED - , IN min_dollar_amount_purchased DECIMAL(10,2) - , OUT count_rewardees INT -) -LANGUAGE SQL -NOT DETERMINISTIC -READS SQL DATA -SQL SECURITY DEFINER -COMMENT 'Provides a customizable report on best customers' -proc: BEGIN - - DECLARE last_month_start DATE; - DECLARE last_month_end DATE; - - /* Some sanity checks... */ - IF min_monthly_purchases = 0 THEN - SELECT 'Minimum monthly purchases parameter must be > 0'; - LEAVE proc; - END IF; - IF min_dollar_amount_purchased = 0.00 THEN - SELECT 'Minimum monthly dollar amount purchased parameter must be > $0.00'; - LEAVE proc; - END IF; - - /* Determine start and end time periods */ - SET last_month_start = DATE_SUB(CURRENT_DATE(), INTERVAL 1 MONTH); - SET last_month_start = STR_TO_DATE(CONCAT(YEAR(last_month_start),'-',MONTH(last_month_start),'-01'),'%Y-%m-%d'); - SET last_month_end = LAST_DAY(last_month_start); - - /* - Create a temporary storage area for - Customer IDs. - */ - CREATE TEMPORARY TABLE tmpCustomer (customer_id SMALLINT UNSIGNED NOT NULL PRIMARY KEY); - - /* - Find all customers meeting the - monthly purchase requirements - */ - INSERT INTO tmpCustomer (customer_id) - SELECT p.customer_id - FROM payment AS p - WHERE DATE(p.payment_date) BETWEEN last_month_start AND last_month_end - GROUP BY customer_id - HAVING SUM(p.amount) > min_dollar_amount_purchased - AND COUNT(customer_id) > min_monthly_purchases; - - /* Populate OUT parameter with count of found customers */ - SELECT COUNT(*) FROM tmpCustomer INTO count_rewardees; - - /* - Output ALL customer information of matching rewardees. - Customize output as needed. - */ - SELECT c.* - FROM tmpCustomer AS t - INNER JOIN customer AS c ON t.customer_id = c.customer_id; - - /* Clean up */ - DROP TABLE tmpCustomer; -END // - -DELIMITER ; - -DELIMITER $$ - -CREATE FUNCTION get_customer_balance(p_customer_id INT, p_effective_date DATETIME) RETURNS DECIMAL(5,2) - DETERMINISTIC - READS SQL DATA -BEGIN - - #OK, WE NEED TO CALCULATE THE CURRENT BALANCE GIVEN A CUSTOMER_ID AND A DATE - #THAT WE WANT THE BALANCE TO BE EFFECTIVE FOR. THE BALANCE IS: - # 1) RENTAL FEES FOR ALL PREVIOUS RENTALS - # 2) ONE DOLLAR FOR EVERY DAY THE PREVIOUS RENTALS ARE OVERDUE - # 3) IF A FILM IS MORE THAN RENTAL_DURATION * 2 OVERDUE, CHARGE THE REPLACEMENT_COST - # 4) SUBTRACT ALL PAYMENTS MADE BEFORE THE DATE SPECIFIED - - DECLARE v_rentfees DECIMAL(5,2); #FEES PAID TO RENT THE VIDEOS INITIALLY - DECLARE v_overfees INTEGER; #LATE FEES FOR PRIOR RENTALS - DECLARE v_payments DECIMAL(5,2); #SUM OF PAYMENTS MADE PREVIOUSLY - - SELECT IFNULL(SUM(film.rental_rate),0) INTO v_rentfees - FROM film, inventory, rental - WHERE film.film_id = inventory.film_id - AND inventory.inventory_id = rental.inventory_id - AND rental.rental_date <= p_effective_date - AND rental.customer_id = p_customer_id; - - SELECT IFNULL(SUM(IF((TO_DAYS(rental.return_date) - TO_DAYS(rental.rental_date)) > film.rental_duration, - ((TO_DAYS(rental.return_date) - TO_DAYS(rental.rental_date)) - film.rental_duration),0)),0) INTO v_overfees - FROM rental, inventory, film - WHERE film.film_id = inventory.film_id - AND inventory.inventory_id = rental.inventory_id - AND rental.rental_date <= p_effective_date - AND rental.customer_id = p_customer_id; - - - SELECT IFNULL(SUM(payment.amount),0) INTO v_payments - FROM payment - - WHERE payment.payment_date <= p_effective_date - AND payment.customer_id = p_customer_id; - - RETURN v_rentfees + v_overfees - v_payments; -END $$ - -DELIMITER ; - -DELIMITER $$ - -CREATE PROCEDURE film_in_stock(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT) -READS SQL DATA -BEGIN - SELECT inventory_id - FROM inventory - WHERE film_id = p_film_id - AND store_id = p_store_id - AND inventory_in_stock(inventory_id); - - SELECT COUNT(*) - FROM inventory - WHERE film_id = p_film_id - AND store_id = p_store_id - AND inventory_in_stock(inventory_id) - INTO p_film_count; -END $$ - -DELIMITER ; - -DELIMITER $$ - -CREATE PROCEDURE film_not_in_stock(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT) -READS SQL DATA -BEGIN - SELECT inventory_id - FROM inventory - WHERE film_id = p_film_id - AND store_id = p_store_id - AND NOT inventory_in_stock(inventory_id); - - SELECT COUNT(*) - FROM inventory - WHERE film_id = p_film_id - AND store_id = p_store_id - AND NOT inventory_in_stock(inventory_id) - INTO p_film_count; -END $$ - -DELIMITER ; - -DELIMITER $$ - -CREATE FUNCTION inventory_held_by_customer(p_inventory_id INT) RETURNS INT -READS SQL DATA -BEGIN - DECLARE v_customer_id INT; - DECLARE EXIT HANDLER FOR NOT FOUND RETURN NULL; - - SELECT customer_id INTO v_customer_id - FROM rental - WHERE return_date IS NULL - AND inventory_id = p_inventory_id; - - RETURN v_customer_id; -END $$ - -DELIMITER ; - -DELIMITER $$ - -CREATE FUNCTION inventory_in_stock(p_inventory_id INT) RETURNS BOOLEAN -READS SQL DATA -BEGIN - DECLARE v_rentals INT; - DECLARE v_out INT; - - #AN ITEM IS IN-STOCK IF THERE ARE EITHER NO ROWS IN THE rental TABLE - #FOR THE ITEM OR ALL ROWS HAVE return_date POPULATED - - SELECT COUNT(*) INTO v_rentals - FROM rental - WHERE inventory_id = p_inventory_id; - - IF v_rentals = 0 THEN - RETURN TRUE; - END IF; - - SELECT COUNT(rental_id) INTO v_out - FROM inventory LEFT JOIN rental USING(inventory_id) - WHERE inventory.inventory_id = p_inventory_id - AND rental.return_date IS NULL; - - IF v_out > 0 THEN - RETURN FALSE; - ELSE - RETURN TRUE; - END IF; -END $$ - -DELIMITER ; - -SET SQL_MODE=@OLD_SQL_MODE; -SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; -SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; - -SET NAMES utf8mb4; -SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; -SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; -SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL'; -SET @old_autocommit=@@autocommit; - -INSERT INTO actor VALUES (1,'PENELOPE','GUINESS','2006-02-15 04:34:33'), -(2,'NICK','WAHLBERG','2006-02-15 04:34:33'), -(3,'ED','CHASE','2006-02-15 04:34:33'); - -INSERT INTO `address` VALUES (1,'47 MySakila Drive',NULL,'Alberta',1,'','',/*!50705 0x0000000001010000003E0A325D63345CC0761FDB8D99D94840,*/'2014-09-25 22:30:27'), -(2,'28 MySQL Boulevard',NULL,'QLD',2,'','',/*!50705 0x0000000001010000008E10D4DF812463404EE08C5022A23BC0,*/'2014-09-25 22:30:09'), -(3,'23 Workhaven Lane',NULL,'Alberta',3,'','14033335568',/*!50705 0x000000000101000000CDC4196863345CC01DEE7E7099D94840,*/'2014-09-25 22:30:27'); - -INSERT INTO category VALUES (1,'Action','2006-02-15 04:46:27'), -(2,'Animation','2006-02-15 04:46:27'), -(3,'Children','2006-02-15 04:46:27'); - -INSERT INTO city VALUES (1,'A Corua (La Corua)',1,'2006-02-15 04:45:25'), -(2,'Abha',2,'2006-02-15 04:45:25'), -(3,'Abu Dhabi',3,'2006-02-15 04:45:25'); - -INSERT INTO country VALUES (1,'Afghanistan','2006-02-15 04:44:00'), -(2,'Algeria','2006-02-15 04:44:00'), -(3,'American Samoa','2006-02-15 04:44:00'); - -INSERT INTO customer VALUES (1,1,'MARY','SMITH','MARY.SMITH@sakilacustomer.org',1,1,'2006-02-14 22:04:36','2006-02-15 04:57:20'), -(2,2,'PATRICIA','JOHNSON','PATRICIA.JOHNSON@sakilacustomer.org',2,2,'2006-02-14 22:04:36','2006-02-15 04:57:20'), -(3,3,'LINDA','WILLIAMS','LINDA.WILLIAMS@sakilacustomer.org',3,3,'2006-02-14 22:04:36','2006-02-15 04:57:20'); - -INSERT INTO film VALUES (1,'ACADEMY DINOSAUR','A Epic Drama of a Feminist And a Mad Scientist who must Battle a Teacher in The Canadian Rockies',2006,1,NULL,1,'0.99',1,'20.99','PG','Deleted Scenes,Behind the Scenes','2006-02-15 05:03:42'), -(2,'ACE GOLDFINGER','A Astounding Epistle of a Database Administrator And a Explorer who must Find a Car in Ancient China',2006,2,NULL,2,'4.99',2,'12.99','G','Trailers,Deleted Scenes','2006-02-15 05:03:42'), -(3,'ADAPTATION HOLES','A Astounding Reflection of a Lumberjack And a Car who must Sink a Lumberjack in A Baloon Factory',2006,3,NULL,3,'2.99',3,'18.99','NC-17','Trailers,Deleted Scenes','2006-02-15 05:03:42'); - -INSERT INTO film_actor VALUES (1,1,'2006-02-15 05:05:03'), -(2,2,'2006-02-15 05:05:03'), -(3,3,'2006-02-15 05:05:03'); - -INSERT INTO film_category VALUES (1,1,'2006-02-15 05:07:09'), -(2,2,'2006-02-15 05:07:09'), -(3,3,'2006-02-15 05:07:09'); - -INSERT INTO inventory VALUES (1,1,1,'2006-02-15 05:09:17'), -(2,2,2,'2006-02-15 05:09:17'), -(3,3,3,'2006-02-15 05:09:17'); - -INSERT INTO language VALUES (1,'English','2006-02-15 05:02:19'), -(2,'Italian','2006-02-15 05:02:19'), -(3,'Japanese','2006-02-15 05:02:19'); - -INSERT INTO payment VALUES (1,1,1,1,'2.99','2005-05-25 11:30:37','2006-02-15 22:12:30'), -(2,2,2,2,'0.99','2005-05-28 10:35:23','2006-02-15 22:12:30'), -(3,3,3,3,'5.99','2005-06-15 00:54:12','2006-02-15 22:12:30'); - -INSERT INTO rental VALUES (1,'2005-05-24 22:53:30',1,1,'2005-05-26 22:04:30',1,'2006-02-15 21:30:53'), -(2,'2005-05-24 22:54:33',2,2,'2005-05-28 19:40:33',2,'2006-02-15 21:30:53'), -(3,'2005-05-24 23:03:39',3,3,'2005-06-01 22:12:39',3,'2006-02-15 21:30:53'); - -SET SQL_MODE=@OLD_SQL_MODE; -SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; -SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; -SET autocommit=@old_autocommit; diff --git a/tests/e2e/mysql2mysql/snapshot_without_pk/check_db_test.go b/tests/e2e/mysql2mysql/snapshot_without_pk/check_db_test.go deleted file mode 100644 index d28edf17..00000000 --- a/tests/e2e/mysql2mysql/snapshot_without_pk/check_db_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package light - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - require.NoError(t, tasks.ActivateDelivery(context.TODO(), nil, client2.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - - // Storages without primary keys cannot be compared at the moment -} diff --git a/tests/e2e/mysql2mysql/snapshot_without_pk/dump/dump.sql b/tests/e2e/mysql2mysql/snapshot_without_pk/dump/dump.sql deleted file mode 100644 index d03b708c..00000000 --- a/tests/e2e/mysql2mysql/snapshot_without_pk/dump/dump.sql +++ /dev/null @@ -1,15 +0,0 @@ -SET NAMES utf8mb4; -SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; -SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; -SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL'; - -CREATE TABLE actor ( - actor_id SMALLINT UNSIGNED NOT NULL, - first_name VARCHAR(45) NOT NULL, - last_name VARCHAR(45) NOT NULL, - last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -INSERT INTO actor VALUES (1,'PENELOPE','GUINESS','2006-02-15 04:34:33'), -(2,'NICK','WAHLBERG','2006-02-15 04:34:33'), -(3,'ED','CHASE','2006-02-15 04:34:33'); diff --git a/tests/e2e/mysql2mysql/tx_boundaries/check_db_test.go b/tests/e2e/mysql2mysql/tx_boundaries/check_db_test.go deleted file mode 100644 index 04dd1dda..00000000 --- a/tests/e2e/mysql2mysql/tx_boundaries/check_db_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - Source.BufferLimit = 100 * 1024 // 100kb to init flush between TX - Target.PerTransactionPush = true - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - - targetCfg := mysql_client.NewConfig() - targetCfg.Addr = fmt.Sprintf("%v:%v", Target.Host, Target.Port) - targetCfg.User = Target.User - targetCfg.Passwd = string(Target.Password) - targetCfg.DBName = Target.Database - targetCfg.Net = "tcp" - - targetMysqlConnector, err := mysql_client.NewConnector(targetCfg) - require.NoError(t, err) - targetDB := sql.OpenDB(targetMysqlConnector) - - tracker, err := mysql.NewTableProgressTracker(targetDB, Target.Database) - require.NoError(t, err) - state, err := tracker.GetCurrentState() - require.NoError(t, err) - logger.Log.Info("replication progress", log.Any("progress", state)) - require.Equal(t, 1, len(state)) - require.Equal(t, mysql.SyncWait, state[`"target"."products"`].Status) - require.True(t, state[`"target"."products"`].LSN > 0) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := &server.Transfer{ - ID: "test-id", - Src: &Source, - Dst: &Target, - } - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - srcCfg := mysql_client.NewConfig() - srcCfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - srcCfg.User = Source.User - srcCfg.Passwd = string(Source.Password) - srcCfg.DBName = Source.Database - srcCfg.Net = "tcp" - - srcMysqlConnector, err := mysql_client.NewConnector(srcCfg) - require.NoError(t, err) - srcDB := sql.OpenDB(srcMysqlConnector) - - srcConn, err := srcDB.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - "delete from products where id > 10", - } - - for _, request := range requests { - rows, err := srcConn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = srcConn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "products", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - time.Minute)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - - targetCfg := mysql_client.NewConfig() - targetCfg.Addr = fmt.Sprintf("%v:%v", Target.Host, Target.Port) - targetCfg.User = Target.User - targetCfg.Passwd = string(Target.Password) - targetCfg.DBName = Target.Database - targetCfg.Net = "tcp" - - targetMysqlConnector, err := mysql_client.NewConnector(targetCfg) - require.NoError(t, err) - targetDB := sql.OpenDB(targetMysqlConnector) - - tracker, err := mysql.NewTableProgressTracker(targetDB, Target.Database) - require.NoError(t, err) - state, err := tracker.GetCurrentState() - require.NoError(t, err) - logger.Log.Info("replication progress", log.Any("progress", state)) - require.Equal(t, 1, len(state)) - require.Equal(t, mysql.InSync, state[`"target"."products"`].Status) - require.True(t, state[`"target"."products"`].LSN > 0) -} diff --git a/tests/e2e/mysql2mysql/tx_boundaries/dump/update.sql b/tests/e2e/mysql2mysql/tx_boundaries/dump/update.sql deleted file mode 100644 index f8e7aa06..00000000 --- a/tests/e2e/mysql2mysql/tx_boundaries/dump/update.sql +++ /dev/null @@ -1,34 +0,0 @@ -create table products -( - id int auto_increment - primary key, - title varchar(256) not null -); - -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) values (LEFT(MD5(RAND()), 250)); -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 10; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 20; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 40; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 20; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 100; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 200; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 400; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 800; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 400; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 2000; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 2000; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 2000; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 2000; -- 10k rows here -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 10000; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 10000; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 10000; -insert into products (title) select LEFT(MD5(RAND()), 250) from products limit 10000; -- 50k rows here \ No newline at end of file diff --git a/tests/e2e/mysql2mysql/update/check_db_test.go b/tests/e2e/mysql2mysql/update/check_db_test.go deleted file mode 100644 index c6524274..00000000 --- a/tests/e2e/mysql2mysql/update/check_db_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - "update customers set status = 'active,waiting' where customerNumber in (131, 141);", - "update customers set status = '' where customerNumber in (103, 141);", - "update customers set contactLastName = '', contactFirstName = NULL where customerNumber in (129, 131, 141);", - "update customers set contactLastName = 'Lollers', contactFirstName = 'Kekus' where customerNumber in (103, 112, 114, 119);", - "update customers set customerName = 'Kabanchik INC', city = 'Los Hogas' where customerNumber in (121, 124, 125, 128);", - "delete from customers where customerNumber = 114", - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/update/dump/update.sql b/tests/e2e/mysql2mysql/update/dump/update.sql deleted file mode 100644 index e92c1b5c..00000000 --- a/tests/e2e/mysql2mysql/update/dump/update.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE `customers` ( - `customerNumber` int(11) NOT NULL, - `customerName` varchar(50) NOT NULL, - `contactLastName` varchar(50), - `contactFirstName` varchar(50), - `phone` varchar(50) NOT NULL, - `addressLine1` varchar(50) NOT NULL, - `addressLine2` varchar(50) DEFAULT NULL, - `city` varchar(50) NOT NULL, - `state` varchar(50) DEFAULT NULL, - `postalCode` varchar(15) DEFAULT NULL, - `country` varchar(50) NOT NULL, - `creditLimit` decimal(10,2) DEFAULT NULL, - `status` set('active', 'waiting', 'suspend', 'canceled') NOT NULL, - PRIMARY KEY (`customerNumber`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -insert into `customers`(`customerNumber`,`customerName`,`contactLastName`,`contactFirstName`,`phone`,`addressLine1`,`addressLine2`,`city`,`state`,`postalCode`,`country`,`creditLimit`, `status`) values -(103,'Atelier graphique','Schmitt','Carine ','40.32.2555','54, rue Royale',NULL,'Nantes',NULL,'44000','France','21000.00', 'active,waiting'), -(112,'Signal Gift Stores','King','Jean','7025551838','8489 Strong St.',NULL,'Las Vegas','NV','83030','USA','71800.00', 'active,suspend'), -(114,'Australian Collectors, Co.','Ferguson','Peter','03 9520 4555','636 St Kilda Road','Level 3','Melbourne','Victoria','3004','Australia','117300.00', 'active,waiting'), -(119,'La Rochelle Gifts','Labrune','Janine ','40.67.8555','67, rue des Cinquante Otages',NULL,'Nantes',NULL,'44000','France','118200.00', 'active,suspend'), -(121,'Baane Mini Imports','Bergulfsen','Jonas ','07-98 9555','Erling Skakkes gate 78',NULL,'Stavern',NULL,'4110','Norway','81700.00', ''), -(124,'Mini Gifts Distributors Ltd.','Nelson','Susan','4155551450','5677 Strong St.',NULL,'San Rafael','CA','97562','USA','210500.00', ''), -(125,'Havel & Zbyszek Co','Piestrzeniewicz','Zbyszek ','(26) 642-7555','ul. Filtrowa 68',NULL,'Warszawa',NULL,'01-012','Poland','0.00', ''), -(128,'Blauer See Auto, Co.','Keitel','Roland','+49 69 66 90 2555','Lyonerstr. 34',NULL,'Frankfurt',NULL,'60528','Germany','59700.00', 'canceled'), -(129,'Mini Wheels Co.','Murphy','Julie','6505555787','5557 North Pendale Street',NULL,'San Francisco','CA','94217','USA','64600.00', 'canceled'), -(131,'Land of Toys Inc.','Lee','Kwai','2125557818','897 Long Airport Avenue',NULL,'NYC','NY','10022','USA','114900.00', 'canceled'), -(141,'Euro+ Shopping Channel','Freyre','Diego ','(91) 555 94 44','C/ Moralzarzal, 86',NULL,'Madrid',NULL,'28034','Spain','227600.00', 'canceled'); diff --git a/tests/e2e/mysql2mysql/update_cp1251/check_db_test.go b/tests/e2e/mysql2mysql/update_cp1251/check_db_test.go deleted file mode 100644 index a1f50eec..00000000 --- a/tests/e2e/mysql2mysql/update_cp1251/check_db_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - "update customers set status = 'active,waiting' where customerNumber in (131, 141);", - "update customers set status = '' where customerNumber in (103, 141);", - "update customers set contactLastName = '', contactFirstName = NULL where customerNumber in (129, 131, 141);", - "update customers set contactLastName = 'Быстрая коричневая лиса', contactFirstName = 'перепрыгивает ленивую собаку' where customerNumber in (103, 112, 114, 119);", - "update customers set customerName = 'Съешь ещё этих мягких французских булок', city = 'да выпей чаю' where customerNumber in (121, 124, 125, 128);", - "delete from customers where customerNumber = 114", - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/update_cp1251/dump/update.sql b/tests/e2e/mysql2mysql/update_cp1251/dump/update.sql deleted file mode 100644 index 3882e369..00000000 --- a/tests/e2e/mysql2mysql/update_cp1251/dump/update.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE `customers` ( - `customerNumber` int(11) NOT NULL, - `customerName` varchar(50) NOT NULL, - `contactLastName` varchar(50), - `contactFirstName` varchar(50), - `phone` varchar(50) NOT NULL, - `addressLine1` varchar(50) NOT NULL, - `addressLine2` varchar(50) DEFAULT NULL, - `city` varchar(50) NOT NULL, - `state` varchar(50) DEFAULT NULL, - `postalCode` varchar(15) DEFAULT NULL, - `country` varchar(50) NOT NULL, - `creditLimit` decimal(10,2) DEFAULT NULL, - `status` set('active', 'waiting', 'suspend', 'canceled') NOT NULL, - PRIMARY KEY (`customerNumber`) -) ENGINE=InnoDB DEFAULT CHARSET=cp1251; - -insert into `customers`(`customerNumber`,`customerName`,`contactLastName`,`contactFirstName`,`phone`,`addressLine1`,`addressLine2`,`city`,`state`,`postalCode`,`country`,`creditLimit`, `status`) values -(103,'Съешь','Schmitt','Carine ','40.32.2555','54, rue Royale',NULL,'Nantes',NULL,'44000','France','21000.00', 'active,waiting'), -(112,'ещё','King','Jean','7025551838','8489 Strong St.',NULL,'Las Vegas','NV','83030','USA','71800.00', 'active,suspend'), -(114,'этих','Ferguson','Peter','03 9520 4555','636 St Kilda Road','Level 3','Melbourne','Victoria','3004','Australia','117300.00', 'active,waiting'), -(119,'мягких','Labrune','Janine ','40.67.8555','67, rue des Cinquante Otages',NULL,'Nantes',NULL,'44000','France','118200.00', 'active,suspend'), -(121,'французских','Bergulfsen','Jonas ','07-98 9555','Erling Skakkes gate 78',NULL,'Stavern',NULL,'4110','Norway','81700.00', ''), -(124,'булок','Nelson','Susan','4155551450','5677 Strong St.',NULL,'San Rafael','CA','97562','USA','210500.00', ''), -(125,'да','Piestrzeniewicz','Zbyszek ','(26) 642-7555','ul. Filtrowa 68',NULL,'Warszawa',NULL,'01-012','Poland','0.00', ''), -(128,'выпей','Keitel','Roland','+49 69 66 90 2555','Lyonerstr. 34',NULL,'Frankfurt',NULL,'60528','Germany','59700.00', 'canceled'), -(129,'чаю','Murphy','Julie','6505555787','5557 North Pendale Street',NULL,'San Francisco','CA','94217','USA','64600.00', 'canceled'), -(131,'Быстрая коричневая лиса','Lee','Kwai','2125557818','897 Long Airport Avenue',NULL,'NYC','NY','10022','USA','114900.00', 'canceled'), -(141,'перепрыгивает ленивую собаку','Freyre','Diego ','(91) 555 94 44','C/ Moralzarzal, 86',NULL,'Madrid',NULL,'28034','Spain','227600.00', 'canceled'); diff --git a/tests/e2e/mysql2mysql/update_minimal/check_db_test.go b/tests/e2e/mysql2mysql/update_minimal/check_db_test.go deleted file mode 100644 index 5349a29f..00000000 --- a/tests/e2e/mysql2mysql/update_minimal/check_db_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := &server.Transfer{ - ID: "test-id", - Src: &Source, - Dst: &Target, - } - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - "update customers set status = 'active,waiting' where customerNumber in (131, 141);", - "update customers set status = '' where customerNumber in (103, 141);", - "update customers set contactLastName = '', contactFirstName = NULL where customerNumber in (129, 131, 141);", - "update customers set contactLastName = 'Lollers', contactFirstName = 'Kekus' where customerNumber in (103, 112, 114, 119);", - "update customers set customerName = 'Kabanchik INC', city = 'Los Hogas' where customerNumber in (121, 124, 125, 128);", - "delete from customers where customerNumber = 114", - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/update_minimal/dump/update_minimal.sql b/tests/e2e/mysql2mysql/update_minimal/dump/update_minimal.sql deleted file mode 100644 index bf799838..00000000 --- a/tests/e2e/mysql2mysql/update_minimal/dump/update_minimal.sql +++ /dev/null @@ -1,31 +0,0 @@ -set @@GLOBAL.binlog_row_image = 'minimal'; - -CREATE TABLE `customers` ( - `customerNumber` int(11) NOT NULL, - `customerName` varchar(50) NOT NULL, - `contactLastName` varchar(50), - `contactFirstName` varchar(50), - `phone` varchar(50) NOT NULL, - `addressLine1` varchar(50) NOT NULL, - `addressLine2` varchar(50) DEFAULT NULL, - `city` varchar(50) NOT NULL, - `state` varchar(50) DEFAULT NULL, - `postalCode` varchar(15) DEFAULT NULL, - `country` varchar(50) NOT NULL, - `creditLimit` decimal(10,2) DEFAULT NULL, - `status` set('active', 'waiting', 'suspend', 'canceled') NOT NULL, - PRIMARY KEY (`customerNumber`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -insert into `customers`(`customerNumber`,`customerName`,`contactLastName`,`contactFirstName`,`phone`,`addressLine1`,`addressLine2`,`city`,`state`,`postalCode`,`country`,`creditLimit`, `status`) values -(103,'Atelier graphique','Schmitt','Carine ','40.32.2555','54, rue Royale',NULL,'Nantes',NULL,'44000','France','21000.00', 'active,waiting'), -(112,'Signal Gift Stores','King','Jean','7025551838','8489 Strong St.',NULL,'Las Vegas','NV','83030','USA','71800.00', 'active,suspend'), -(114,'Australian Collectors, Co.','Ferguson','Peter','03 9520 4555','636 St Kilda Road','Level 3','Melbourne','Victoria','3004','Australia','117300.00', 'active,waiting'), -(119,'La Rochelle Gifts','Labrune','Janine ','40.67.8555','67, rue des Cinquante Otages',NULL,'Nantes',NULL,'44000','France','118200.00', 'active,suspend'), -(121,'Baane Mini Imports','Bergulfsen','Jonas ','07-98 9555','Erling Skakkes gate 78',NULL,'Stavern',NULL,'4110','Norway','81700.00', ''), -(124,'Mini Gifts Distributors Ltd.','Nelson','Susan','4155551450','5677 Strong St.',NULL,'San Rafael','CA','97562','USA','210500.00', ''), -(125,'Havel & Zbyszek Co','Piestrzeniewicz','Zbyszek ','(26) 642-7555','ul. Filtrowa 68',NULL,'Warszawa',NULL,'01-012','Poland','0.00', ''), -(128,'Blauer See Auto, Co.','Keitel','Roland','+49 69 66 90 2555','Lyonerstr. 34',NULL,'Frankfurt',NULL,'60528','Germany','59700.00', 'canceled'), -(129,'Mini Wheels Co.','Murphy','Julie','6505555787','5557 North Pendale Street',NULL,'San Francisco','CA','94217','USA','64600.00', 'canceled'), -(131,'Land of Toys Inc.','Lee','Kwai','2125557818','897 Long Airport Avenue',NULL,'NYC','NY','10022','USA','114900.00', 'canceled'), -(141,'Euro+ Shopping Channel','Freyre','Diego ','(91) 555 94 44','C/ Moralzarzal, 86',NULL,'Madrid',NULL,'28034','Spain','227600.00', 'canceled'); diff --git a/tests/e2e/mysql2mysql/update_unicode/check_db_test.go b/tests/e2e/mysql2mysql/update_unicode/check_db_test.go deleted file mode 100644 index 25a87047..00000000 --- a/tests/e2e/mysql2mysql/update_unicode/check_db_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/mysql" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *helpers.RecipeMysqlSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - - dpconfig.InternalCloud = &dpconfig.InternalCloudConfig{} -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: Target.Port}, - )) - }() - - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := mysql.NewStorage(Source.ToStorageParams()) - require.NoError(t, err) - _, err = mysql.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - sourceAsDestination := mysql.MysqlDestination{ - Host: Source.Host, - User: Source.User, - Password: Source.Password, - Database: Source.Database, - Port: Source.Port, - } - sourceAsDestination.WithDefaults() - _, err := mysql.NewSinker(logger.Log, &sourceAsDestination, helpers.EmptyRegistry()) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.NoTracking = false - Source.ServerID = 7 // it's some hash from connector.ID - err = mysql.SyncBinlogPosition(&Source, "", coordinator.NewFakeClient()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", Source.Host, Source.Port) - cfg.User = Source.User - cfg.Passwd = string(Source.Password) - cfg.DBName = Source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - requests := []string{ - "update customers set status = 'active,waiting' where customerNumber in (131, 141);", - "update customers set status = '' where customerNumber in (103, 141);", - "update customers set contactLastName = '', contactFirstName = NULL where customerNumber in (129, 131, 141);", - "update customers set contactLastName = 'Lollers 😂 🍆 ☎ Ы', contactFirstName = 'Kekus' where customerNumber in (103, 112, 114, 119);", - "update customers set customerName = 'Kabanchik INC 😂 🍆 ☎ Ы', city = 'Los Hogas' where customerNumber in (121, 124, 125, 128);", - "delete from customers where customerNumber = 114", - } - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCountDifferentSchemas(t, - Source.Database, Target.Database, "customers", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/update_unicode/dump/update.sql b/tests/e2e/mysql2mysql/update_unicode/dump/update.sql deleted file mode 100644 index 670e7f1e..00000000 --- a/tests/e2e/mysql2mysql/update_unicode/dump/update.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE `customers` ( - `customerNumber` int(11) NOT NULL, - `customerName` varchar(50) NOT NULL, - `contactLastName` varchar(50), - `contactFirstName` varchar(50), - `phone` varchar(50) NOT NULL, - `addressLine1` varchar(50) NOT NULL, - `addressLine2` varchar(50) DEFAULT NULL, - `city` varchar(50) NOT NULL, - `state` varchar(50) DEFAULT NULL, - `postalCode` varchar(15) DEFAULT NULL, - `country` varchar(50) NOT NULL, - `creditLimit` decimal(10,2) DEFAULT NULL, - `status` set('active', 'waiting', 'suspend', 'canceled') NOT NULL, - PRIMARY KEY (`customerNumber`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -insert into `customers`(`customerNumber`,`customerName`,`contactLastName`,`contactFirstName`,`phone`,`addressLine1`,`addressLine2`,`city`,`state`,`postalCode`,`country`,`creditLimit`, `status`) values -(103,'😂 🍆 ☎ Ы Atelier graphique','Schmitt','Carine ','40.32.2555','54, rue Royale',NULL,'Nantes',NULL,'44000','France','21000.00', 'active,waiting'), -(112,'😂 🍆 ☎ Ы Signal Gift Stores','King','Jean','7025551838','8489 Strong St.',NULL,'Las Vegas','NV','83030','USA','71800.00', 'active,suspend'), -(114,'😂 🍆 ☎ Ы Australian Collectors, Co.','Ferguson','Peter','03 9520 4555','636 St Kilda Road','Level 3','Melbourne','Victoria','3004','Australia','117300.00', 'active,waiting'), -(119,'😂 🍆 ☎ Ы La Rochelle Gifts','Labrune','Janine ','40.67.8555','67, rue des Cinquante Otages',NULL,'Nantes',NULL,'44000','France','118200.00', 'active,suspend'), -(121,'😂 🍆 ☎ Ы Baane Mini Imports','Bergulfsen','Jonas ','07-98 9555','Erling Skakkes gate 78',NULL,'Stavern',NULL,'4110','Norway','81700.00', ''), -(124,'😂 🍆 ☎ Ы Mini Gifts Distributors Ltd.','Nelson','Susan','4155551450','5677 Strong St.',NULL,'San Rafael','CA','97562','USA','210500.00', ''), -(125,'😂 🍆 ☎ Ы Havel & Zbyszek Co','Piestrzeniewicz','Zbyszek ','(26) 642-7555','ul. Filtrowa 68',NULL,'Warszawa',NULL,'01-012','Poland','0.00', ''), -(128,'😂 🍆 ☎ Ы Blauer See Auto, Co.','Keitel','Roland','+49 69 66 90 2555','Lyonerstr. 34',NULL,'Frankfurt',NULL,'60528','Germany','59700.00', 'canceled'), -(129,'😂 🍆 ☎ Ы Mini Wheels Co.','Murphy','Julie','6505555787','5557 North Pendale Street',NULL,'San Francisco','CA','94217','USA','64600.00', 'canceled'), -(131,'😂 🍆 ☎ Ы Land of Toys Inc.','Lee','Kwai','2125557818','897 Long Airport Avenue',NULL,'NYC','NY','10022','USA','114900.00', 'canceled'), -(141,'😂 🍆 ☎ Ы Euro+ Shopping Channel','Freyre','Diego ','(91) 555 94 44','C/ Moralzarzal, 86',NULL,'Madrid',NULL,'28034','Spain','227600.00', 'canceled'); diff --git a/tests/e2e/mysql2mysql/view/check_db_test.go b/tests/e2e/mysql2mysql/view/check_db_test.go deleted file mode 100644 index 3b2f8b7b..00000000 --- a/tests/e2e/mysql2mysql/view/check_db_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package light - -import ( - "context" - "database/sql" - "fmt" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/tests/helpers" - mysql_client "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" -) - -func TestSnapshotAndReplicationViewsCompatibility(t *testing.T) { - source := *helpers.RecipeMysqlSource() - source.PreSteps.View = true - target := *helpers.RecipeMysqlTarget() - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mysql source", Port: source.Port}, - helpers.LabeledPort{Label: "Mysql target", Port: target.Port}, - )) - transfer := helpers.MakeTransfer("fake", &source, &target, abstract.TransferTypeSnapshotAndIncrement) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - require.NoError(t, helpers.CompareStorages(t, source, target, helpers.NewCompareStorageParams())) - - requests := []string{ - "update test set name = 'Test Name' where id = 1;", - "insert into test2(name, email, age) values ('name2', 'email2', 44);", - } - - cfg := mysql_client.NewConfig() - cfg.Addr = fmt.Sprintf("%v:%v", source.Host, source.Port) - cfg.User = source.User - cfg.Passwd = string(source.Password) - cfg.DBName = source.Database - cfg.Net = "tcp" - - mysqlConnector, err := mysql_client.NewConnector(cfg) - require.NoError(t, err) - db := sql.OpenDB(mysqlConnector) - - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - - for _, request := range requests { - rows, err := conn.QueryContext(context.Background(), request) - require.NoError(t, err) - require.NoError(t, rows.Close()) - } - - err = conn.Close() - require.NoError(t, err) - require.NoError(t, helpers.CompareStorages(t, source, target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mysql2mysql/view/dump/update.sql b/tests/e2e/mysql2mysql/view/dump/update.sql deleted file mode 100644 index e9d29584..00000000 --- a/tests/e2e/mysql2mysql/view/dump/update.sql +++ /dev/null @@ -1,30 +0,0 @@ -CREATE TABLE test ( - id INT PRIMARY KEY AUTO_INCREMENT, - name VARCHAR(50), - email VARCHAR(100), - age INT -); - -CREATE TABLE test2 ( - id INT PRIMARY KEY AUTO_INCREMENT, - name VARCHAR(50), - email VARCHAR(100), - age INT -); - -INSERT INTO test(name, email, age) VALUES ('Franklin', 'mailadam', 71); -INSERT INTO test(name, email, age) VALUES ('not Franklin', 'test', 20); -INSERT INTO test2(name, email, age) VALUES ('Adam', 'mail', 21); -INSERT INTO test2(name, email, age) VALUES ('Not Adam', 'test2', 37); - -CREATE VIEW test_view (v_name, v_age, v_email) AS SELECT test.name, test.age, test.email FROM test; -CREATE VIEW test_view2 (v_name1, v_age1, v_email2) AS SELECT test.name, test.age, test2.email FROM test, test2; - - --- We get views by alphabetical order in GetViewDDLs(...) transfer_manager/go/pkg/providers/mysql/schema_copy.go --- So for such queries: -CREATE VIEW b AS SELECT * FROM test; -- DO NOT RENAME VIEW without reading comments -CREATE VIEW a AS SELECT * FROM b; -- DO NOT RENAME VIEW without reading comments - --- DDLs will be in other order: firstly code will try to CREATE VIEW a ... FROM b. --- So by those queries we check logic of applyDDLs's dependence handling. \ No newline at end of file diff --git a/tests/e2e/pg2ch/alters/alters_test.go b/tests/e2e/pg2ch/alters/alters_test.go deleted file mode 100644 index 57c115db..00000000 --- a/tests/e2e/pg2ch/alters/alters_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package alters - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/jackc/pgx/v4" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithPrefix("")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestAlter(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &Source) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - Target.ProtocolUnspecified = true - Target.MigrationOptions = &model.ChSinkMigrationOptions{ - AddNewColumns: true, - } - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - var terminateErr error - localWorker := helpers.Activate(t, transfer, func(err error) { - terminateErr = err - }) - defer localWorker.Close(t) - - t.Run("ADD COLUMN", func(t *testing.T) { - rows, err := conn.Query(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (6, 6, 'c')") - require.NoError(t, err) - rows.Close() - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - - rows, err = conn.Query(context.Background(), "ALTER TABLE __test ADD COLUMN new_val INTEGER") - require.NoError(t, err) - rows.Close() - - time.Sleep(10 * time.Second) - - rows, err = conn.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val) VALUES (7, 7, 'd', 7)") - require.NoError(t, err) - rows.Close() - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) - }) - - t.Run("ADD COLUMN single transaction", func(t *testing.T) { - // force INSERTs with different schemas to be pushed with one ApplyChangeItems call - err := conn.BeginFunc(context.Background(), func(tx pgx.Tx) error { - rows, err := tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (8, 8, 'e')") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "ALTER TABLE __test ADD COLUMN new_val2 INTEGER") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val2) VALUES (9, 9, 'f', 9)") - require.NoError(t, err) - rows.Close() - return nil - }) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) - }) - - // Add temporary column, shall terminate replication - t.Run("ADD TEMPORARY COLUMN", func(t *testing.T) { - // add column, with one new change - require.NoError(t, conn.BeginFunc(context.Background(), func(tx pgx.Tx) error { - rows, err := tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val2) VALUES (10, 10, 'f', 10)") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "ALTER TABLE __test ADD COLUMN new_val3 INTEGER") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val2, new_val3) VALUES (11, 11, 'f', 11, 11)") - require.NoError(t, err) - rows.Close() - - return nil - })) - - // delete new column, with one new change without this column - require.NoError(t, conn.BeginFunc(context.Background(), func(tx pgx.Tx) error { - rows, err := tx.Query(context.Background(), "ALTER TABLE __test DROP COLUMN new_val3") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val2) VALUES (12, 12, 'f', 12)") - require.NoError(t, err) - rows.Close() - - return nil - })) - - //------------------------------------------------------------------------------------ - // wait termination - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) - }) - - require.NoError(t, terminateErr) -} diff --git a/tests/e2e/pg2ch/alters/dump/ch/dump.sql b/tests/e2e/pg2ch/alters/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/alters/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/alters/dump/pg/dump.sql b/tests/e2e/pg2ch/alters/dump/pg/dump.sql deleted file mode 100644 index f4c3e888..00000000 --- a/tests/e2e/pg2ch/alters/dump/pg/dump.sql +++ /dev/null @@ -1,13 +0,0 @@ --- needs to be sure there is db1 -create table __test -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test (id, val1, val2) -values (1, 1, 'a'), - (2, 2, 'XcTIan6Sk2JTT98F41uOn9BVdIapLVCu1fOfbVu8GC0q6q8dGQoF7BQU4GiTlj5DgXnp0E9mJX5SwD2BCNWri6jvODz8Gp4AMgEUZxLOjjFmt1VkgPrU67YIrmNCwre1b0SNJ90mvU5yFOoF3FWB3U2uT04wonF4wuwSWrWY9SExpormD7KOuLLYAjaGTd0bWH6ttDoVQLRkFofUYMz5cLJcSntWdMAU872qudaMG624AwCec5sOLm9b6QhHY3eusgV9pGHbXm7XmI6RF7lqSVDzxGzvyahYNMvkc6Cf6ccFK3fFUFO3WZkY5fT1ad3QTIqsP8WmyZEzol4GAiuzZAHvB2szeq1keaSzEeSoI6YPJXFevyRFzlVGJN7OxErxHnYd8TPPOyhQI0PwpQ7MY1cX9cWiqrxTl8lcDp23kntMsbmouacyEsHeFkagozm8muqnEM4w3qQhXNIOkV8pkoD0s2rxo5tytlBbW0OpgnKp6UxLAp7QqfmWXcOLIePdL3bOVI2WJfBXrgsnfVlnNukoH22rn4Vb3pvcsIyT4x8loFZzeVmXfR4xLeT73Vs5KDYYOGZOWdzh5KVWdvGTcpVU2fSNYl1GeDps45o7mTj2ycllkewLbGD84QNVP67aDujad7gLmt8jYrzwxS04AX7k2tz7tBE4gEqOefBwXyCBy1t9j7vSA9tg8ZupGMsy0QNzw1vRCo5jmNt3f4AjwWqBGYIIjYaS27vZwKOGdTTEqpbebWW45sBkxe9DrvrDYUi11wLMtr1sxKNzvZgfS65ROvjdXYJfkVXWtiqo8jpwf1KNdvTDJscQUFgh9e9XfCMAZTUOoBtQmQhDVQe4CON8JGVm4pDnKf7acwhAzxZU8X7HZblEQeYCKIA07MalK4f0XBzEL5rHmhLOry1a6uPFmaqx2DAHPegthCqcvgeNCXA48nrXXwgG04TLvNU4Xk3Lwwhug24btNMauk5w0cYPMl0DZ3CmnMleYe2u0pndVLsOY1PlKOLs8nrZEp6VKXrb3ZdkcZZ6c9h88dXIAkrrGoHh5cB2RtCTyZyBS0Y8akHDODUVh7LIYkd9vjZ4W9sPqxxnbGQfYIMWCm7zGLbhhOrf8GBN1dBdQvEZYWOsqrvGd2z1C8WiGXvrTjdUXnudsT1XYCniHyqpAVPLyQGZ3CSWaswmOi1bjeDOSN2t3fH50pyznZPmFbJfL8R1QFV3mCPCxkKc4o3eI24hOkX4MPepi6HlBadwgFbY69KDjKs9fphhUA2SYxvHWr3igc5Wp9ZmyBW88c1BxykzK8xbJseGrdavV4uSl96L0GnSpRhbJuKfX1QUDU42yImShSgdyXVci4O3lXVrJYqFHFrTd2jl2spp3V2VJqu3noUxrFZVmBCPOvg3Mqx0uAefGXtBI3T9vNJSrgFVNO4xFOa03oOlG1bRvT1I4bk7sBBAiVyQ0c445CxVPhhUuExt44BocoXFUDYh6EZGEw0OU56znN7wWqUaegqZpOMtRYZk5MpSIFauHyDXIVv17A6OHTN1zsW5hHIiWdQ8g5T362HvHiMLH3IhK1yL4jf29V5GqkKMkMb7kKPWTEn6ICkJQ4CBZSSKbEQhDZZoch6LHvI4HbOAIM3aTLR8O9hPeudAPJ9OgzvlZhfVLlK4QJRb8ADYfYCI3AyZb4xF7mEUQLUbZ9EiIkfHNBl8fzzyqhMeTY6oxK4sAatyu0Ku67CgfJR4AxOLHUKd0vVTcQ4eswNVGBIapEKbMexGrmL4FtV0c5rcu5xa6PiEVDNLvkD5KcxMvxbgDPnxhunvW5c5aQeSuiHYOVkiURaTDnP4JIcgDwH4MpcJfZtbwZezcE5XJwVDDAzlACaLZV642JQdQ7VSXTdLuJfHNheAtnaTdLPLawjktf1JpMZU6DveZVUTGUcgvN1hbPBTgxRMIXy2sVJJPrFXv9pjRItkDw8ivGX6972kheAex0HZML789Ks2eG6mI9Gp1JN2lw4hc78YYwBvDyi2vLoDP9Vcn32Cd9Ca6Rq9Pmi5nbUXUqbi3QNqjo5W1h1ekjL6rSG9ExJtZLCR3jwfSn9gdemwiMRi7M6eCnyvlKzVtPxOYGA223k2wjynuWuGHUOT7TrQ42wmDjXMfp0mhbCJxsivHULCC81hAozkgd1BaNFJ4cIAH1BgJJvunlB7pAcnyDqvN2sBvupw9As8uLUB0ochRf5E9o2qrm3R7cGDTM6RpGJ5D4DO48BViras5HIIOAf5ebrsfBskkK9fHe3sRbI1miceFOfXKMAlt1gkUIX7I7givW1bRuiIz5QXunwS7GY8xjLIdHpSwF94zy1JFgZP5wgkJs9fpMbrrbdHi1rILa5Rl9AnmsFiq1jONgT5DoucvFJ0MyXM2UyvODEACRwFzSI0EFMqCTVVPZwxjl6XTYB064Pk6ZNF7Hkl1a7VieyPxNoYE6Ngik4lslJg80djZwNm3PXOHTAJHiG7hszqYD5lYnxtnqInF2NIWRFtVRXzR1eJpKP0tJzR4x5FOCYg0tNm57meCAIjwanu7fMBsbrqDOMM1txXOuxcR3S1ohi9JlRyWapfSjjbaByKP7AtCB55pUhVrY0asrInRIW8OUZH1ti9rj9eSVLORpw0Pa5wqNhcnqFMDJgw9vo721WkwGHEpETAX1Pk7GE8adIwClJIYm9zYDYofkvfhrIDtqFrvmEF3Rq5n5K4hbprEoHogKzHemGkBYw6luv2qfN2vQS4QQICwXranq0fUY25f6Uzuu1IHgho2cVHSsurt4y9BhB6s1ZMwGwymykpt0mVmXXbt13U482VW45umJGOWcieCi7TjqmrNhwgZyScviPwfVhlg9CG4SW2NKc3yp9PoB1t8ffXMJBKgEmZ7ODbZ3ya00TQmamoQ1hqeifsdh5Kgck5ZxiaTMmrhIKC7cKx83P1AnT2t3PgFVV466YG1hX7Shyc51ykA1PoGcK50Irh0zDoZpc941oQSsCHoHDFneg50dxJZUMO7KYY0kApEsbnkAnXH74giY7TW96f1uvpgpEGB2vscWoEKpeswScNaIPwJJCOzWUC5tsfbZSdQqLTOq26d2H0dKYbaxi3LZvxGFQs4PgMszQiglc3cprfpsKKJmwPXnKm1lw8XtfImvlZvbSv4XyAaoSPDbCBPnI0C3hDoMfEG89WkGi4maOxeVccRWnYR4pWJIlAKb5JbwiK4FhoXnSdk5WN8XaYiqhHtSqob8tMW89OfENwXgvEg3PMkscbP16Fk9YsXylW73JZJncFQYL5evKZv21YoUAxEohqIlbR7Qjda4XHfDaYohURcP51Bs4W2vlcJihCehZ4HGb5KiWwWq8CrzKqXoDxEgA8hKjYMSiTj8osUhM0kTMTk79LGErZ90mOj6BvPIsWYnHiy4AyHDzuh7DFejzMnWmx0gEI88pNn4zvuwAAaPn9TANmZmsTmPhtS7dIbMoXKC2kbryesKLPDkxjBQDRoHkbkHPuBYxOciKimIGf6irMhj06rAZLxNYftaujnwxE4EoerhLYuHk7K2FEFiGw49xv3Ytqw99UGmLBiRkxIE2LtXpcNzoxcsQWEFqSs0MLHUvkHEgVtuuSw014qjvHAdZcqDFqforUf8HPa5yp7kxI5umQVHaKQl08yEvvhF1mFXKdLFsMHt1GOUMqyxRveYbCGJEWfwfeYeweMC7GyhHRoInzfhmaBkdnq0d7u4YQQt3cz82PfxVE5z7sl4WirUm4m7CzGCWMfbjdl3aGPvD1x73zREaHQBnPpw5HAThR0uXuwZEbHeXzz8esCsjAxiYvyR2C8H3mS9q4M2J8hDQOFFQMutM15m6Eclh6LVwvl4n3HFhsfRBy2ZZyKDS30A93PQHijIdp8J2KRN4ntTTBbEchsCm1Bvub58l7vhdxZTJWnN8VFIqlJhjNzvws4qeLXFdavHDvpW85rEmdnm624EkGMKb0sP9OinlKujpg48e1jBEuojxDNbklBcSaIiQNRGcHKezAe414KOlImg2TNbMAb9Y6nhbIb5SiMcgRYh5TAJMky7dlVJiMcTjzJ85hkzd961igKU81bB9Vecuj6cPQDqyjDKaPTxZMUMUluVcBGPHSVdiH7v4z967MBUaBPLSquVwPvxlt2lhN57vCukko6QVZkpKwbm1AM1KNCytRYe1S7lreye6Wwb0lrYma97rySUMbJQgucxONLkTgINxWrLfYSEF0QHxUL4SAatew6PGaxHccNXuQ2Tr2LcLSHgpvwdM32Axe7pvb1nBLvVO7MyweIH1NN089GhFUxUGl9Pcnax13GpZyjG8Bz58cynLQAz5OyshIbsRy6893aBOiYt5Fj8AEHjld5spPdHrEl6ec9O5o6n5hDx9EdjTuJIL4csC4taQqfjinqW9BuFrBoYGO2KmhjjQGLAvu6F0zTtSDLPvxWipTJU8ltiYJo0BsUQVfihyHGUEDWfNgnjtKosRydmLuQypdRNiYhBSajqGupS7jj5brvbrmJFuesbitd5qKIRBrAd2wTPzUOPre5WQziMK4dobCjffZlQualudKv60iz4aqE5NbGMgW8OAXTzN6MaHpaGpls6QNcnrgIhexb1E2jf1bDbVsbm6QK4CqOdwonbp8WZtEWzzbCFiUdwj0DfS880RtDYrQyNUBidXcgpKTEOpWK0Q9y9lJfUffREZKoiV1PPRYPjvCLBlqZ4YKbtxEo6DgjPnNFg4J0gHVa4fv3bATVmf2wK8wnjLo7sj29FsXOpKvGCRQpR4aBOzDdAGFJxOMO8Mj1UJTmRChf0TL1GxioCpkZrWRiqx8B8nVKTbS44KrIxqAc7vZIZLnMndSMWHI8KYzODdfZ5SDMBTTAJdPIgk2oOaeZ7drz8ho4N45vF2EfBd9l2YYxo7yOYv9j8rk4SWBbbmQMey5uy7dAHd7mUCFM2OH0sMi8AMT9ffGxonnizZf7qdoUA1okdUKiCW9lIo5CWn4ZlwizP4Li1Z0TQwqC6nW2e8nyMvePQBbMiEIaRc0K4LQGFr7PX3XoZ2BYI5VW5jHaoCzq5FbjLmx1HyiVkVdCHjxrn33CCntzp7ayMxatewEubeBTO0AbdnFqAg38rcblEppRCTz02O1un2BUKYI8MU0jyjaRLMvskhqKiNG1xA6K4QGPCBfAbHfejmonuG1IrVdm7HQWlAew2cxOUgi0NEsABlwuC0jVrHIq6RBu4I0EkY77J6zytmQNXYcqlLRVnsChKOmWsDv8xEhkbfQGsAAo9OB0oZoW5e0fIWz9DvA8RmBdg59Oxps8IB6g4sr111RrNiV11ilIDoUg8AV4uGGI80ANcpIEX9G4cFuY2Ny4uBqXVR8O7KQo3ICFHbIBwRsXNclcRP6m5nymyOFvICqq7h6x7O71jMAdmCBxmTP7g6mu5CV7riPLiqh1PBEWYncSztU4Q5TUloaQshdLImc52lOblcHkQJMhMbGKtYueXrPH0FPN1zGv0g7lkA29jNAigcWTEqVljSNbTlESpo6Gaf1zoYsiyDFS1fjoU5AO1Stb0SqhvqtYtIbxDKQAuNWavYJGd0A7wcBCMIQHmye7rgYaNYMimQymPIayusvgzL0f9zpLtEiRKLGMJY92F4BHBzKXQK6tJvxLV9uSeJcdDoLJPcNi68fdFUcrufAHIzEajDjlUrh5X3nETxdgyU3L4Yp5kUYfm9YTBCUYMZovEDbJRG2zYQHg36JtR6YyztyCzokTJXHmnT8GJPQVuJSl35IO7tgKERO3Guwy6cTtvr8aoSZk5XBubN7ty9URnNEfegkK2cXv3irpUfGqtlvFlk0daKQSXO99V3OPhj95GdZfeDXWyqOT806adHTqbeRIRR9bbDUW3ZDVf7IzExpA28JrQOE3rrgk3dGF4n5wisgNMVNSWwhpRSU0OZcNFSw0ZqtSz9XoPa4imdBe2WKvoSyUwYLGjbXNsvNd0rLeItBhNRxhy6tMwQqRaIdN6yGz04VFMsGvJOMenAgt5XR0EzQEt2LS6zpgT9FaBz9MRdIMshZUs5Tki4y1aqDTI479IDFfB8JFslcaGl6XKswef0xt3S74ufccCpwsu9ksn8cGcRemMYmnas3ObMTQVjyF7WKPizJJAsJj43rri51EnGH0k8fDKwWyAegutZgWsy9HUchQ0RuZSYI4Ect8OL29zGKiCtHIJv041TRcYxnConTY8jaPco13gock3zw3xb5khJQBe9AOG9OOOcgEBwjnmgI6S6fSOB5CSLaulZUTF00KbTvU0M4omiuUFMH93kU1JQQ7KIIjjjziUYebG0O19KopV4oyir16Saoyw9gpLChGEeIGmobSBpOmfivFlUBlkun7iloLaTqLOaBjAaJxxKEwHBwXHO9QH6Fp1gugBP77YPVIzIETaBtRSYLKL1t8s70NZeAzWJIk8jcBHbzhISSyTLfD8vmkGZwQNSQdI2BAxixA6MfPFeppv3NqSN6DcNkQVYOhocKa3kRnv7nc1gctNaYrMO113wbhlTLzEc7Ji4yRge7rJ2rWZcDjLYEWhZCwwZU4U1ARQqZJ3g4v5Z99W3ni0YnPuhpyGd929J4Ap8gikJLF7oYCaFrZ9oMbME1cLtw6GIIyfpSfUM6CfZAKXFl6TY7hepkrTXacYLFAMEde52YeNZ32J6pdR6otgrrhkpnPtXjI5voNu3YgwCeZoK6KZoc8kJ17P5rPTqqKxNTmS0rUI9l9CIL5DunJBdsWetHQHWf6LwThz671AgogPllGhShafHUFYFpRM1mNVIZC2LAwLwEqVW5G0YLXcW358kYXxzZ4XRvDcQfxtXqWyw9sM4j0z63daSxZrI3f0GljKdFe9GLBrYrj3deNeyqqsdTFTUVoNHjOoRBdNFHM0uuOK2JvBh0elBiTKPfcFXrUL6iSDBcEjrKTp354zeK6YmGHLfPYcLDtE3lpHsdjQncoXQox9C96X65RWqAZ29GPGS7lAAmUgKgvY9c64LHr56jAzBIIpDpabNTh0COMJhFvybmqkSV7oSkEEZeY1GCZDbhRuPUrWIahI6YwcM4gZgOSSwwUdbyaQjO2ynZffX3dZi5U9WtHGmHQNwJlUlaheo5ZPRcgcopnbxxwKSlA442obfGBCj1EkTjlwCMF9l7UIqdDSeRsT4D0QQpJrUG9AoNujQWSOUtW8lehlUJekbQqWTTfGvCiJeXpVqL4qHI2nstv4ttE3X0W8DtIcMfCSAeKpam1KDzyKOud8t89RfikSX7Q80xKYxgcFaSPqtfGbbGGc58FGi3BkW7DHHkkLRIufLJ33RvUt7ZgZmM23uBnqBRYp53zXbuRfSrAcsf3GMyWnqEfmty4Wx6diCyOnUP7xsUKIbwBcZWLuFVPTQ4rT7BXcghbsOca9jdUMQ0TGRhrTj5oDl5apYRbtAuddOjmF4XqUOHVQYAaL1yicIrdUqjZx5rbCbCL9bw3kz08lXh868vyIqnQQhKBSjhboppEMa7UfJBYWU5VKuQwFreuaYphUjE5xutjeuBNoanSqWNLu9AaeKcg7DGkKFmFsmySTsgGq48eAi5XIA1gQ1oqlWhOEeppUc4Y2R5UZuyAPBcmKCJ1BNMlRwPYO5iIdAvG3z6Xj19YxUaRvwFGtA6WLt8eUtMgzC2cNgIGLVDGWTF8ssd3X5FXyTSs3pOPpvo8BYGvo2bKqBK8zkaFZ46nCiBA3rkv5PIOwouUuRvcvuOTqqNb1mmcNB9f1yJxylO0ZJQN7h2gGyeKZPycjAHBmJb00g8NL3FcDbWwara17CjwoI1eqdLe1rIDR9IrjBcBEAbUJhExeIVacZgPQvOJeYZwgGiwZQAsBZMLyOA2sNH5EIt0suHLlsmXMSQFyDZb9I2vzozzpw1V80HPEQgrwYdiGyjRUFxm3ifuWGCicn9R9wDWHzsh2cSmIOzL7wyA1YKyLu8wA0UJfhDp0NFhCjxPHCK0etBkN0amvM2ikoczNanK7vJ37kGLnz8tBpc2n12CVZJc1qJnfVsitk9D6XDLXXQgOP6PoMZre2x5t7L2Y0cOlJoUzy1RjdvXucX9KypIQZ7CD9szNmCglwgxzIgrB2RqIEQWRQCkVuywUH7Z3p8CudyGHGDxs6fcOC9Wjy92D95RcNkZYZK1MWU1du7GGW6mSbvSVba3Faa74oBlxEm4RyC') - -- long string value in val2 - for TOAST testing. It should be random, bcs 'to TOAST or not to TOAST' decision happens after compression of values diff --git a/tests/e2e/pg2ch/alters_with_defaults/alters_test.go b/tests/e2e/pg2ch/alters_with_defaults/alters_test.go deleted file mode 100644 index 5e2d50e6..00000000 --- a/tests/e2e/pg2ch/alters_with_defaults/alters_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package alters - -import ( - "context" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/jackc/pgx/v4" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithPrefix("")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestAlter(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &Source) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - Target.ProtocolUnspecified = true - Target.MigrationOptions = &model.ChSinkMigrationOptions{ - AddNewColumns: true, - } - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{"public.__test"}} - var terminateErr error - localWorker := helpers.Activate(t, transfer, func(err error) { - terminateErr = err - }) - defer localWorker.Close(t) - - t.Run("ADD COLUMN with defaults", func(t *testing.T) { - // force INSERTs with different schemas to be pushed with one ApplyChangeItems call - err := conn.BeginFunc(context.Background(), func(tx pgx.Tx) error { - rows, err := tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (3, 3, 'e')") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "ALTER TABLE __test ADD COLUMN new_val1 TEXT DEFAULT 'test default value'") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "ALTER TABLE __test ADD COLUMN new_val2 INTEGER DEFAULT 1") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val1, new_val2) VALUES (4, 4, 'f', '4', 4)") - require.NoError(t, err) - rows.Close() - return nil - }) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) - }) - - t.Run("ADD COLUMN with complex defaults", func(t *testing.T) { - // force INSERTs with different schemas to be pushed with one ApplyChangeItems call - err := conn.BeginFunc(context.Background(), func(tx pgx.Tx) error { - rows, err := tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (5, 5, 'e')") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "ALTER TABLE __test ADD COLUMN new_val3 TEXT DEFAULT pg_size_pretty(EXTRACT(EPOCH from now())::bigint)") - require.NoError(t, err) - rows.Close() - - rows, err = tx.Query(context.Background(), "INSERT INTO __test (id, val1, val2, new_val1, new_val2) VALUES (6, 6, 'f', '6', 6)") - require.NoError(t, err) - rows.Close() - return nil - }) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // wait & compare - - st := time.Now() - for time.Since(st) < time.Minute { - time.Sleep(time.Second) - if terminateErr != nil { - break - } - } - require.Error(t, terminateErr) - require.True(t, abstract.IsFatal(terminateErr)) - }) -} diff --git a/tests/e2e/pg2ch/alters_with_defaults/dump/ch/dump.sql b/tests/e2e/pg2ch/alters_with_defaults/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/alters_with_defaults/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/alters_with_defaults/dump/pg/dump.sql b/tests/e2e/pg2ch/alters_with_defaults/dump/pg/dump.sql deleted file mode 100644 index f4c3e888..00000000 --- a/tests/e2e/pg2ch/alters_with_defaults/dump/pg/dump.sql +++ /dev/null @@ -1,13 +0,0 @@ --- needs to be sure there is db1 -create table __test -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test (id, val1, val2) -values (1, 1, 'a'), - (2, 2, 'XcTIan6Sk2JTT98F41uOn9BVdIapLVCu1fOfbVu8GC0q6q8dGQoF7BQU4GiTlj5DgXnp0E9mJX5SwD2BCNWri6jvODz8Gp4AMgEUZxLOjjFmt1VkgPrU67YIrmNCwre1b0SNJ90mvU5yFOoF3FWB3U2uT04wonF4wuwSWrWY9SExpormD7KOuLLYAjaGTd0bWH6ttDoVQLRkFofUYMz5cLJcSntWdMAU872qudaMG624AwCec5sOLm9b6QhHY3eusgV9pGHbXm7XmI6RF7lqSVDzxGzvyahYNMvkc6Cf6ccFK3fFUFO3WZkY5fT1ad3QTIqsP8WmyZEzol4GAiuzZAHvB2szeq1keaSzEeSoI6YPJXFevyRFzlVGJN7OxErxHnYd8TPPOyhQI0PwpQ7MY1cX9cWiqrxTl8lcDp23kntMsbmouacyEsHeFkagozm8muqnEM4w3qQhXNIOkV8pkoD0s2rxo5tytlBbW0OpgnKp6UxLAp7QqfmWXcOLIePdL3bOVI2WJfBXrgsnfVlnNukoH22rn4Vb3pvcsIyT4x8loFZzeVmXfR4xLeT73Vs5KDYYOGZOWdzh5KVWdvGTcpVU2fSNYl1GeDps45o7mTj2ycllkewLbGD84QNVP67aDujad7gLmt8jYrzwxS04AX7k2tz7tBE4gEqOefBwXyCBy1t9j7vSA9tg8ZupGMsy0QNzw1vRCo5jmNt3f4AjwWqBGYIIjYaS27vZwKOGdTTEqpbebWW45sBkxe9DrvrDYUi11wLMtr1sxKNzvZgfS65ROvjdXYJfkVXWtiqo8jpwf1KNdvTDJscQUFgh9e9XfCMAZTUOoBtQmQhDVQe4CON8JGVm4pDnKf7acwhAzxZU8X7HZblEQeYCKIA07MalK4f0XBzEL5rHmhLOry1a6uPFmaqx2DAHPegthCqcvgeNCXA48nrXXwgG04TLvNU4Xk3Lwwhug24btNMauk5w0cYPMl0DZ3CmnMleYe2u0pndVLsOY1PlKOLs8nrZEp6VKXrb3ZdkcZZ6c9h88dXIAkrrGoHh5cB2RtCTyZyBS0Y8akHDODUVh7LIYkd9vjZ4W9sPqxxnbGQfYIMWCm7zGLbhhOrf8GBN1dBdQvEZYWOsqrvGd2z1C8WiGXvrTjdUXnudsT1XYCniHyqpAVPLyQGZ3CSWaswmOi1bjeDOSN2t3fH50pyznZPmFbJfL8R1QFV3mCPCxkKc4o3eI24hOkX4MPepi6HlBadwgFbY69KDjKs9fphhUA2SYxvHWr3igc5Wp9ZmyBW88c1BxykzK8xbJseGrdavV4uSl96L0GnSpRhbJuKfX1QUDU42yImShSgdyXVci4O3lXVrJYqFHFrTd2jl2spp3V2VJqu3noUxrFZVmBCPOvg3Mqx0uAefGXtBI3T9vNJSrgFVNO4xFOa03oOlG1bRvT1I4bk7sBBAiVyQ0c445CxVPhhUuExt44BocoXFUDYh6EZGEw0OU56znN7wWqUaegqZpOMtRYZk5MpSIFauHyDXIVv17A6OHTN1zsW5hHIiWdQ8g5T362HvHiMLH3IhK1yL4jf29V5GqkKMkMb7kKPWTEn6ICkJQ4CBZSSKbEQhDZZoch6LHvI4HbOAIM3aTLR8O9hPeudAPJ9OgzvlZhfVLlK4QJRb8ADYfYCI3AyZb4xF7mEUQLUbZ9EiIkfHNBl8fzzyqhMeTY6oxK4sAatyu0Ku67CgfJR4AxOLHUKd0vVTcQ4eswNVGBIapEKbMexGrmL4FtV0c5rcu5xa6PiEVDNLvkD5KcxMvxbgDPnxhunvW5c5aQeSuiHYOVkiURaTDnP4JIcgDwH4MpcJfZtbwZezcE5XJwVDDAzlACaLZV642JQdQ7VSXTdLuJfHNheAtnaTdLPLawjktf1JpMZU6DveZVUTGUcgvN1hbPBTgxRMIXy2sVJJPrFXv9pjRItkDw8ivGX6972kheAex0HZML789Ks2eG6mI9Gp1JN2lw4hc78YYwBvDyi2vLoDP9Vcn32Cd9Ca6Rq9Pmi5nbUXUqbi3QNqjo5W1h1ekjL6rSG9ExJtZLCR3jwfSn9gdemwiMRi7M6eCnyvlKzVtPxOYGA223k2wjynuWuGHUOT7TrQ42wmDjXMfp0mhbCJxsivHULCC81hAozkgd1BaNFJ4cIAH1BgJJvunlB7pAcnyDqvN2sBvupw9As8uLUB0ochRf5E9o2qrm3R7cGDTM6RpGJ5D4DO48BViras5HIIOAf5ebrsfBskkK9fHe3sRbI1miceFOfXKMAlt1gkUIX7I7givW1bRuiIz5QXunwS7GY8xjLIdHpSwF94zy1JFgZP5wgkJs9fpMbrrbdHi1rILa5Rl9AnmsFiq1jONgT5DoucvFJ0MyXM2UyvODEACRwFzSI0EFMqCTVVPZwxjl6XTYB064Pk6ZNF7Hkl1a7VieyPxNoYE6Ngik4lslJg80djZwNm3PXOHTAJHiG7hszqYD5lYnxtnqInF2NIWRFtVRXzR1eJpKP0tJzR4x5FOCYg0tNm57meCAIjwanu7fMBsbrqDOMM1txXOuxcR3S1ohi9JlRyWapfSjjbaByKP7AtCB55pUhVrY0asrInRIW8OUZH1ti9rj9eSVLORpw0Pa5wqNhcnqFMDJgw9vo721WkwGHEpETAX1Pk7GE8adIwClJIYm9zYDYofkvfhrIDtqFrvmEF3Rq5n5K4hbprEoHogKzHemGkBYw6luv2qfN2vQS4QQICwXranq0fUY25f6Uzuu1IHgho2cVHSsurt4y9BhB6s1ZMwGwymykpt0mVmXXbt13U482VW45umJGOWcieCi7TjqmrNhwgZyScviPwfVhlg9CG4SW2NKc3yp9PoB1t8ffXMJBKgEmZ7ODbZ3ya00TQmamoQ1hqeifsdh5Kgck5ZxiaTMmrhIKC7cKx83P1AnT2t3PgFVV466YG1hX7Shyc51ykA1PoGcK50Irh0zDoZpc941oQSsCHoHDFneg50dxJZUMO7KYY0kApEsbnkAnXH74giY7TW96f1uvpgpEGB2vscWoEKpeswScNaIPwJJCOzWUC5tsfbZSdQqLTOq26d2H0dKYbaxi3LZvxGFQs4PgMszQiglc3cprfpsKKJmwPXnKm1lw8XtfImvlZvbSv4XyAaoSPDbCBPnI0C3hDoMfEG89WkGi4maOxeVccRWnYR4pWJIlAKb5JbwiK4FhoXnSdk5WN8XaYiqhHtSqob8tMW89OfENwXgvEg3PMkscbP16Fk9YsXylW73JZJncFQYL5evKZv21YoUAxEohqIlbR7Qjda4XHfDaYohURcP51Bs4W2vlcJihCehZ4HGb5KiWwWq8CrzKqXoDxEgA8hKjYMSiTj8osUhM0kTMTk79LGErZ90mOj6BvPIsWYnHiy4AyHDzuh7DFejzMnWmx0gEI88pNn4zvuwAAaPn9TANmZmsTmPhtS7dIbMoXKC2kbryesKLPDkxjBQDRoHkbkHPuBYxOciKimIGf6irMhj06rAZLxNYftaujnwxE4EoerhLYuHk7K2FEFiGw49xv3Ytqw99UGmLBiRkxIE2LtXpcNzoxcsQWEFqSs0MLHUvkHEgVtuuSw014qjvHAdZcqDFqforUf8HPa5yp7kxI5umQVHaKQl08yEvvhF1mFXKdLFsMHt1GOUMqyxRveYbCGJEWfwfeYeweMC7GyhHRoInzfhmaBkdnq0d7u4YQQt3cz82PfxVE5z7sl4WirUm4m7CzGCWMfbjdl3aGPvD1x73zREaHQBnPpw5HAThR0uXuwZEbHeXzz8esCsjAxiYvyR2C8H3mS9q4M2J8hDQOFFQMutM15m6Eclh6LVwvl4n3HFhsfRBy2ZZyKDS30A93PQHijIdp8J2KRN4ntTTBbEchsCm1Bvub58l7vhdxZTJWnN8VFIqlJhjNzvws4qeLXFdavHDvpW85rEmdnm624EkGMKb0sP9OinlKujpg48e1jBEuojxDNbklBcSaIiQNRGcHKezAe414KOlImg2TNbMAb9Y6nhbIb5SiMcgRYh5TAJMky7dlVJiMcTjzJ85hkzd961igKU81bB9Vecuj6cPQDqyjDKaPTxZMUMUluVcBGPHSVdiH7v4z967MBUaBPLSquVwPvxlt2lhN57vCukko6QVZkpKwbm1AM1KNCytRYe1S7lreye6Wwb0lrYma97rySUMbJQgucxONLkTgINxWrLfYSEF0QHxUL4SAatew6PGaxHccNXuQ2Tr2LcLSHgpvwdM32Axe7pvb1nBLvVO7MyweIH1NN089GhFUxUGl9Pcnax13GpZyjG8Bz58cynLQAz5OyshIbsRy6893aBOiYt5Fj8AEHjld5spPdHrEl6ec9O5o6n5hDx9EdjTuJIL4csC4taQqfjinqW9BuFrBoYGO2KmhjjQGLAvu6F0zTtSDLPvxWipTJU8ltiYJo0BsUQVfihyHGUEDWfNgnjtKosRydmLuQypdRNiYhBSajqGupS7jj5brvbrmJFuesbitd5qKIRBrAd2wTPzUOPre5WQziMK4dobCjffZlQualudKv60iz4aqE5NbGMgW8OAXTzN6MaHpaGpls6QNcnrgIhexb1E2jf1bDbVsbm6QK4CqOdwonbp8WZtEWzzbCFiUdwj0DfS880RtDYrQyNUBidXcgpKTEOpWK0Q9y9lJfUffREZKoiV1PPRYPjvCLBlqZ4YKbtxEo6DgjPnNFg4J0gHVa4fv3bATVmf2wK8wnjLo7sj29FsXOpKvGCRQpR4aBOzDdAGFJxOMO8Mj1UJTmRChf0TL1GxioCpkZrWRiqx8B8nVKTbS44KrIxqAc7vZIZLnMndSMWHI8KYzODdfZ5SDMBTTAJdPIgk2oOaeZ7drz8ho4N45vF2EfBd9l2YYxo7yOYv9j8rk4SWBbbmQMey5uy7dAHd7mUCFM2OH0sMi8AMT9ffGxonnizZf7qdoUA1okdUKiCW9lIo5CWn4ZlwizP4Li1Z0TQwqC6nW2e8nyMvePQBbMiEIaRc0K4LQGFr7PX3XoZ2BYI5VW5jHaoCzq5FbjLmx1HyiVkVdCHjxrn33CCntzp7ayMxatewEubeBTO0AbdnFqAg38rcblEppRCTz02O1un2BUKYI8MU0jyjaRLMvskhqKiNG1xA6K4QGPCBfAbHfejmonuG1IrVdm7HQWlAew2cxOUgi0NEsABlwuC0jVrHIq6RBu4I0EkY77J6zytmQNXYcqlLRVnsChKOmWsDv8xEhkbfQGsAAo9OB0oZoW5e0fIWz9DvA8RmBdg59Oxps8IB6g4sr111RrNiV11ilIDoUg8AV4uGGI80ANcpIEX9G4cFuY2Ny4uBqXVR8O7KQo3ICFHbIBwRsXNclcRP6m5nymyOFvICqq7h6x7O71jMAdmCBxmTP7g6mu5CV7riPLiqh1PBEWYncSztU4Q5TUloaQshdLImc52lOblcHkQJMhMbGKtYueXrPH0FPN1zGv0g7lkA29jNAigcWTEqVljSNbTlESpo6Gaf1zoYsiyDFS1fjoU5AO1Stb0SqhvqtYtIbxDKQAuNWavYJGd0A7wcBCMIQHmye7rgYaNYMimQymPIayusvgzL0f9zpLtEiRKLGMJY92F4BHBzKXQK6tJvxLV9uSeJcdDoLJPcNi68fdFUcrufAHIzEajDjlUrh5X3nETxdgyU3L4Yp5kUYfm9YTBCUYMZovEDbJRG2zYQHg36JtR6YyztyCzokTJXHmnT8GJPQVuJSl35IO7tgKERO3Guwy6cTtvr8aoSZk5XBubN7ty9URnNEfegkK2cXv3irpUfGqtlvFlk0daKQSXO99V3OPhj95GdZfeDXWyqOT806adHTqbeRIRR9bbDUW3ZDVf7IzExpA28JrQOE3rrgk3dGF4n5wisgNMVNSWwhpRSU0OZcNFSw0ZqtSz9XoPa4imdBe2WKvoSyUwYLGjbXNsvNd0rLeItBhNRxhy6tMwQqRaIdN6yGz04VFMsGvJOMenAgt5XR0EzQEt2LS6zpgT9FaBz9MRdIMshZUs5Tki4y1aqDTI479IDFfB8JFslcaGl6XKswef0xt3S74ufccCpwsu9ksn8cGcRemMYmnas3ObMTQVjyF7WKPizJJAsJj43rri51EnGH0k8fDKwWyAegutZgWsy9HUchQ0RuZSYI4Ect8OL29zGKiCtHIJv041TRcYxnConTY8jaPco13gock3zw3xb5khJQBe9AOG9OOOcgEBwjnmgI6S6fSOB5CSLaulZUTF00KbTvU0M4omiuUFMH93kU1JQQ7KIIjjjziUYebG0O19KopV4oyir16Saoyw9gpLChGEeIGmobSBpOmfivFlUBlkun7iloLaTqLOaBjAaJxxKEwHBwXHO9QH6Fp1gugBP77YPVIzIETaBtRSYLKL1t8s70NZeAzWJIk8jcBHbzhISSyTLfD8vmkGZwQNSQdI2BAxixA6MfPFeppv3NqSN6DcNkQVYOhocKa3kRnv7nc1gctNaYrMO113wbhlTLzEc7Ji4yRge7rJ2rWZcDjLYEWhZCwwZU4U1ARQqZJ3g4v5Z99W3ni0YnPuhpyGd929J4Ap8gikJLF7oYCaFrZ9oMbME1cLtw6GIIyfpSfUM6CfZAKXFl6TY7hepkrTXacYLFAMEde52YeNZ32J6pdR6otgrrhkpnPtXjI5voNu3YgwCeZoK6KZoc8kJ17P5rPTqqKxNTmS0rUI9l9CIL5DunJBdsWetHQHWf6LwThz671AgogPllGhShafHUFYFpRM1mNVIZC2LAwLwEqVW5G0YLXcW358kYXxzZ4XRvDcQfxtXqWyw9sM4j0z63daSxZrI3f0GljKdFe9GLBrYrj3deNeyqqsdTFTUVoNHjOoRBdNFHM0uuOK2JvBh0elBiTKPfcFXrUL6iSDBcEjrKTp354zeK6YmGHLfPYcLDtE3lpHsdjQncoXQox9C96X65RWqAZ29GPGS7lAAmUgKgvY9c64LHr56jAzBIIpDpabNTh0COMJhFvybmqkSV7oSkEEZeY1GCZDbhRuPUrWIahI6YwcM4gZgOSSwwUdbyaQjO2ynZffX3dZi5U9WtHGmHQNwJlUlaheo5ZPRcgcopnbxxwKSlA442obfGBCj1EkTjlwCMF9l7UIqdDSeRsT4D0QQpJrUG9AoNujQWSOUtW8lehlUJekbQqWTTfGvCiJeXpVqL4qHI2nstv4ttE3X0W8DtIcMfCSAeKpam1KDzyKOud8t89RfikSX7Q80xKYxgcFaSPqtfGbbGGc58FGi3BkW7DHHkkLRIufLJ33RvUt7ZgZmM23uBnqBRYp53zXbuRfSrAcsf3GMyWnqEfmty4Wx6diCyOnUP7xsUKIbwBcZWLuFVPTQ4rT7BXcghbsOca9jdUMQ0TGRhrTj5oDl5apYRbtAuddOjmF4XqUOHVQYAaL1yicIrdUqjZx5rbCbCL9bw3kz08lXh868vyIqnQQhKBSjhboppEMa7UfJBYWU5VKuQwFreuaYphUjE5xutjeuBNoanSqWNLu9AaeKcg7DGkKFmFsmySTsgGq48eAi5XIA1gQ1oqlWhOEeppUc4Y2R5UZuyAPBcmKCJ1BNMlRwPYO5iIdAvG3z6Xj19YxUaRvwFGtA6WLt8eUtMgzC2cNgIGLVDGWTF8ssd3X5FXyTSs3pOPpvo8BYGvo2bKqBK8zkaFZ46nCiBA3rkv5PIOwouUuRvcvuOTqqNb1mmcNB9f1yJxylO0ZJQN7h2gGyeKZPycjAHBmJb00g8NL3FcDbWwara17CjwoI1eqdLe1rIDR9IrjBcBEAbUJhExeIVacZgPQvOJeYZwgGiwZQAsBZMLyOA2sNH5EIt0suHLlsmXMSQFyDZb9I2vzozzpw1V80HPEQgrwYdiGyjRUFxm3ifuWGCicn9R9wDWHzsh2cSmIOzL7wyA1YKyLu8wA0UJfhDp0NFhCjxPHCK0etBkN0amvM2ikoczNanK7vJ37kGLnz8tBpc2n12CVZJc1qJnfVsitk9D6XDLXXQgOP6PoMZre2x5t7L2Y0cOlJoUzy1RjdvXucX9KypIQZ7CD9szNmCglwgxzIgrB2RqIEQWRQCkVuywUH7Z3p8CudyGHGDxs6fcOC9Wjy92D95RcNkZYZK1MWU1du7GGW6mSbvSVba3Faa74oBlxEm4RyC') - -- long string value in val2 - for TOAST testing. It should be random, bcs 'to TOAST or not to TOAST' decision happens after compression of values diff --git a/tests/e2e/pg2ch/chmapper/canondata/chmapper.chmapper.TestSnapshot/extracted b/tests/e2e/pg2ch/chmapper/canondata/chmapper.chmapper.TestSnapshot/extracted deleted file mode 100644 index 7e795fe7..00000000 --- a/tests/e2e/pg2ch/chmapper/canondata/chmapper.chmapper.TestSnapshot/extracted +++ /dev/null @@ -1,325 +0,0 @@ - -"public"."remapped_table" -{ - "meta": - [ - { - "name": "first_id", - "type": "UInt64" - }, - { - "name": "second_id", - "type": "UInt64" - }, - { - "name": "third_id", - "type": "LowCardinality(String)" - }, - { - "name": "f", - "type": "Float64" - }, - { - "name": "d", - "type": "Float64" - }, - { - "name": "i", - "type": "Int32" - }, - { - "name": "bi", - "type": "Int64" - }, - { - "name": "biu", - "type": "Int64" - }, - { - "name": "bit", - "type": "String" - }, - { - "name": "da", - "type": "Date" - }, - { - "name": "ts", - "type": "DateTime64(6)" - }, - { - "name": "dt", - "type": "DateTime64(6)" - }, - { - "name": "c", - "type": "Nullable(String)" - }, - { - "name": "t", - "type": "String" - } - ], - - "data": - [ - { - "first_id": "0", - "second_id": "1", - "third_id": "hello", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "1", - "second_id": "0", - "third_id": "hello, friend of mine", - "f": 1.45e-10, - "d": 3.14e-100, - "i": -8388605, - "bi": "2147483642", - "biu": "9223372036854775804", - "bit": "", - "da": "2005-03-04", - "ts": "1999-03-06 00:00:00.000000", - "dt": "2005-03-12 00:00:00.000000", - "c": "1", - "t": "e1bf50dc0107c34c2c31bad477dd0cd3113109d2cea69724d323952f4fea8128" - }, - { - "first_id": "2", - "second_id": "1", - "third_id": "another hello", - "f": 1.34e-10, - "d": 0, - "i": -1294129412, - "bi": "112412412421941041", - "biu": "129491244912401240", - "bit": "", - "da": "1999-03-04", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": "2", - "t": "64f039e84325d37c8fde00ee3253352731ada8d664690e9c2a436590dc08f8bd" - }, - { - "first_id": "3", - "second_id": "4", - "third_id": "another another hello", - "f": 5.34e-10, - "d": 0, - "i": 294129412, - "bi": "-784124124219410491", - "biu": "129491098649360240", - "bit": "", - "da": "1999-03-05", - "ts": "1999-03-06 00:00:00.000000", - "dt": "2005-03-12 00:00:00.000000", - "c": "c", - "t": "3fc1c9a1c704a2eda7a27d525d42cf6ceb4218763824bacc9949f425130dee5b" - }, - { - "first_id": "100", - "second_id": "11", - "third_id": "100", - "f": null, - "d": null, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "101", - "second_id": "12", - "third_id": "101", - "f": null, - "d": null, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "102", - "second_id": "13", - "third_id": "102", - "f": null, - "d": null, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "201", - "second_id": "6", - "third_id": "nvaapsijfapfn", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "2005-03-12", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "202", - "second_id": "7", - "third_id": "Day the creator of this code was born", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1999-09-16", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "214", - "second_id": "2", - "third_id": "aaa", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "322", - "second_id": "8", - "third_id": "Coronavirus made me leave", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "2020-06-03", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "333", - "second_id": "10", - "third_id": "Remember me, my name is hazzus", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "2005-03-12", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "422", - "second_id": "9", - "third_id": "But Ill be back, this is public promise", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "2005-03-12", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "1234", - "second_id": "4", - "third_id": "agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "12345", - "second_id": "5", - "third_id": "aagiangsfnaofasoasvboas", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - }, - { - "first_id": "124124", - "second_id": "3", - "third_id": "vvvv", - "f": 0, - "d": 0, - "i": 0, - "bi": "0", - "biu": "0", - "bit": "", - "da": "1970-01-01", - "ts": "1999-03-06 00:00:00.000000", - "dt": "1970-01-01 00:00:00.000000", - "c": null, - "t": "5f2b7ee157423948a1dc38e58997df87efa382dd37af23f8df7c1e31fcb4fef0" - } - ], - - "rows": 16 -} diff --git a/tests/e2e/pg2ch/chmapper/canondata/result.json b/tests/e2e/pg2ch/chmapper/canondata/result.json deleted file mode 100644 index 0727872a..00000000 --- a/tests/e2e/pg2ch/chmapper/canondata/result.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "chmapper.chmapper.TestSnapshot": { - "uri": "file://chmapper.chmapper.TestSnapshot/extracted" - } -} diff --git a/tests/e2e/pg2ch/chmapper/check_db_test.go b/tests/e2e/pg2ch/chmapper/check_db_test.go deleted file mode 100644 index 43169252..00000000 --- a/tests/e2e/pg2ch/chmapper/check_db_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package chmapper - -import ( - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/transformer" - "github.com/doublecloud/tross/pkg/transformer/registry/chmapper" - "github.com/doublecloud/tross/tests/canon/reference" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotOnly - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - helpers.InitSrcDst(helpers.TransferID, Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func testSnapshot(t *testing.T, source *postgres.PgSource, target model.ChDestination) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "CH target Native", Port: target.NativePort}, - helpers.LabeledPort{Label: "CH target HTTP", Port: target.HTTPPort}, - )) - }() - - Target.ForceJSONMode = false - transfer := helpers.MakeTransfer(helpers.TransferID, source, &target, TransferType) - transfer.TypeSystemVersion = 8 - transfer.Transformation = &server.Transformation{Transformers: &transformer.Transformers{Transformers: []transformer.Transformer{ - { - chmapper.TransformerType: chmapper.Config{ - Table: chmapper.TableRemap{ - OriginalTable: abstract.TableID{ - Namespace: "public", - Name: "__test", - }, - RenamedTable: abstract.TableID{ - Namespace: "", - Name: "remapped_table", - }, - Columns: []chmapper.ColumnRemap{ - {Name: "first_id", Path: "id", Key: true, Type: schema.TypeInt64.String(), TargetType: "UInt64"}, - {Name: "second_id", Path: "aid", Key: true, Type: schema.TypeInt32.String(), TargetType: "UInt64"}, - {Name: "third_id", Path: "str", Key: true, Type: schema.TypeString.String(), TargetType: "LowCardinality(String)"}, - {Name: "f", Type: schema.TypeFloat64.String()}, - {Name: "d", Type: schema.TypeFloat64.String()}, - {Name: "i", Type: schema.TypeInt32.String()}, - {Name: "bi", Type: schema.TypeInt64.String()}, - {Name: "biu", Type: schema.TypeUint64.String()}, - {Name: "bit", Type: schema.TypeInt8.String()}, - {Name: "da", Type: schema.TypeDate.String(), TargetType: "Date"}, - {Name: "ts", Type: schema.TypeTimestamp.String(), TargetType: "DateTime(6)"}, - {Name: "dt", Type: schema.TypeDatetime.String(), TargetType: "DateTime(6)"}, - {Name: "c", Type: schema.TypeString.String(), TargetType: "Nullable(String)"}, - {Name: "t", Masked: true, Type: schema.TypeString.String(), TargetType: "String"}, - }, - DDL: &chmapper.DDLParams{ - PartitionBy: "toDate(ts)", - OrderBy: "`first_id`, `second_id`, `third_id`", - Engine: "MergeTree()", - TTL: "toDate(ts) + INTERVAL 700 year", - }, - }, - Salt: "test", - }, - }, - }}} - helpers.Activate(t, transfer, func(err error) { - require.NoError(t, err) - }) - reference.Dump(t, &model.ChSource{ - Database: "public", - ShardsList: []model.ClickHouseShard{{Name: "_", Hosts: []string{"localhost"}}}, - NativePort: target.NativePort, - HTTPPort: target.HTTPPort, - User: target.User, - }) - require.NoError(t, - helpers.WaitEqualRowsCountDifferentTables( - t, - "public", - "__test", - target.Database, - "remapped_table", - helpers.GetSampleableStorageByModel(t, Source), - helpers.GetSampleableStorageByModel(t, Target), - 10*time.Second, - ), - ) -} - -func TestSnapshot(t *testing.T) { - target := Target - - testSnapshot(t, Source, target) -} diff --git a/tests/e2e/pg2ch/chmapper/dump/ch/dump.sql b/tests/e2e/pg2ch/chmapper/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/chmapper/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/chmapper/dump/pg/dump.sql b/tests/e2e/pg2ch/chmapper/dump/pg/dump.sql deleted file mode 100644 index 126c4760..00000000 --- a/tests/e2e/pg2ch/chmapper/dump/pg/dump.sql +++ /dev/null @@ -1,91 +0,0 @@ --- needs to be sure there is db1 -create table __test ( - id bigint not null, - aid serial, - f float, - d double precision, - de decimal(10,2), - i int, - bi bigint, - biu bigint, - b bit(8), - da date, - ts timestamp default '1999-03-06'::timestamp, - dt timestamp, - c char, - str varchar(256), - t text, - primary key (aid, str, id) -- test multi pk and reverse order keys -); - -insert into __test values ( - 1, - 0, - 1.45e-10, - 3.14e-100, - 2.5, - -8388605, - 2147483642, - 9223372036854775804, - b'10101111', - '2005-03-04', - '1999-03-06', - '2005-03-12', - '1', - 'hello, friend of mine', - 'okay, now bye-bye' -) -, -( - 2, - 1, - 1.34e-10, - null, - null, - -1294129412, - 112412412421941041, - 129491244912401240, - b'10000001', - '1999-03-04', - '1999-03-06', - null, - '2', - 'another hello', - 'okay, another bye' -) -, -( - 3, - 4, - 5.34e-10, - null, - 123, - 294129412, - -784124124219410491, - 129491098649360240, - b'10000010', - '1999-03-05', - '1999-03-06', - '2005-03-12', - 'c', - 'another another hello', - 'okay, another another bye' -) -; - -insert into __test (str, id) values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) values ('nvaapsijfapfn', 201, '2005-03-12'), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But Ill be back, this is public promise', 422, '2005-03-12'), - ('Remember me, my name is hazzus', 333, '2005-03-12'); - -insert into __test (str, id, f, d, de) values ('100', 100, 'NaN'::real, 'NaN'::double precision, 'NaN'::numeric); -insert into __test (str, id, f, d) values - ('101', 101, '+Inf'::real, '+Inf'::double precision), - ('102', 102, '-Inf'::real, '-Inf'::double precision); diff --git a/tests/e2e/pg2ch/comparator.go b/tests/e2e/pg2ch/comparator.go deleted file mode 100644 index 2a9dbcb5..00000000 --- a/tests/e2e/pg2ch/comparator.go +++ /dev/null @@ -1,83 +0,0 @@ -package pg2ch - -import ( - "fmt" - "strings" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/util/jsonx" - "github.com/doublecloud/tross/yt/go/schema" -) - -// PG2CHDataTypesComparator properly compares data types in pg2ch transfers, skipping special cases. -func PG2CHDataTypesComparator(l, r string) bool { - // ClickHouse converts any to string - if l == schema.TypeAny.String() && r == "string" { - return true - } - // ClickHouse converts utf8 to string - if l == schema.TypeString.String() && r == "string" { - return true - } - if l == schema.TypeBoolean.String() && r == schema.TypeUint8.String() { - return true - } - return l == r -} - -// PG2CHDataTypesComparatorV1 properly compares data types in pg2ch transfers, skipping special cases. -// -// This is a version of this comparator for type system `1`. -func PG2CHDataTypesComparatorV1(l, r string) bool { - // ClickHouse converts YT DateTime and Timestamp to utf8 - if l == schema.TypeTimestamp.String() || l == schema.TypeDatetime.String() || l == schema.TypeDate.String() { - return r == "string" - } - return PG2CHDataTypesComparator(l, r) -} - -func ValueComparator(l interface{}, lCol abstract.ColSchema, r interface{}, rCol abstract.ColSchema, _ bool) (comparable bool, result bool, err error) { - if boolV, ok := l.(bool); ok { - return true, boolV == (r == uint8(1)), nil - } - if lfloat64V, ok := l.(float64); ok { - if rfloat64V, ok := r.(float64); ok { - // TODO: Fix and Remove - // looks like CH looze some data - // (source) 2.802596928649634e-45 [float64 | "pg:real" | "real"] != 2.8026e-45 [float64 | "ch:Nullable(Float64)" | ""] - return true, fmt.Sprintf("%.8f", lfloat64V) == fmt.Sprintf("%.8f", rfloat64V), nil - } - } - if lCol.OriginalType == "pg:time without time zone" { - // ch drop rest of zeros - lString := l.(string) - rString := r.(string) - lString = strings.ReplaceAll(lString, ".000000", "") - return true, strings.TrimRightFunc(lString, func(r rune) bool { - return r == '0' - }) == rString, nil - } - if lCol.OriginalType == "pg:interval" { - // ch drop tail of .00000 - lString := l.(string) - rString := r.(string) - - return true, strings.ReplaceAll(lString, ".000000", "") == rString, nil - } - return false, false, nil -} - -func ValueJSONNullComparator(l interface{}, lCol abstract.ColSchema, r interface{}, rCol abstract.ColSchema, _ bool) (comparable bool, result bool, err error) { - _, lIsJSONNull := l.(jsonx.JSONNull) - if !lIsJSONNull { - return false, false, nil - } - - if r == nil { - return true, true, nil - } - if rString, rStringCasts := r.(string); rStringCasts { - return true, rString == "null", nil - } - return true, false, nil -} diff --git a/tests/e2e/pg2ch/date_overflow/check_db_test.go b/tests/e2e/pg2ch/date_overflow/check_db_test.go deleted file mode 100644 index c0003d13..00000000 --- a/tests/e2e/pg2ch/date_overflow/check_db_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/test/yatest" - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = pgrecipe.RecipeSource(pgrecipe.WithFiles(yatest.SourcePath("transfer_manager/go/tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql"))) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func testSnapshot(t *testing.T, source *postgres.PgSource, target model.ChDestination) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "CH target Native", Port: target.NativePort}, - helpers.LabeledPort{Label: "CH target HTTP", Port: target.HTTPPort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, source, &target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(client2.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.Error(t, err) - require.True(t, abstract.IsFatal(err)) -} - -func TestSnapshot(t *testing.T) { - target := Target - - testSnapshot(t, Source, target) -} diff --git a/tests/e2e/pg2ch/date_overflow/dump/ch/dump.sql b/tests/e2e/pg2ch/date_overflow/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/date_overflow/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql b/tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql deleted file mode 100644 index 2aeb520b..00000000 --- a/tests/e2e/pg2ch/date_overflow/dump/pg/dump.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE "invoice" -( - "invoice_id" INT NOT NULL, - "customer_id" INT NOT NULL, - "invoice_date" TIMESTAMP NOT NULL, - "billing_address" VARCHAR(70), - "billing_city" VARCHAR(40), - "billing_state" VARCHAR(40), - "billing_country" VARCHAR(40), - "billing_postal_code" VARCHAR(10), - "total" NUMERIC(10,2) NOT NULL, - CONSTRAINT "PK_Invoice" PRIMARY KEY ("invoice_id") -); - -insert into public.invoice (invoice_id, customer_id, invoice_date, billing_address, billing_city, billing_state, billing_country, billing_postal_code, total) values (1, 1, '0001-01-01 00:00:00.000000', null, null, null, null, null, 492.00); diff --git a/tests/e2e/pg2ch/dbt/check_db_test.go b/tests/e2e/pg2ch/dbt/check_db_test.go deleted file mode 100644 index c77dd306..00000000 --- a/tests/e2e/pg2ch/dbt/check_db_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package dbt - -import ( - "fmt" - "os" - "testing" - - "github.com/doublecloud/tross/internal/test/yatest" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/shared/pod" - transformers_registry "github.com/doublecloud/tross/pkg/transformer" - "github.com/doublecloud/tross/pkg/transformer/registry/dbt" - _ "github.com/doublecloud/tross/pkg/transformer/registry/dbt/clickhouse" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestSnapshot(t *testing.T) { - if os.Getenv("TEAMCITY_RUN") != "1" || os.Getenv("USE_TESTCONTAINERS") != "1" { - t.Skip() - } - - source := pgrecipe.RecipeSource( - pgrecipe.WithInitFiles(yatest.SourcePath("transfer_manager/go/tests/e2e/pg2ch/dbt/init_pg.sql")), - pgrecipe.WithoutPgDump(), - ) - target := chrecipe.MustTarget( - chrecipe.WithInitFile(yatest.SourcePath("transfer_manager/go/tests/e2e/pg2ch/dbt/init_ch.sql")), - chrecipe.WithDatabase("dbttest"), - ) - - // TODO: LoadConfig() for preprod should be used here. However, the config cannot be loaded partially, thus its load requires proper environment in which it is available - dpconfig.AWS = &dpconfig.AWSConfig{ - ContainerRegistryService: "public.ecr.aws/t9p9v8b9", - DBTImageTag: "12197361.preprod", - } - pod.SharedDir = "/tmp" - - githubPAT := os.Getenv("DOUBLECLOUD_GITHUB_PERSONAL_ACCESS_TOKEN") - if githubPAT == "" { - t.Skip("DOUBLECLOUD_GITHUB_PERSONAL_ACCESS_TOKEN not provided") - } - require.NotEmpty(t, githubPAT) - - // Source.WithDefaults() // has already been initialized by the `helpers` package - target.WithDefaults() - target.ProtocolUnspecified = true - target.UseSchemaInTableName = true - target.Cleanup = server.Drop - transfer := helpers.MakeTransfer("testtransfer", source, target, abstract.TransferTypeSnapshotOnly) - addTransformationToTransfer(transfer, dbt.Config{ - GitRepositoryLink: fmt.Sprintf("https://%s@github.com/doublecloud/tests-clickhouse-dbt.git", githubPAT), - ProfileName: "clickhouse", - Operation: "run", - }) - - _ = helpers.Activate(t, transfer) - - targetAsStorage := helpers.GetSampleableStorageByModel(t, target) - targetTables, err := targetAsStorage.TableList(nil) - require.NoError(t, err) - require.Contains(t, targetTables, *abstract.NewTableID("dbttest", "v1")) - require.Contains(t, targetTables, *abstract.NewTableID("dbttest", "v2")) - require.Contains(t, targetTables, *abstract.NewTableID("dbttest", "v3")) -} - -func addTransformationToTransfer(transfer *server.Transfer, config dbt.Config) { - if transfer.Transformation == nil { - transfer.Transformation = &server.Transformation{ - ExtraTransformers: nil, - Executor: nil, - } - } - if transfer.Transformation.Transformers == nil { - transfer.Transformation.Transformers = new(transformers_registry.Transformers) - } - transfer.Transformation.Transformers.Transformers = append(transfer.Transformation.Transformers.Transformers, transformers_registry.Transformer{ - dbt.TransformerType: config, - }) -} diff --git a/tests/e2e/pg2ch/dbt/init_ch.sql b/tests/e2e/pg2ch/dbt/init_ch.sql deleted file mode 100644 index 8a9ce95d..00000000 --- a/tests/e2e/pg2ch/dbt/init_ch.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE IF NOT EXISTS dbttest; diff --git a/tests/e2e/pg2ch/dbt/init_pg.sql b/tests/e2e/pg2ch/dbt/init_pg.sql deleted file mode 100644 index dd892cfb..00000000 --- a/tests/e2e/pg2ch/dbt/init_pg.sql +++ /dev/null @@ -1,11 +0,0 @@ -DROP TABLE IF EXISTS public.t1; -CREATE TABLE public.t1(i INT PRIMARY KEY, t TEXT); -INSERT INTO public.t1(i, t) SELECT gs, gs::TEXT FROM generate_series(1, 100) AS gs; - -DROP TABLE IF EXISTS public.t2; -CREATE TABLE public.t2(i INT PRIMARY KEY, f FLOAT); -INSERT INTO public.t2(i, f) SELECT gs, gs::FLOAT FROM generate_series(5, 104) AS gs; - -DROP TABLE IF EXISTS public.t3; -CREATE TABLE public.t3(t TEXT PRIMARY KEY, i INT UNIQUE NOT NULL); -INSERT INTO public.t3(t, i) SELECT gs::TEXT, gs FROM generate_series(10, 109) AS gs; diff --git a/tests/e2e/pg2ch/empty_keys/check_db_test.go b/tests/e2e/pg2ch/empty_keys/check_db_test.go deleted file mode 100644 index 917f5764..00000000 --- a/tests/e2e/pg2ch/empty_keys/check_db_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithPrefix("")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - //------------------------------------------------------------------------------------ - // start worker - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err := tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) -} diff --git a/tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql b/tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/empty_keys/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql b/tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql deleted file mode 100644 index f7e54d53..00000000 --- a/tests/e2e/pg2ch/empty_keys/dump/pg/dump.sql +++ /dev/null @@ -1,17 +0,0 @@ --- needs to be sure there is db1 -create table __test -( - id int, - val1 int, - val2 varchar -); - -insert into __test (id, val1, val2) -values (1, 1, 'some'), - (2, 1, 'string'), - (2, 2, 'values'), - (3, 4, 'values'), - (4, 3, 'here'), - (null, 3, 'here'), - (4, null, 'here'), - (4, 3, null) diff --git a/tests/e2e/pg2ch/replication/check_db_test.go b/tests/e2e/pg2ch/replication/check_db_test.go deleted file mode 100644 index fa78e5c7..00000000 --- a/tests/e2e/pg2ch/replication/check_db_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithPrefix("")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &Source) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // insert/update/delete several record - - rows, err := conn.Query(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (3, 3, 'c'), (4, 4, 'd'), (5, 5, 'e')") - require.NoError(t, err) - rows.Close() - - rows, err = conn.Query(context.Background(), "UPDATE __test SET val1=22 WHERE id=2;") - require.NoError(t, err) - rows.Close() - - rows, err = conn.Query(context.Background(), "DELETE FROM __test WHERE id=3;") - require.NoError(t, err) - rows.Close() - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) - - //------------------------------------------------------------------------------------ - // check DELETE + INSERT case - _, err = conn.Exec(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (10, 1, 'attempt1')") - require.NoError(t, err) - - tctx := context.Background() - tx, err := conn.Begin(tctx) - require.NoError(t, err) - _, err = tx.Exec(tctx, "DELETE FROM __test WHERE id = 10") - require.NoError(t, err) - _, err = tx.Exec(tctx, "INSERT INTO __test (id, val1, val2) VALUES (10, 2, 'attempt2')") - require.NoError(t, err) - require.NoError(t, tx.Commit(tctx)) - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) -} diff --git a/tests/e2e/pg2ch/replication/dump/ch/dump.sql b/tests/e2e/pg2ch/replication/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/replication/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/replication/dump/pg/dump.sql b/tests/e2e/pg2ch/replication/dump/pg/dump.sql deleted file mode 100644 index f4c3e888..00000000 --- a/tests/e2e/pg2ch/replication/dump/pg/dump.sql +++ /dev/null @@ -1,13 +0,0 @@ --- needs to be sure there is db1 -create table __test -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test (id, val1, val2) -values (1, 1, 'a'), - (2, 2, 'XcTIan6Sk2JTT98F41uOn9BVdIapLVCu1fOfbVu8GC0q6q8dGQoF7BQU4GiTlj5DgXnp0E9mJX5SwD2BCNWri6jvODz8Gp4AMgEUZxLOjjFmt1VkgPrU67YIrmNCwre1b0SNJ90mvU5yFOoF3FWB3U2uT04wonF4wuwSWrWY9SExpormD7KOuLLYAjaGTd0bWH6ttDoVQLRkFofUYMz5cLJcSntWdMAU872qudaMG624AwCec5sOLm9b6QhHY3eusgV9pGHbXm7XmI6RF7lqSVDzxGzvyahYNMvkc6Cf6ccFK3fFUFO3WZkY5fT1ad3QTIqsP8WmyZEzol4GAiuzZAHvB2szeq1keaSzEeSoI6YPJXFevyRFzlVGJN7OxErxHnYd8TPPOyhQI0PwpQ7MY1cX9cWiqrxTl8lcDp23kntMsbmouacyEsHeFkagozm8muqnEM4w3qQhXNIOkV8pkoD0s2rxo5tytlBbW0OpgnKp6UxLAp7QqfmWXcOLIePdL3bOVI2WJfBXrgsnfVlnNukoH22rn4Vb3pvcsIyT4x8loFZzeVmXfR4xLeT73Vs5KDYYOGZOWdzh5KVWdvGTcpVU2fSNYl1GeDps45o7mTj2ycllkewLbGD84QNVP67aDujad7gLmt8jYrzwxS04AX7k2tz7tBE4gEqOefBwXyCBy1t9j7vSA9tg8ZupGMsy0QNzw1vRCo5jmNt3f4AjwWqBGYIIjYaS27vZwKOGdTTEqpbebWW45sBkxe9DrvrDYUi11wLMtr1sxKNzvZgfS65ROvjdXYJfkVXWtiqo8jpwf1KNdvTDJscQUFgh9e9XfCMAZTUOoBtQmQhDVQe4CON8JGVm4pDnKf7acwhAzxZU8X7HZblEQeYCKIA07MalK4f0XBzEL5rHmhLOry1a6uPFmaqx2DAHPegthCqcvgeNCXA48nrXXwgG04TLvNU4Xk3Lwwhug24btNMauk5w0cYPMl0DZ3CmnMleYe2u0pndVLsOY1PlKOLs8nrZEp6VKXrb3ZdkcZZ6c9h88dXIAkrrGoHh5cB2RtCTyZyBS0Y8akHDODUVh7LIYkd9vjZ4W9sPqxxnbGQfYIMWCm7zGLbhhOrf8GBN1dBdQvEZYWOsqrvGd2z1C8WiGXvrTjdUXnudsT1XYCniHyqpAVPLyQGZ3CSWaswmOi1bjeDOSN2t3fH50pyznZPmFbJfL8R1QFV3mCPCxkKc4o3eI24hOkX4MPepi6HlBadwgFbY69KDjKs9fphhUA2SYxvHWr3igc5Wp9ZmyBW88c1BxykzK8xbJseGrdavV4uSl96L0GnSpRhbJuKfX1QUDU42yImShSgdyXVci4O3lXVrJYqFHFrTd2jl2spp3V2VJqu3noUxrFZVmBCPOvg3Mqx0uAefGXtBI3T9vNJSrgFVNO4xFOa03oOlG1bRvT1I4bk7sBBAiVyQ0c445CxVPhhUuExt44BocoXFUDYh6EZGEw0OU56znN7wWqUaegqZpOMtRYZk5MpSIFauHyDXIVv17A6OHTN1zsW5hHIiWdQ8g5T362HvHiMLH3IhK1yL4jf29V5GqkKMkMb7kKPWTEn6ICkJQ4CBZSSKbEQhDZZoch6LHvI4HbOAIM3aTLR8O9hPeudAPJ9OgzvlZhfVLlK4QJRb8ADYfYCI3AyZb4xF7mEUQLUbZ9EiIkfHNBl8fzzyqhMeTY6oxK4sAatyu0Ku67CgfJR4AxOLHUKd0vVTcQ4eswNVGBIapEKbMexGrmL4FtV0c5rcu5xa6PiEVDNLvkD5KcxMvxbgDPnxhunvW5c5aQeSuiHYOVkiURaTDnP4JIcgDwH4MpcJfZtbwZezcE5XJwVDDAzlACaLZV642JQdQ7VSXTdLuJfHNheAtnaTdLPLawjktf1JpMZU6DveZVUTGUcgvN1hbPBTgxRMIXy2sVJJPrFXv9pjRItkDw8ivGX6972kheAex0HZML789Ks2eG6mI9Gp1JN2lw4hc78YYwBvDyi2vLoDP9Vcn32Cd9Ca6Rq9Pmi5nbUXUqbi3QNqjo5W1h1ekjL6rSG9ExJtZLCR3jwfSn9gdemwiMRi7M6eCnyvlKzVtPxOYGA223k2wjynuWuGHUOT7TrQ42wmDjXMfp0mhbCJxsivHULCC81hAozkgd1BaNFJ4cIAH1BgJJvunlB7pAcnyDqvN2sBvupw9As8uLUB0ochRf5E9o2qrm3R7cGDTM6RpGJ5D4DO48BViras5HIIOAf5ebrsfBskkK9fHe3sRbI1miceFOfXKMAlt1gkUIX7I7givW1bRuiIz5QXunwS7GY8xjLIdHpSwF94zy1JFgZP5wgkJs9fpMbrrbdHi1rILa5Rl9AnmsFiq1jONgT5DoucvFJ0MyXM2UyvODEACRwFzSI0EFMqCTVVPZwxjl6XTYB064Pk6ZNF7Hkl1a7VieyPxNoYE6Ngik4lslJg80djZwNm3PXOHTAJHiG7hszqYD5lYnxtnqInF2NIWRFtVRXzR1eJpKP0tJzR4x5FOCYg0tNm57meCAIjwanu7fMBsbrqDOMM1txXOuxcR3S1ohi9JlRyWapfSjjbaByKP7AtCB55pUhVrY0asrInRIW8OUZH1ti9rj9eSVLORpw0Pa5wqNhcnqFMDJgw9vo721WkwGHEpETAX1Pk7GE8adIwClJIYm9zYDYofkvfhrIDtqFrvmEF3Rq5n5K4hbprEoHogKzHemGkBYw6luv2qfN2vQS4QQICwXranq0fUY25f6Uzuu1IHgho2cVHSsurt4y9BhB6s1ZMwGwymykpt0mVmXXbt13U482VW45umJGOWcieCi7TjqmrNhwgZyScviPwfVhlg9CG4SW2NKc3yp9PoB1t8ffXMJBKgEmZ7ODbZ3ya00TQmamoQ1hqeifsdh5Kgck5ZxiaTMmrhIKC7cKx83P1AnT2t3PgFVV466YG1hX7Shyc51ykA1PoGcK50Irh0zDoZpc941oQSsCHoHDFneg50dxJZUMO7KYY0kApEsbnkAnXH74giY7TW96f1uvpgpEGB2vscWoEKpeswScNaIPwJJCOzWUC5tsfbZSdQqLTOq26d2H0dKYbaxi3LZvxGFQs4PgMszQiglc3cprfpsKKJmwPXnKm1lw8XtfImvlZvbSv4XyAaoSPDbCBPnI0C3hDoMfEG89WkGi4maOxeVccRWnYR4pWJIlAKb5JbwiK4FhoXnSdk5WN8XaYiqhHtSqob8tMW89OfENwXgvEg3PMkscbP16Fk9YsXylW73JZJncFQYL5evKZv21YoUAxEohqIlbR7Qjda4XHfDaYohURcP51Bs4W2vlcJihCehZ4HGb5KiWwWq8CrzKqXoDxEgA8hKjYMSiTj8osUhM0kTMTk79LGErZ90mOj6BvPIsWYnHiy4AyHDzuh7DFejzMnWmx0gEI88pNn4zvuwAAaPn9TANmZmsTmPhtS7dIbMoXKC2kbryesKLPDkxjBQDRoHkbkHPuBYxOciKimIGf6irMhj06rAZLxNYftaujnwxE4EoerhLYuHk7K2FEFiGw49xv3Ytqw99UGmLBiRkxIE2LtXpcNzoxcsQWEFqSs0MLHUvkHEgVtuuSw014qjvHAdZcqDFqforUf8HPa5yp7kxI5umQVHaKQl08yEvvhF1mFXKdLFsMHt1GOUMqyxRveYbCGJEWfwfeYeweMC7GyhHRoInzfhmaBkdnq0d7u4YQQt3cz82PfxVE5z7sl4WirUm4m7CzGCWMfbjdl3aGPvD1x73zREaHQBnPpw5HAThR0uXuwZEbHeXzz8esCsjAxiYvyR2C8H3mS9q4M2J8hDQOFFQMutM15m6Eclh6LVwvl4n3HFhsfRBy2ZZyKDS30A93PQHijIdp8J2KRN4ntTTBbEchsCm1Bvub58l7vhdxZTJWnN8VFIqlJhjNzvws4qeLXFdavHDvpW85rEmdnm624EkGMKb0sP9OinlKujpg48e1jBEuojxDNbklBcSaIiQNRGcHKezAe414KOlImg2TNbMAb9Y6nhbIb5SiMcgRYh5TAJMky7dlVJiMcTjzJ85hkzd961igKU81bB9Vecuj6cPQDqyjDKaPTxZMUMUluVcBGPHSVdiH7v4z967MBUaBPLSquVwPvxlt2lhN57vCukko6QVZkpKwbm1AM1KNCytRYe1S7lreye6Wwb0lrYma97rySUMbJQgucxONLkTgINxWrLfYSEF0QHxUL4SAatew6PGaxHccNXuQ2Tr2LcLSHgpvwdM32Axe7pvb1nBLvVO7MyweIH1NN089GhFUxUGl9Pcnax13GpZyjG8Bz58cynLQAz5OyshIbsRy6893aBOiYt5Fj8AEHjld5spPdHrEl6ec9O5o6n5hDx9EdjTuJIL4csC4taQqfjinqW9BuFrBoYGO2KmhjjQGLAvu6F0zTtSDLPvxWipTJU8ltiYJo0BsUQVfihyHGUEDWfNgnjtKosRydmLuQypdRNiYhBSajqGupS7jj5brvbrmJFuesbitd5qKIRBrAd2wTPzUOPre5WQziMK4dobCjffZlQualudKv60iz4aqE5NbGMgW8OAXTzN6MaHpaGpls6QNcnrgIhexb1E2jf1bDbVsbm6QK4CqOdwonbp8WZtEWzzbCFiUdwj0DfS880RtDYrQyNUBidXcgpKTEOpWK0Q9y9lJfUffREZKoiV1PPRYPjvCLBlqZ4YKbtxEo6DgjPnNFg4J0gHVa4fv3bATVmf2wK8wnjLo7sj29FsXOpKvGCRQpR4aBOzDdAGFJxOMO8Mj1UJTmRChf0TL1GxioCpkZrWRiqx8B8nVKTbS44KrIxqAc7vZIZLnMndSMWHI8KYzODdfZ5SDMBTTAJdPIgk2oOaeZ7drz8ho4N45vF2EfBd9l2YYxo7yOYv9j8rk4SWBbbmQMey5uy7dAHd7mUCFM2OH0sMi8AMT9ffGxonnizZf7qdoUA1okdUKiCW9lIo5CWn4ZlwizP4Li1Z0TQwqC6nW2e8nyMvePQBbMiEIaRc0K4LQGFr7PX3XoZ2BYI5VW5jHaoCzq5FbjLmx1HyiVkVdCHjxrn33CCntzp7ayMxatewEubeBTO0AbdnFqAg38rcblEppRCTz02O1un2BUKYI8MU0jyjaRLMvskhqKiNG1xA6K4QGPCBfAbHfejmonuG1IrVdm7HQWlAew2cxOUgi0NEsABlwuC0jVrHIq6RBu4I0EkY77J6zytmQNXYcqlLRVnsChKOmWsDv8xEhkbfQGsAAo9OB0oZoW5e0fIWz9DvA8RmBdg59Oxps8IB6g4sr111RrNiV11ilIDoUg8AV4uGGI80ANcpIEX9G4cFuY2Ny4uBqXVR8O7KQo3ICFHbIBwRsXNclcRP6m5nymyOFvICqq7h6x7O71jMAdmCBxmTP7g6mu5CV7riPLiqh1PBEWYncSztU4Q5TUloaQshdLImc52lOblcHkQJMhMbGKtYueXrPH0FPN1zGv0g7lkA29jNAigcWTEqVljSNbTlESpo6Gaf1zoYsiyDFS1fjoU5AO1Stb0SqhvqtYtIbxDKQAuNWavYJGd0A7wcBCMIQHmye7rgYaNYMimQymPIayusvgzL0f9zpLtEiRKLGMJY92F4BHBzKXQK6tJvxLV9uSeJcdDoLJPcNi68fdFUcrufAHIzEajDjlUrh5X3nETxdgyU3L4Yp5kUYfm9YTBCUYMZovEDbJRG2zYQHg36JtR6YyztyCzokTJXHmnT8GJPQVuJSl35IO7tgKERO3Guwy6cTtvr8aoSZk5XBubN7ty9URnNEfegkK2cXv3irpUfGqtlvFlk0daKQSXO99V3OPhj95GdZfeDXWyqOT806adHTqbeRIRR9bbDUW3ZDVf7IzExpA28JrQOE3rrgk3dGF4n5wisgNMVNSWwhpRSU0OZcNFSw0ZqtSz9XoPa4imdBe2WKvoSyUwYLGjbXNsvNd0rLeItBhNRxhy6tMwQqRaIdN6yGz04VFMsGvJOMenAgt5XR0EzQEt2LS6zpgT9FaBz9MRdIMshZUs5Tki4y1aqDTI479IDFfB8JFslcaGl6XKswef0xt3S74ufccCpwsu9ksn8cGcRemMYmnas3ObMTQVjyF7WKPizJJAsJj43rri51EnGH0k8fDKwWyAegutZgWsy9HUchQ0RuZSYI4Ect8OL29zGKiCtHIJv041TRcYxnConTY8jaPco13gock3zw3xb5khJQBe9AOG9OOOcgEBwjnmgI6S6fSOB5CSLaulZUTF00KbTvU0M4omiuUFMH93kU1JQQ7KIIjjjziUYebG0O19KopV4oyir16Saoyw9gpLChGEeIGmobSBpOmfivFlUBlkun7iloLaTqLOaBjAaJxxKEwHBwXHO9QH6Fp1gugBP77YPVIzIETaBtRSYLKL1t8s70NZeAzWJIk8jcBHbzhISSyTLfD8vmkGZwQNSQdI2BAxixA6MfPFeppv3NqSN6DcNkQVYOhocKa3kRnv7nc1gctNaYrMO113wbhlTLzEc7Ji4yRge7rJ2rWZcDjLYEWhZCwwZU4U1ARQqZJ3g4v5Z99W3ni0YnPuhpyGd929J4Ap8gikJLF7oYCaFrZ9oMbME1cLtw6GIIyfpSfUM6CfZAKXFl6TY7hepkrTXacYLFAMEde52YeNZ32J6pdR6otgrrhkpnPtXjI5voNu3YgwCeZoK6KZoc8kJ17P5rPTqqKxNTmS0rUI9l9CIL5DunJBdsWetHQHWf6LwThz671AgogPllGhShafHUFYFpRM1mNVIZC2LAwLwEqVW5G0YLXcW358kYXxzZ4XRvDcQfxtXqWyw9sM4j0z63daSxZrI3f0GljKdFe9GLBrYrj3deNeyqqsdTFTUVoNHjOoRBdNFHM0uuOK2JvBh0elBiTKPfcFXrUL6iSDBcEjrKTp354zeK6YmGHLfPYcLDtE3lpHsdjQncoXQox9C96X65RWqAZ29GPGS7lAAmUgKgvY9c64LHr56jAzBIIpDpabNTh0COMJhFvybmqkSV7oSkEEZeY1GCZDbhRuPUrWIahI6YwcM4gZgOSSwwUdbyaQjO2ynZffX3dZi5U9WtHGmHQNwJlUlaheo5ZPRcgcopnbxxwKSlA442obfGBCj1EkTjlwCMF9l7UIqdDSeRsT4D0QQpJrUG9AoNujQWSOUtW8lehlUJekbQqWTTfGvCiJeXpVqL4qHI2nstv4ttE3X0W8DtIcMfCSAeKpam1KDzyKOud8t89RfikSX7Q80xKYxgcFaSPqtfGbbGGc58FGi3BkW7DHHkkLRIufLJ33RvUt7ZgZmM23uBnqBRYp53zXbuRfSrAcsf3GMyWnqEfmty4Wx6diCyOnUP7xsUKIbwBcZWLuFVPTQ4rT7BXcghbsOca9jdUMQ0TGRhrTj5oDl5apYRbtAuddOjmF4XqUOHVQYAaL1yicIrdUqjZx5rbCbCL9bw3kz08lXh868vyIqnQQhKBSjhboppEMa7UfJBYWU5VKuQwFreuaYphUjE5xutjeuBNoanSqWNLu9AaeKcg7DGkKFmFsmySTsgGq48eAi5XIA1gQ1oqlWhOEeppUc4Y2R5UZuyAPBcmKCJ1BNMlRwPYO5iIdAvG3z6Xj19YxUaRvwFGtA6WLt8eUtMgzC2cNgIGLVDGWTF8ssd3X5FXyTSs3pOPpvo8BYGvo2bKqBK8zkaFZ46nCiBA3rkv5PIOwouUuRvcvuOTqqNb1mmcNB9f1yJxylO0ZJQN7h2gGyeKZPycjAHBmJb00g8NL3FcDbWwara17CjwoI1eqdLe1rIDR9IrjBcBEAbUJhExeIVacZgPQvOJeYZwgGiwZQAsBZMLyOA2sNH5EIt0suHLlsmXMSQFyDZb9I2vzozzpw1V80HPEQgrwYdiGyjRUFxm3ifuWGCicn9R9wDWHzsh2cSmIOzL7wyA1YKyLu8wA0UJfhDp0NFhCjxPHCK0etBkN0amvM2ikoczNanK7vJ37kGLnz8tBpc2n12CVZJc1qJnfVsitk9D6XDLXXQgOP6PoMZre2x5t7L2Y0cOlJoUzy1RjdvXucX9KypIQZ7CD9szNmCglwgxzIgrB2RqIEQWRQCkVuywUH7Z3p8CudyGHGDxs6fcOC9Wjy92D95RcNkZYZK1MWU1du7GGW6mSbvSVba3Faa74oBlxEm4RyC') - -- long string value in val2 - for TOAST testing. It should be random, bcs 'to TOAST or not to TOAST' decision happens after compression of values diff --git a/tests/e2e/pg2ch/replication_mv/check_db_test.go b/tests/e2e/pg2ch/replication_mv/check_db_test.go deleted file mode 100644 index bfa4518f..00000000 --- a/tests/e2e/pg2ch/replication_mv/check_db_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithPrefix("")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - Target.Cleanup = server.DisabledCleanup - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &Source) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // start worker - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - err = tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.Error(t, err) // MatView failed - - Target.InsertParams = model.InsertParams{MaterializedViewsIgnoreErrors: true} - err = tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // insert/update/delete several record - - rows, err := conn.Query(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (3, 3, 'c'), (4, 4, 'd'), (5, 5, 'e')") - require.NoError(t, err) - rows.Close() - - rows, err = conn.Query(context.Background(), "UPDATE __test SET val1=22 WHERE id=2;") - require.NoError(t, err) - rows.Close() - - rows, err = conn.Query(context.Background(), "DELETE FROM __test WHERE id=3;") - require.NoError(t, err) - rows.Close() - - //------------------------------------------------------------------------------------ - // wait & compare - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) - - //------------------------------------------------------------------------------------ - // check DELETE + INSERT case - _, err = conn.Exec(context.Background(), "INSERT INTO __test (id, val1, val2) VALUES (10, 1, 'attempt1')") - require.NoError(t, err) - - tctx := context.Background() - tx, err := conn.Begin(tctx) - require.NoError(t, err) - _, err = tx.Exec(tctx, "DELETE FROM __test WHERE id = 10") - require.NoError(t, err) - _, err = tx.Exec(tctx, "INSERT INTO __test (id, val1, val2) VALUES (10, 2, 'attempt2')") - require.NoError(t, err) - require.NoError(t, tx.Commit(tctx)) - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) -} diff --git a/tests/e2e/pg2ch/replication_mv/dump/ch/dump.sql b/tests/e2e/pg2ch/replication_mv/dump/ch/dump.sql deleted file mode 100644 index fd91c181..00000000 --- a/tests/e2e/pg2ch/replication_mv/dump/ch/dump.sql +++ /dev/null @@ -1,30 +0,0 @@ -CREATE DATABASE public; - -CREATE TABLE IF NOT EXISTS public.__test -( - `id` Int32, - `val1` Nullable(Int32), - `val2` Nullable(String), - `__data_transfer_commit_time` UInt64, - `__data_transfer_delete_time` UInt64 - ) - ENGINE = ReplacingMergeTree(__data_transfer_commit_time) - ORDER BY id; - -CREATE TABLE public.__test_aggr -( - `is_even` Int8, - `sumVal` UInt64 -) - ENGINE = SummingMergeTree() -ORDER BY (is_even); - -CREATE MATERIALIZED VIEW public.__test_mv -TO public.__test_aggr -AS -SELECT - coalesce(val1 / 2, 0) is_even, - sum(val1) AS sumVal -- at replication we will try to insert null, it should fail sum -FROM public.__test -GROUP BY - is_even; diff --git a/tests/e2e/pg2ch/replication_mv/dump/pg/dump.sql b/tests/e2e/pg2ch/replication_mv/dump/pg/dump.sql deleted file mode 100644 index 46012bf0..00000000 --- a/tests/e2e/pg2ch/replication_mv/dump/pg/dump.sql +++ /dev/null @@ -1,14 +0,0 @@ --- needs to be sure there is db1 -create table __test -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test (id, val1, val2) -values (1, 1, 'a'), - (2, 2, 'XcTIan6Sk2JTT98F41uOn9BVdIapLVCu1fOfbVu8GC0q6q8dGQoF7BQU4GiTlj5DgXnp0E9mJX5SwD2BCNWri6jvODz8Gp4AMgEUZxLOjjFmt1VkgPrU67YIrmNCwre1b0SNJ90mvU5yFOoF3FWB3U2uT04wonF4wuwSWrWY9SExpormD7KOuLLYAjaGTd0bWH6ttDoVQLRkFofUYMz5cLJcSntWdMAU872qudaMG624AwCec5sOLm9b6QhHY3eusgV9pGHbXm7XmI6RF7lqSVDzxGzvyahYNMvkc6Cf6ccFK3fFUFO3WZkY5fT1ad3QTIqsP8WmyZEzol4GAiuzZAHvB2szeq1keaSzEeSoI6YPJXFevyRFzlVGJN7OxErxHnYd8TPPOyhQI0PwpQ7MY1cX9cWiqrxTl8lcDp23kntMsbmouacyEsHeFkagozm8muqnEM4w3qQhXNIOkV8pkoD0s2rxo5tytlBbW0OpgnKp6UxLAp7QqfmWXcOLIePdL3bOVI2WJfBXrgsnfVlnNukoH22rn4Vb3pvcsIyT4x8loFZzeVmXfR4xLeT73Vs5KDYYOGZOWdzh5KVWdvGTcpVU2fSNYl1GeDps45o7mTj2ycllkewLbGD84QNVP67aDujad7gLmt8jYrzwxS04AX7k2tz7tBE4gEqOefBwXyCBy1t9j7vSA9tg8ZupGMsy0QNzw1vRCo5jmNt3f4AjwWqBGYIIjYaS27vZwKOGdTTEqpbebWW45sBkxe9DrvrDYUi11wLMtr1sxKNzvZgfS65ROvjdXYJfkVXWtiqo8jpwf1KNdvTDJscQUFgh9e9XfCMAZTUOoBtQmQhDVQe4CON8JGVm4pDnKf7acwhAzxZU8X7HZblEQeYCKIA07MalK4f0XBzEL5rHmhLOry1a6uPFmaqx2DAHPegthCqcvgeNCXA48nrXXwgG04TLvNU4Xk3Lwwhug24btNMauk5w0cYPMl0DZ3CmnMleYe2u0pndVLsOY1PlKOLs8nrZEp6VKXrb3ZdkcZZ6c9h88dXIAkrrGoHh5cB2RtCTyZyBS0Y8akHDODUVh7LIYkd9vjZ4W9sPqxxnbGQfYIMWCm7zGLbhhOrf8GBN1dBdQvEZYWOsqrvGd2z1C8WiGXvrTjdUXnudsT1XYCniHyqpAVPLyQGZ3CSWaswmOi1bjeDOSN2t3fH50pyznZPmFbJfL8R1QFV3mCPCxkKc4o3eI24hOkX4MPepi6HlBadwgFbY69KDjKs9fphhUA2SYxvHWr3igc5Wp9ZmyBW88c1BxykzK8xbJseGrdavV4uSl96L0GnSpRhbJuKfX1QUDU42yImShSgdyXVci4O3lXVrJYqFHFrTd2jl2spp3V2VJqu3noUxrFZVmBCPOvg3Mqx0uAefGXtBI3T9vNJSrgFVNO4xFOa03oOlG1bRvT1I4bk7sBBAiVyQ0c445CxVPhhUuExt44BocoXFUDYh6EZGEw0OU56znN7wWqUaegqZpOMtRYZk5MpSIFauHyDXIVv17A6OHTN1zsW5hHIiWdQ8g5T362HvHiMLH3IhK1yL4jf29V5GqkKMkMb7kKPWTEn6ICkJQ4CBZSSKbEQhDZZoch6LHvI4HbOAIM3aTLR8O9hPeudAPJ9OgzvlZhfVLlK4QJRb8ADYfYCI3AyZb4xF7mEUQLUbZ9EiIkfHNBl8fzzyqhMeTY6oxK4sAatyu0Ku67CgfJR4AxOLHUKd0vVTcQ4eswNVGBIapEKbMexGrmL4FtV0c5rcu5xa6PiEVDNLvkD5KcxMvxbgDPnxhunvW5c5aQeSuiHYOVkiURaTDnP4JIcgDwH4MpcJfZtbwZezcE5XJwVDDAzlACaLZV642JQdQ7VSXTdLuJfHNheAtnaTdLPLawjktf1JpMZU6DveZVUTGUcgvN1hbPBTgxRMIXy2sVJJPrFXv9pjRItkDw8ivGX6972kheAex0HZML789Ks2eG6mI9Gp1JN2lw4hc78YYwBvDyi2vLoDP9Vcn32Cd9Ca6Rq9Pmi5nbUXUqbi3QNqjo5W1h1ekjL6rSG9ExJtZLCR3jwfSn9gdemwiMRi7M6eCnyvlKzVtPxOYGA223k2wjynuWuGHUOT7TrQ42wmDjXMfp0mhbCJxsivHULCC81hAozkgd1BaNFJ4cIAH1BgJJvunlB7pAcnyDqvN2sBvupw9As8uLUB0ochRf5E9o2qrm3R7cGDTM6RpGJ5D4DO48BViras5HIIOAf5ebrsfBskkK9fHe3sRbI1miceFOfXKMAlt1gkUIX7I7givW1bRuiIz5QXunwS7GY8xjLIdHpSwF94zy1JFgZP5wgkJs9fpMbrrbdHi1rILa5Rl9AnmsFiq1jONgT5DoucvFJ0MyXM2UyvODEACRwFzSI0EFMqCTVVPZwxjl6XTYB064Pk6ZNF7Hkl1a7VieyPxNoYE6Ngik4lslJg80djZwNm3PXOHTAJHiG7hszqYD5lYnxtnqInF2NIWRFtVRXzR1eJpKP0tJzR4x5FOCYg0tNm57meCAIjwanu7fMBsbrqDOMM1txXOuxcR3S1ohi9JlRyWapfSjjbaByKP7AtCB55pUhVrY0asrInRIW8OUZH1ti9rj9eSVLORpw0Pa5wqNhcnqFMDJgw9vo721WkwGHEpETAX1Pk7GE8adIwClJIYm9zYDYofkvfhrIDtqFrvmEF3Rq5n5K4hbprEoHogKzHemGkBYw6luv2qfN2vQS4QQICwXranq0fUY25f6Uzuu1IHgho2cVHSsurt4y9BhB6s1ZMwGwymykpt0mVmXXbt13U482VW45umJGOWcieCi7TjqmrNhwgZyScviPwfVhlg9CG4SW2NKc3yp9PoB1t8ffXMJBKgEmZ7ODbZ3ya00TQmamoQ1hqeifsdh5Kgck5ZxiaTMmrhIKC7cKx83P1AnT2t3PgFVV466YG1hX7Shyc51ykA1PoGcK50Irh0zDoZpc941oQSsCHoHDFneg50dxJZUMO7KYY0kApEsbnkAnXH74giY7TW96f1uvpgpEGB2vscWoEKpeswScNaIPwJJCOzWUC5tsfbZSdQqLTOq26d2H0dKYbaxi3LZvxGFQs4PgMszQiglc3cprfpsKKJmwPXnKm1lw8XtfImvlZvbSv4XyAaoSPDbCBPnI0C3hDoMfEG89WkGi4maOxeVccRWnYR4pWJIlAKb5JbwiK4FhoXnSdk5WN8XaYiqhHtSqob8tMW89OfENwXgvEg3PMkscbP16Fk9YsXylW73JZJncFQYL5evKZv21YoUAxEohqIlbR7Qjda4XHfDaYohURcP51Bs4W2vlcJihCehZ4HGb5KiWwWq8CrzKqXoDxEgA8hKjYMSiTj8osUhM0kTMTk79LGErZ90mOj6BvPIsWYnHiy4AyHDzuh7DFejzMnWmx0gEI88pNn4zvuwAAaPn9TANmZmsTmPhtS7dIbMoXKC2kbryesKLPDkxjBQDRoHkbkHPuBYxOciKimIGf6irMhj06rAZLxNYftaujnwxE4EoerhLYuHk7K2FEFiGw49xv3Ytqw99UGmLBiRkxIE2LtXpcNzoxcsQWEFqSs0MLHUvkHEgVtuuSw014qjvHAdZcqDFqforUf8HPa5yp7kxI5umQVHaKQl08yEvvhF1mFXKdLFsMHt1GOUMqyxRveYbCGJEWfwfeYeweMC7GyhHRoInzfhmaBkdnq0d7u4YQQt3cz82PfxVE5z7sl4WirUm4m7CzGCWMfbjdl3aGPvD1x73zREaHQBnPpw5HAThR0uXuwZEbHeXzz8esCsjAxiYvyR2C8H3mS9q4M2J8hDQOFFQMutM15m6Eclh6LVwvl4n3HFhsfRBy2ZZyKDS30A93PQHijIdp8J2KRN4ntTTBbEchsCm1Bvub58l7vhdxZTJWnN8VFIqlJhjNzvws4qeLXFdavHDvpW85rEmdnm624EkGMKb0sP9OinlKujpg48e1jBEuojxDNbklBcSaIiQNRGcHKezAe414KOlImg2TNbMAb9Y6nhbIb5SiMcgRYh5TAJMky7dlVJiMcTjzJ85hkzd961igKU81bB9Vecuj6cPQDqyjDKaPTxZMUMUluVcBGPHSVdiH7v4z967MBUaBPLSquVwPvxlt2lhN57vCukko6QVZkpKwbm1AM1KNCytRYe1S7lreye6Wwb0lrYma97rySUMbJQgucxONLkTgINxWrLfYSEF0QHxUL4SAatew6PGaxHccNXuQ2Tr2LcLSHgpvwdM32Axe7pvb1nBLvVO7MyweIH1NN089GhFUxUGl9Pcnax13GpZyjG8Bz58cynLQAz5OyshIbsRy6893aBOiYt5Fj8AEHjld5spPdHrEl6ec9O5o6n5hDx9EdjTuJIL4csC4taQqfjinqW9BuFrBoYGO2KmhjjQGLAvu6F0zTtSDLPvxWipTJU8ltiYJo0BsUQVfihyHGUEDWfNgnjtKosRydmLuQypdRNiYhBSajqGupS7jj5brvbrmJFuesbitd5qKIRBrAd2wTPzUOPre5WQziMK4dobCjffZlQualudKv60iz4aqE5NbGMgW8OAXTzN6MaHpaGpls6QNcnrgIhexb1E2jf1bDbVsbm6QK4CqOdwonbp8WZtEWzzbCFiUdwj0DfS880RtDYrQyNUBidXcgpKTEOpWK0Q9y9lJfUffREZKoiV1PPRYPjvCLBlqZ4YKbtxEo6DgjPnNFg4J0gHVa4fv3bATVmf2wK8wnjLo7sj29FsXOpKvGCRQpR4aBOzDdAGFJxOMO8Mj1UJTmRChf0TL1GxioCpkZrWRiqx8B8nVKTbS44KrIxqAc7vZIZLnMndSMWHI8KYzODdfZ5SDMBTTAJdPIgk2oOaeZ7drz8ho4N45vF2EfBd9l2YYxo7yOYv9j8rk4SWBbbmQMey5uy7dAHd7mUCFM2OH0sMi8AMT9ffGxonnizZf7qdoUA1okdUKiCW9lIo5CWn4ZlwizP4Li1Z0TQwqC6nW2e8nyMvePQBbMiEIaRc0K4LQGFr7PX3XoZ2BYI5VW5jHaoCzq5FbjLmx1HyiVkVdCHjxrn33CCntzp7ayMxatewEubeBTO0AbdnFqAg38rcblEppRCTz02O1un2BUKYI8MU0jyjaRLMvskhqKiNG1xA6K4QGPCBfAbHfejmonuG1IrVdm7HQWlAew2cxOUgi0NEsABlwuC0jVrHIq6RBu4I0EkY77J6zytmQNXYcqlLRVnsChKOmWsDv8xEhkbfQGsAAo9OB0oZoW5e0fIWz9DvA8RmBdg59Oxps8IB6g4sr111RrNiV11ilIDoUg8AV4uGGI80ANcpIEX9G4cFuY2Ny4uBqXVR8O7KQo3ICFHbIBwRsXNclcRP6m5nymyOFvICqq7h6x7O71jMAdmCBxmTP7g6mu5CV7riPLiqh1PBEWYncSztU4Q5TUloaQshdLImc52lOblcHkQJMhMbGKtYueXrPH0FPN1zGv0g7lkA29jNAigcWTEqVljSNbTlESpo6Gaf1zoYsiyDFS1fjoU5AO1Stb0SqhvqtYtIbxDKQAuNWavYJGd0A7wcBCMIQHmye7rgYaNYMimQymPIayusvgzL0f9zpLtEiRKLGMJY92F4BHBzKXQK6tJvxLV9uSeJcdDoLJPcNi68fdFUcrufAHIzEajDjlUrh5X3nETxdgyU3L4Yp5kUYfm9YTBCUYMZovEDbJRG2zYQHg36JtR6YyztyCzokTJXHmnT8GJPQVuJSl35IO7tgKERO3Guwy6cTtvr8aoSZk5XBubN7ty9URnNEfegkK2cXv3irpUfGqtlvFlk0daKQSXO99V3OPhj95GdZfeDXWyqOT806adHTqbeRIRR9bbDUW3ZDVf7IzExpA28JrQOE3rrgk3dGF4n5wisgNMVNSWwhpRSU0OZcNFSw0ZqtSz9XoPa4imdBe2WKvoSyUwYLGjbXNsvNd0rLeItBhNRxhy6tMwQqRaIdN6yGz04VFMsGvJOMenAgt5XR0EzQEt2LS6zpgT9FaBz9MRdIMshZUs5Tki4y1aqDTI479IDFfB8JFslcaGl6XKswef0xt3S74ufccCpwsu9ksn8cGcRemMYmnas3ObMTQVjyF7WKPizJJAsJj43rri51EnGH0k8fDKwWyAegutZgWsy9HUchQ0RuZSYI4Ect8OL29zGKiCtHIJv041TRcYxnConTY8jaPco13gock3zw3xb5khJQBe9AOG9OOOcgEBwjnmgI6S6fSOB5CSLaulZUTF00KbTvU0M4omiuUFMH93kU1JQQ7KIIjjjziUYebG0O19KopV4oyir16Saoyw9gpLChGEeIGmobSBpOmfivFlUBlkun7iloLaTqLOaBjAaJxxKEwHBwXHO9QH6Fp1gugBP77YPVIzIETaBtRSYLKL1t8s70NZeAzWJIk8jcBHbzhISSyTLfD8vmkGZwQNSQdI2BAxixA6MfPFeppv3NqSN6DcNkQVYOhocKa3kRnv7nc1gctNaYrMO113wbhlTLzEc7Ji4yRge7rJ2rWZcDjLYEWhZCwwZU4U1ARQqZJ3g4v5Z99W3ni0YnPuhpyGd929J4Ap8gikJLF7oYCaFrZ9oMbME1cLtw6GIIyfpSfUM6CfZAKXFl6TY7hepkrTXacYLFAMEde52YeNZ32J6pdR6otgrrhkpnPtXjI5voNu3YgwCeZoK6KZoc8kJ17P5rPTqqKxNTmS0rUI9l9CIL5DunJBdsWetHQHWf6LwThz671AgogPllGhShafHUFYFpRM1mNVIZC2LAwLwEqVW5G0YLXcW358kYXxzZ4XRvDcQfxtXqWyw9sM4j0z63daSxZrI3f0GljKdFe9GLBrYrj3deNeyqqsdTFTUVoNHjOoRBdNFHM0uuOK2JvBh0elBiTKPfcFXrUL6iSDBcEjrKTp354zeK6YmGHLfPYcLDtE3lpHsdjQncoXQox9C96X65RWqAZ29GPGS7lAAmUgKgvY9c64LHr56jAzBIIpDpabNTh0COMJhFvybmqkSV7oSkEEZeY1GCZDbhRuPUrWIahI6YwcM4gZgOSSwwUdbyaQjO2ynZffX3dZi5U9WtHGmHQNwJlUlaheo5ZPRcgcopnbxxwKSlA442obfGBCj1EkTjlwCMF9l7UIqdDSeRsT4D0QQpJrUG9AoNujQWSOUtW8lehlUJekbQqWTTfGvCiJeXpVqL4qHI2nstv4ttE3X0W8DtIcMfCSAeKpam1KDzyKOud8t89RfikSX7Q80xKYxgcFaSPqtfGbbGGc58FGi3BkW7DHHkkLRIufLJ33RvUt7ZgZmM23uBnqBRYp53zXbuRfSrAcsf3GMyWnqEfmty4Wx6diCyOnUP7xsUKIbwBcZWLuFVPTQ4rT7BXcghbsOca9jdUMQ0TGRhrTj5oDl5apYRbtAuddOjmF4XqUOHVQYAaL1yicIrdUqjZx5rbCbCL9bw3kz08lXh868vyIqnQQhKBSjhboppEMa7UfJBYWU5VKuQwFreuaYphUjE5xutjeuBNoanSqWNLu9AaeKcg7DGkKFmFsmySTsgGq48eAi5XIA1gQ1oqlWhOEeppUc4Y2R5UZuyAPBcmKCJ1BNMlRwPYO5iIdAvG3z6Xj19YxUaRvwFGtA6WLt8eUtMgzC2cNgIGLVDGWTF8ssd3X5FXyTSs3pOPpvo8BYGvo2bKqBK8zkaFZ46nCiBA3rkv5PIOwouUuRvcvuOTqqNb1mmcNB9f1yJxylO0ZJQN7h2gGyeKZPycjAHBmJb00g8NL3FcDbWwara17CjwoI1eqdLe1rIDR9IrjBcBEAbUJhExeIVacZgPQvOJeYZwgGiwZQAsBZMLyOA2sNH5EIt0suHLlsmXMSQFyDZb9I2vzozzpw1V80HPEQgrwYdiGyjRUFxm3ifuWGCicn9R9wDWHzsh2cSmIOzL7wyA1YKyLu8wA0UJfhDp0NFhCjxPHCK0etBkN0amvM2ikoczNanK7vJ37kGLnz8tBpc2n12CVZJc1qJnfVsitk9D6XDLXXQgOP6PoMZre2x5t7L2Y0cOlJoUzy1RjdvXucX9KypIQZ7CD9szNmCglwgxzIgrB2RqIEQWRQCkVuywUH7Z3p8CudyGHGDxs6fcOC9Wjy92D95RcNkZYZK1MWU1du7GGW6mSbvSVba3Faa74oBlxEm4RyC'), - (15, null, 'b') - -- long string value in val2 - for TOAST testing. It should be random, bcs 'to TOAST or not to TOAST' decision happens after compression of values diff --git a/tests/e2e/pg2ch/snapshot/check_db_test.go b/tests/e2e/pg2ch/snapshot/check_db_test.go deleted file mode 100644 index 7fc7291d..00000000 --- a/tests/e2e/pg2ch/snapshot/check_db_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotOnly - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func testSnapshot(t *testing.T, source *postgres.PgSource, target model.ChDestination) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "CH target Native", Port: target.NativePort}, - helpers.LabeledPort{Label: "CH target HTTP", Port: target.HTTPPort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, source, &target, TransferType) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(client2.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, source, target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) -} - -func TestSnapshot(t *testing.T) { - target := Target - - testSnapshot(t, Source, target) -} diff --git a/tests/e2e/pg2ch/snapshot/dump/ch/dump.sql b/tests/e2e/pg2ch/snapshot/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/snapshot/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/snapshot/dump/pg/dump.sql b/tests/e2e/pg2ch/snapshot/dump/pg/dump.sql deleted file mode 100644 index e0b5db3c..00000000 --- a/tests/e2e/pg2ch/snapshot/dump/pg/dump.sql +++ /dev/null @@ -1,160 +0,0 @@ --- needs to be sure there is db1 -create table __test ( - id bigint not null, - aid serial, - - -- numeric - f float, - d double precision, - de decimal(10,2), --- ti tinyint, --- mi mediumint, - i int, - bi bigint, - biu bigint, - b bit(8), - - -- date time - da date, - ts timestamp, - dt timestamp, --- tm time, --- y year, - - -- strings - c char, - str varchar(256), - t text, --- bb blob, - - -- binary --- bin binary(10), --- vbin varbinary(100), - - -- other --- e enum ("e1", "e2"), --- se set('a', 'b', 'c'), --- j json, - primary key (aid, str, id) -- test multi pk and reverse order keys -); - -insert into __test values ( - 1, - 0, - 1.45e-10, - 3.14e-100, - 2.5, --- -124, -- ti --- 32765, -- mi - -8388605, - 2147483642, - 9223372036854775804, - - b'10101111', - - '2005-03-04', - now(), - now(), --- now(), --- '2099', -- year - - '1', - 'hello, friend of mine', - 'okay, now bye-bye' --- 'this it actually text but blob', -- blob --- 'a\0deadbeef', -- bin --- 'cafebabe', -- vbin --- "e1", -- e --- 'a', -- se --- '{"yandex is the best place to work at": ["wish i", "would stay", 4.15, {"here after":"the "}, ["i", ["n", ["t", "e r n s h i"], "p"]]]}' -) -, -( - 2, - 1, - 1.34e-10, - null, - null, --- -12, -- ti --- 1123, -- mi - -1294129412, - 112412412421941041, - 129491244912401240, - - b'10000001', - - '1999-03-04', - now(), - null, --- now(), --- '1971', -- year - - '2', - 'another hello', - 'okay, another bye' --- 'another blob', -- blob --- 'cafebabeda', -- bin --- '\0\0\0\0\1', -- vbin --- "e2", -- e --- 'b', -- se --- '{"simpler": ["than", 13e-10, {"it": {"could": "be"}}]}' -) -, -( - 3, - 4, - 5.34e-10, - null, - 123, --- -122, -- ti --- -1123, -- mi - 294129412, - -784124124219410491, - 129491098649360240, - - b'10000010', - - '1999-03-05', - null, - now(), --- now(), --- '1972', -- year - - 'c', - 'another another hello', - 'okay, another another bye' --- 'another another blob but looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg', -- blob --- 'caafebabee', -- bin --- '\0\0\0\0\1abcd124edb', -- vbin --- "e1", -- e --- 'c', -- se --- '{"simpler": ["than", 13e-10, {"it": {"could": ["be", "no", "ideas ", " again"], "sorry": null}}]}' -) -; - -insert into __test (str, id) values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) values ('nvaapsijfapfn', 201, now()), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But Ill be back, this is public promise', 422, now()), - ('Remember me, my name is hazzus', 333, now()); - -insert into __test (str, id, f, d, de) values ('100', 100, 'NaN'::real, 'NaN'::double precision, 'NaN'::numeric); -insert into __test (str, id, f, d) values - ('101', 101, '+Inf'::real, '+Inf'::double precision), - ('102', 102, '-Inf'::real, '-Inf'::double precision); diff --git a/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go b/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go deleted file mode 100644 index 8b40496e..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package replication - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/canon/postgres" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement -) - -func TestSnapshotAndIncrement(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga - - Source := pgrecipe.RecipeSource(pgrecipe.WithPrefix("")) - Target := chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - tableCase := func(tableName string) func(t *testing.T) { - return func(t *testing.T) { - tid, err := abstract.ParseTableID(tableName) - require.NoError(t, err) - conn, err := pgcommon.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), fmt.Sprintf(`drop table if exists %s`, tableName)) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), postgres.TableSQLs[tableName]) - require.NoError(t, err) - - transfer := helpers.MakeTransfer( - tableName, - Source, - Target, - abstract.TransferTypeSnapshotAndIncrement, - ) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{tableName}} - worker := helpers.Activate(t, transfer) - - conn, err = pgcommon.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), postgres.TableSQLs[tableName]) - require.NoError(t, err) - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, tid.Name, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator).WithPriorityComparators(pg2ch.ValueComparator))) - defer worker.Close(t) - } - } - // t.Run("array_types", tableCase("public.array_types")) - t.Run("date_types", tableCase("public.date_types")) - t.Run("geom_types", tableCase("public.geom_types")) - t.Run("numeric_types", tableCase("public.numeric_types")) - t.Run("text_types", tableCase("public.text_types")) - // t.Run("wtf_types", tableCase("public.wtf_types")) -} diff --git a/tests/e2e/pg2ch/snapshot_and_replication_canon_types/dump/ch/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_canon_types/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_canon_types/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/check_db_test.go b/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/check_db_test.go deleted file mode 100644 index f4a32f51..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/check_db_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - SourcePK = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithDBTables(`"public"."multiple_uniq_idxs_pk"`)) - SourceNoPK = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithDBTables(`"public"."multiple_uniq_idxs_no_complete"`)) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &SourcePK, &Target, TransferType) - helpers.InitSrcDst(helpers.TransferID, &SourceNoPK, &Target, TransferType) -} - -func TestSnapshotAndIncrementPK(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: SourcePK.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &SourcePK) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &SourcePK, &Target, TransferType) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - time.Sleep(5 * time.Second) // for the worker to start - - _, err = conn.Exec(context.Background(), "INSERT INTO multiple_uniq_idxs_pk(a, b, c_pk, t) VALUES (3, 50, 500, 'text_5')") - require.NoError(t, err) - _, err = conn.Exec(context.Background(), "UPDATE multiple_uniq_idxs_pk SET t = 'new_text_3' WHERE b = 30") - require.NoError(t, err) - _, err = conn.Exec(context.Background(), "DELETE FROM multiple_uniq_idxs_pk WHERE a = 1") - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "multiple_uniq_idxs_pk", helpers.GetSampleableStorageByModel(t, SourcePK), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, SourcePK, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) -} - -func TestSnapshotAndIncrementNoPK(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: SourceNoPK.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &SourceNoPK, &Target, TransferType) - - _, err := helpers.ActivateErr(transfer) - require.Error(t, err) -} diff --git a/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/ch/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/pg/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/pg/dump.sql deleted file mode 100644 index dbece285..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_multiple_unique_indexes/dump/pg/dump.sql +++ /dev/null @@ -1,26 +0,0 @@ -DROP TABLE IF EXISTS multiple_uniq_idxs_pk; -CREATE TABLE multiple_uniq_idxs_pk( - a INT, - b INT, - c_pk INT PRIMARY KEY, - t TEXT -); -INSERT INTO multiple_uniq_idxs_pk(a, b, c_pk, t) VALUES -(1, 10, 100, 'text_1'), -(2, 20, 200, 'text_2'); -CREATE UNIQUE INDEX ON multiple_uniq_idxs_pk (a) WHERE a IN (0, 2, 4, 6, 8, 10); -CREATE UNIQUE INDEX ON multiple_uniq_idxs_pk (b); -INSERT INTO multiple_uniq_idxs_pk(a, b, c_pk, t) VALUES -(3, 30, 300, 'text_3'), -(3, 40, 400, 'text_4'); - -DROP TABLE IF EXISTS multiple_uniq_idxs_no_complete; -CREATE TABLE multiple_uniq_idxs_no_complete( - a INT, - t TEXT -); -CREATE UNIQUE INDEX ON multiple_uniq_idxs_no_complete(a) WHERE a IN (0, 2, 4, 6, 8, 10); -CREATE UNIQUE INDEX ON multiple_uniq_idxs_no_complete(a) WHERE a IN (1, 3, 5, 7, 9, 11); -INSERT INTO multiple_uniq_idxs_no_complete(a, t) VALUES -(1, 'text_1'), -(2, 'text_2'); diff --git a/tests/e2e/pg2ch/snapshot_and_replication_special_values/check_db_test.go b/tests/e2e/pg2ch/snapshot_and_replication_special_values/check_db_test.go deleted file mode 100644 index 90fd9284..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_special_values/check_db_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump/pg")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &Source) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - time.Sleep(5 * time.Second) // for the worker to start - - _, err = conn.Exec(context.Background(), `INSERT INTO rsv_null_in_json(i, j, jb) VALUES (101, 'null', 'null'), (102, '"null"', '"null"')`) - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "rsv_null_in_json", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator).WithPriorityComparators(pg2ch.ValueJSONNullComparator))) -} diff --git a/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/ch/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/pg/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/pg/dump.sql deleted file mode 100644 index e9ec3fbe..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_special_values/dump/pg/dump.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE rsv_null_in_json( - i SERIAL PRIMARY KEY, - j json NOT NULL, - jb jsonb NOT NULL -); - -INSERT INTO rsv_null_in_json(i, j, jb) VALUES -(1, 'null', 'null'), -(2, '"null"', '"null"'); diff --git a/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/check_db_test.go b/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/check_db_test.go deleted file mode 100644 index 18040051..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/check_db_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump/pg")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - connConfig, err := pgcommon.MakeConnConfigFromSrc(logger.Log, &Source) - require.NoError(t, err) - conn, err := pgcommon.NewPgConnPool(connConfig, logger.Log) - require.NoError(t, err) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - time.Sleep(5 * time.Second) // for the worker to start - - _, err = conn.Exec(context.Background(), "INSERT INTO test (i1, t1, i2, t2, vc1) VALUES (3, '3', 3, 'c', '3'), (4, '4', 4, 'd', '4')") - require.NoError(t, err) - _, err = conn.Exec(context.Background(), "UPDATE test SET t2 = 'test_update' WHERE i1 = 1") - require.NoError(t, err) - _, err = conn.Exec(context.Background(), "DELETE FROM test WHERE i1 = 2") - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCount(t, databaseName, "test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) -} diff --git a/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/ch/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/pg/dump.sql b/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/pg/dump.sql deleted file mode 100644 index bcf7008c..00000000 --- a/tests/e2e/pg2ch/snapshot_and_replication_toast_multifield_pk/dump/pg/dump.sql +++ /dev/null @@ -1,17 +0,0 @@ --- needs to be sure there is db1 -create table test( - i1 INT, - t1 TEXT, - i2 INT, - t2 TEXT, - vc1 VARCHAR, - PRIMARY KEY (i1, t1) -); -CREATE UNIQUE INDEX test_ui2 ON test(i2); - -INSERT INTO test (i1, t1, i2, t2, vc1) -VALUES -(1, '1', 1, 'a', 'qp33K6PAgfb439v7l2KhtB7jSd1cxNQVo32bsVAzzxDkcuUvwyFgFM1tUh71EqvbIviHPx83gK0Xwj5yjHLpfmF6wP8v3ciqZ4GrYySegGqN8KWJ2mg80YYCcLEaTwKiZmTJnoRQjVu3ZilHNlbmhSaiHZY6AhnTZ0pijXLInVlWs4UkNGn1egDOVcRxDYCWLjfvRhJhdEohPFi7qX5b7pydZTd0TOFhJ3aPvXoLqCJEffmgFwvd70FKdw55ZMq3Gfm54jCaZPBUoEV3Xdx64xhguNWUPJwEJdiz4a41CGrt3rPvfFwlAQxmx47SPCK76ic6TTb3658BNtTdApYwFwONr4qr9jrJNNBfsPgUNIQv0X5hpw2e8Ru2LeV5LZgOfT7auH9BipqkqtoIg5XC7fgDqH8P1jIPr4jnAQfFWUMAWZTFhD8abvB4qsmde12zMzSFePrml42Jr7ksd2BRtIt2qkWCNvIC2i2SHV9qjEv9TxrFtTw8wikd7aQGmwILenWc54Ah638NLUoqMpExXbBfcwH8EFESSBR9y9zoDfTIq2nUsqQlUpa1RbTtHA2hI748FCtdbWAKpypc7mdjY8vu5xOlDxbsEb0KP9ADSmhDEzXHgzWZwWkuAoyhVRyxLfa3JHU6udVC1QCxv1FiVC3rfo3tKHQc0gXN27UvptM6geL28GhRhxVJvaYT9B8y7MQV6SmsdRej6BRIifr6ub756iDgEgBb8OBVWsSN597R1kNUpmrHvAn6tdDo5A1O5RaV31MdYGbJhCPt8MuGwPJEJsHvs035GrsA3wZpjacqFuWiz8epYmnZos1fFc7zSGcRtI9CC2gjDGl8kibx8veyueTjhPsuPaYf7MBdySwwiPNtGxvON8AqUb92v9zjkRp7eTOII5sNPthYBZVFcnqyhRPvPWL3WDfbh1a8M21MDZRrhyyaNeQd17nAwmF1dmfJ4tw7GxTFnQJsy1dCgG5M95VIytupJFXDG6x4txwStu1ozWZvSyyOLLiF0tRqEcoYsuVDnr0m4E2I74SpTdyE1CKMgsoggfs1RNz4Z2JwgExWHvKRcpe4dxczLfqUKY2tcHodT47DPj6MzjRFAB7qEm9JUiLFT3QMmWilHczzUgszoEdo7CgEx43oTBTaAUhOYGaHiMqP5FE4ZEzstuzEuE5HqkAGEJeHhxAN0nWZ5aWYK6uSh2agW3sOAqCoXYnmRmAFQ3ZloLrFSVw6fMWUpbaCCw9TKfUzFCq6ghIFQMX6YKp4rggudONTFv0FAd4ssW2Y8qk6IahvLTsDOorpazHr2meg5AvBHTg6gKbkc7lum8xwesau1tMVHhOWSjPkxngMYxLRnAe66WKQ7rs8YfsOO5nqGJtMnUVY5VmluD7wXoeLenIgFm51vifEPhjTyayKnRn7F5YhslMEAbSaHq2wzM6BmHjuEqdNmHtMvfqzVGy7uw1nDUOiHN4gHmqB2UObgITHiwBw5goLeR4lWL0I1uSgZnVBzKJTmuqj1Ed8uEcWv1jaf00C24eXrLg6Suw9NnHGtyi2ao91bXmNwpCTA5ayVDqgZqnrXRYZVDYZGmZuBv6dA3mYcwnJYACQZ797ZoGg5tPKFGjJ0k0rdBGjjLl9q7GfnH8Idk6jBa1h0BhfhRJKeM4qY3c16Cr9r2fiCTWvVpYjHPPulOBQJlmjKq3o7fSpVYsx9Wi6M0tzUXb3BT1mdQg1sGDezTVYmm8UJ59vYPrjyBPcz9tKNaZu5ioGCZy2Yys4clDUsJ52tGTUNTSJBWvOXMzdBGbgDz66yjpi9GaDmfczzJJBTLQ1R8hIp0FIw1MSfWrHFEVvSewrHTlu7kgI1Q8UFmjZVKdjgpaMlBZwrvPuPmrUpqucsFA3qhMDEGGJLjoDbLcg4VKvmssATZUZ8wUXMWD7oGDrvzn3h9nscg2jHENfk6IU6tUK5ZguZWSvMKNagjIv6IM5sBO6ZUybxerAA3WyxJ0zrJieG2KFzSnJykZyxnKKoQRRwwkTQpa87vdArPbrzn9i3fu4aPucmIEOacy3GJhwwdL799SSjbNOFLtlyktr4Tj1lzA02EYTjfDF9BIu6rHAlmuDAESJFqwloX3U4Ppo7kBjLi2Ab4xVzbd3F7i8roxLvSa81AKjyXRmZTBy9zhultAC1OjiMbHbXxcJAYbRKFWp3VcWHrNRJxhC1ZvHEVvNDIqQUENPfj6idtd4xoMqie7OtoAo7mm0SsS8UODLGRMKvn9eYG3UMSsy1Z275f55c56el8Ubhvd5Q2L78PIIGURIaBchjjYixBkkDr4OSbehnyJ6CqZzA6dwbBVy9J8WnX9sfFdQRobpmwJq4JUBqKSlZYXXQMDiNyRtmcZrc8aYUbqKZp4101fQCA0bjG8LNvMgdVTpneiy3srJtxSuEIju4nZgeVaEXVDhbDpZbFbXLXJEf1AjbbLiOP5nETplIVACq8TXJN1guecjMP0wCfmAujmX3n8jR0j1gJSyPCPmboy7lBNqVDWCnjfIkcZwrCoPWmzHwuSkbEhAHVeq8IjbJDaMPLY2hKatrYH9XmLKlRCBcGSlKKoBSdIZiUhPH6jmfC6vGPQ1nlFvrQRV6YeuqvGxMQp4wzkmrnNA9K65HDv8iWW2IT7ZP55bk6ULgKNjBeCSZDHOwYYYcO7wwvVqvr00zREt0hCjZKIw1HwxX83sPOTXGMsC2k1KpJcIqo9SHwXDmRV19ptYy4YVv9gUcMyLgybDNgJxsk5tXALK3Khr0IRpsyF4xnIsOza16b87ZXpTEMOSdnvAeexhNCHArRDJsJwCLkBz2ExDaMVSBlJkBqNsi1TpvV5ePnOjvCbx224xwx1lkdTRYt9sAPcpRXnFf6cfoZTv8ojAgHIHaKyV3m9PJBQkEDxetbxps2jyrqqVjewcUfBVSVQMiWpfFNAjMmJ53FdtfnBBUXjNnWmocGizNMdrMKfNYfgIuOURdkS4mTYSsMSfU55VQ8FZdfLSkrLdbW0WK961ntrKbVvXulIPwsIE1Ey4RNtuMxtMnURPEhDOY37TllZbz2C7hTFHvlJ4wMV9UG53fg9RhJGV8p3LrwXnP1qrB56seY300ZCuoPmvfJQ3xC7hgDqPcxIsIowgkunlmCMwhcptFFb7d0j2vARHguj7ZhTJUmZwhf1Est1duABSSZlJNA4xvdf90ASefmt9SrNczDzDOIQA9Ls8Bc7RJvorBx3IfhFSsFChqOo848x0jETtsOKXdl077DeuYx07EF1Yn4ZmkRZBmc3NbJxjDr80ayopEYS9fTgNL3N3ZTZkShNvkGa3JhkY6rONp4ukJR4j3IZHLArvKJjx2AX7iXCT1bEKhDIjSc9dJcWG233QW4W5VTXMfV0bR4RhNEuaEHp38N8Wxox8LHgghlosW8ACvwcMddbICUwzbPUPKn9upshTYgUyNHHA5Qal9Tsg7FCYyARjJpH4viJpZENZI0BhuFWpDJZcE54jGX9RohN2NBoa3uTVqEWt7IKE3p0alzono6xn7VsldO8Au7xF4zkPDf8XbYMr2WQvxIDoptbaMHV6uW7Q0EfckdIIxOvV6wJ4bTZFAAT8IcryEQ9uzYzoNAkjO7arIAhLViq1csqq8yPBz6y6l7o22HdqrCMmQCrnWq6ZCYWwZQbA79NuGsr18WGI6ddrC4bV89Lo1bvTRzswY9O2HqPFSsdNRS7Y0jzMJEeqZ82Fq6nx8osN2M5qxmCwjuyULXvEz7RUjTwtYf2JsZRNuDsePEwcuH4PocqRuSY3Llh4Z5U0Uc8dGMyzzxWVube0XU7ICKaBFQcJwLQ21mHHHqtSqlBiEF5a7MNobzwAiktb5KM0Y83OXFbAuiP4iHXUbJWOdsa07DcyCEw5hdcOdnsuytGTw9Uq9llfRWe6L7Kd1hvFoQQeO1VF6200NtwVfi1lvNFdwO6BuZC0o1H0ZrI3ug7aHMJfNPPVeguOC6pYSPkJs4dko1boWvtcHrJRq505e5GS1oGcEGhgp4hs1vOwscWZXXD4YeBDAzM3IrhJZ7rJGv5ZalMNo0hME4fe697yy6RFli5ExfbuCxJC3cicn5r7PXh5j5QRlaNGj99LXfNIKjZNXFonlEA36mtw54hb5IUxrQWZ4JuZcqdni67eHlp6aMGpFAXt3oB8gPKQ2ihRX2o9tQuxUF3mGDNEljKwwxWRpjJR2IbHTGEwtgkm3NEJqFFnG3F0Eup3xVAYFILrfQMQpCem8bkhcXBWy0SRREvNN6Pq4WtLZupB79b9qkByupVJLsORu6utAM6RiR7wyO5FaGCkqOEpAtF12cfuIjtz4WqlZT0UQcIjLM5hyaysPoENYH8xOtWi4Q8lqcLaBaP0rNiCWDX9aPA1brV5mVoCYZjsTyyxjrAknNCXQsZKGNOT5BLbdZhRHiAsu90eQV7KfIGYQGhwNzr4hsDv308RpqZknMQZy89OLvyyv8ix3u0eHH1epNjmq8e6354p7Av4oU3zOxpftmFdKpjDERjNhVmfPXVkVMG7gHxfDx8uaCx49LJYZ5HBnEIyYGsfjZg6INZsLMHnnqeDRGHdzkoKYuxfrd58HEVK9frLBQBGptiEQDAZRdSe1wAgGzTwu3f9qhGN4KgiDmA5QiS97O0TPXJGXFBu9Dl1Ysb3VhHOSsI6PdCu7cdsMwYpuCE6bG0soN4Ult4xWJu5KrmenZAEr0t5Ohr1q97EXl9H2TPy80TDHn2XNvaEWwTsGnvCtykbPvS9aBuOFyERTvYnrWRkYHvGNjKeUCveUzLa5h6tcopPB4SqGw1sh2UeWdVR4xQmPDb0ftMTnxZU1RSRidZIG47HSrf1Pu3LSQGVl9GKNMvHvESP7QRECD6VTN7bEYthMmibvoDoTCiHVd5PFoZylBXvwaFhbJdmXxWmIFvXFbki5D9YnEMlm6W6sU2HHQMNDgyJrx3QZhapJn8UoWJmiTyTWddEBsMdAurFJuEwd9EAx3mzeqEaD8gz4OfzynFRrWmlRYK7CFobrlUiu9n26BPCATs7bWelwPRedmoMHf0en0O712lzBBsBP3zmlaNkRFpmL4I10zvJiHdW12hR0RP9h4XPbNjBATA6fRKuYsY8MiAo5BHMo8bYkfVp9Nnc74VfjudCnl414e917GpBii2WiaQwXXaNg7z9XNuq7qwguEmC9slPKZQujmj2K2UGWKXrC1xGMyaMxXiaaXbXmB0prcg7zrBi45FHmotCZLdOTaSUMR52rq6VzMUuG1qHCHK912M2nI0ETHCZaw58uHGBvUxLIYzKulCjj22x5Q8KgvNXw11fS1GfHaDWLEe0WiaObgslZ21x6CnNhYlzhNxzqKfHdxJDR9sooPEdYsXDU9VZvX986ThgETeuCTn60W74aSzCR0bnUICAcwYteuUP5r02xB7XT0OAvhBsNJsuQRlkjIeb2StkpU2un8RExSe6zYoN9rD74foKkMAFKcBYu54XjpvUbYRnDGaA6rvTjqKkwLMS2nxqoLFlmBkH4hknZuJkX3JuyiW7pQ99KeIs8Ip7Tn98GUk65HTatRIviCP7gR2HZGWrT8GT6F03LKr2ikYYcuPS7etqO0oP5vfkWvxXVmrib50KjxvzcJebb4dVgesVvkQFdT6xLcA1maAhfl9yMsZLFK1JG5592gV6MlBf1kA8YOX8PNntGxcxgIRAqfL1wTuCWd77jqjbwuFOrbIbLAoKnV6WLKcRA5a8DQx3kDIOqDJjKZ5rShKkYper8biKbi0CLNhhEbam36Cnkq5fGFEaOpsGJy8kv4k7NFLG0g2CX6nOtfjMhB5MoQ4FR1BKcpgdrR0eHmz2nB2lDTg2LqqBKdU1nPrYaNvD0zdtO7X5se6hoU3w9Imwvc8e0hW5pDkSD4MYNG0nmMj1QbacYrOWBwXlgp3hiqIWwH7AtUXH2DY0uLRmWygvX4YMxCeCc9BgsnHyHKioTZUqAD7gn7Dtz2rwUO6UWaZ879mEyj8mcC0mWXdIfhZKHJqmq9snt4bMlryIrqLXiS5RTLPz0gDWGMjRNr7gzfDk3GesxYOcWf8EC9qwfBd43qDpACXgTD5RChw1uCZARuysOIepOrLsq1KJbFRDNACo8jLIRia3Fr87KrLS02TcHf2WEL2ukPJNP01GOsWUc0otOyDcfF6oJPJZacfvi14WJNrHkhjbt6rywaSrfdN3TktHUB7gKVDMlInv2dC3Z7QaFCvobJcDDZtOABxlWj8pIFEgk7J2mdUymqIA44OxL8hvMbUgOb1pZLfjlm6ksq12uV2IumijYfjrsuCZDTQzkSNdrAnbBHg5FyQf9p9jhgWvG0q7ByitunCsapM96NbY9vJgrpQlqViRQaFSfc2Bw7oVn63jhIsOmB4hvReI8Ae0IPububSvgVNvvj0w2AYkLAY4MzVhneLzlFKJkgsW4qg0Qo2A8V4tzH24cVogW8zYb0HwqdeNPGq9ggyn4cRCmcNnn5HP9W6PY8Cq00wxlMOqTVG19Ft1wtYiZgh6yJZuWcwNIOvQMU1A9aAY6nDSige5FV9oJNqiExpymfODKcD5pgufhAVqtgbiYnJQIohE9qYj2cgGctV7kZKVCczq249VRLc3UJcuYgLkLk8yBoJfXFVSqg5rNuSKnYtOgaTOBMCvxV5eAmrElTCmCvCtF9mwNHbndMfl4nz3F5HLufIiOhPD99fLEszxNwcRzmu5STrKgNgbGAw9xLkwargm5j5NAm6stNgnMcDA9O5TXxPKVd1wnX7zINb5WJpaSYezXTKOeWW5uoQ3pl2DFLAVU5oH8kpXGguSALcorrsdfCt7W4DobQtZSWkNLHXGBCptLc2maYToJqJJPn3FEKN1v9fnIvYnXglwv2QTVkRhSr3jtFcZTo8xqGfOeRRKC303oPl0iyGaZ8cg3PvtyQl4ucljvZXN1D1DthayynesxxmdvhBd0BYPaRFV5c8upPjIFOMac9QNQ1rZlazdB9dVYNMBLSpLK1TbtM9UhThl78YxF1k4P7cd8QbgYeDsuEDqaaeidZc8ZZ8y1OfawPOIYNdGhe0gTEJd8YlBZ02ujDYbCseVvWeTllCrF3SbuzKDjyKg7KxeCa9pqc8h1bI8qCR7UgQQZmACqxsVWzbf056KJC5iTrR7YxbZ5pAddjfJHHXPUpMBELXR3v0CllSh6un4vXoxsjTDtJf0w7LfRQF93QQX02FqAV2pBA6IsnMEDX3I9dXimN7CCEL2n3vYW7XR0fpAbPUoVwZUFWc0JHbDeiTjT2U0NGiu00Fl5TV4pojLmWMpLxsF83PkuSibbOtj2H3V9KZb0YqLUxhJJpTrzVtaTb9hxgR22Pxvz3ANAbYuy3OqoUq22ZIkgqYfzq2Yhl4924EhugkESwA4cNfO4wbdL1U34DXfKvi3JC4UEZzdnf9rPRNrhrqZzx61gTKE97AWzNNkAENEPChgAcnQ9cnTl3quc1nXOVeUlJdbkRgfS2z4aIMmnmvt2XhQAPfNXcmeBMfbfJBBNwupEN8TkvzLpqKuI5K5tc6l9ACLjYl3zBD6fEmAkn0cTKrbLXyqC1cFjVjt4bKJE0VtWNT80igu9EOV1eDuf1b7TaNG7DntsLPSoVSkCsqVfYZx2EzIMkBoRQAOa2ziw9Oj5nz6hvwKQ3S4ODj55Jlvvb03wCcifauZKCfk8q52cFYnYc1TGnKRJYphKzB1z2kCIBllggM9zAsGMchmmGIkA0KewxGoGMS0lHpb5Ywsd7JBP3eih9QojETdTT6jfW8fCOWyH3jbtQeMcONrnFFzxWyfxTUpPRZ2TWky6ebDTrsMqKj12l82ymWVG6KJKeKHzIaLpQOB0myEs7JiWNLq4ahmZP8kUr6Mu0rzf6kA0e6MQ4G4Aw4ExEvnbLfqRFnoB9Xyf4FT06Bdh3A9yFXxOyhiI4ekOtgeyLqT3BQ5E49re51NRTIfQsoXVLP4WDEuq8c9Mc6YVs2RTMzu3bkC65wuM4KCIz19jkVxHHGB0LH7BO8ix6iNRUoy85ZBVwcH1Q1dAVXTfLWAWJ59mqBJxHys2jgjSm3C0a7qgj6jO8JkWGhEwrYIjqrPrQVJsgnx5S1mwlmOouLpa78F2OBpz6mBtKdcuopaysd8HrDJHUH25Q19z9gcNPcrCbF5MFHlKRiIAnOykbSpwFvQhLje8lhkNUHsyQaqqGz6h1kAI7W5bHrZ58TH1DsaF9I1lChbJPO6E8lfUdKeq9SHasBy6bT6soXPZs0mQoa21oQruSbI7Skxkx4nvsJhI9lCrlgwcaf79VnIBY72GvQctnxTvR06Ti3Cucp2JYqJi0pQBTw4gUAKIpzbm0hDi0ZuAYwny8BKuWnV552nw4xwRgJXRgkctXvTUFmgWr3yfXRfgLRpKjWjp0fNhgClqGB3fcYfi9Ci2ZtKASK3SZfR21OwbAcukmw6cnYw05Eu2oATxr1IBrtMaSdIguHkvBwF97goV1XoOTOcYeq8vmR8FCgFPxcGgKWyHzQfVcYxLbYGdCqTmIsYIgo5Yyn744lVeJ7AkDg7AkajzKalO3oE7m2b6yqk1Tfq2PAdAcl7BX0NVQXxMpEgvedA1O95s5nsrxaBq0tLMf67V150ePxQzR8JgFvOsFJmajV0107PzVzGC6i9vcJXHUNHE1uk8rcutOWdq2qOmtfA8RDgY8rHnNvFVCHxdGMkDE5wD3LNU30aQZLY8MyZ6BQULoM4FbdKmsaRgoQ2oR4nav1CDQW08vWLtKz1JOdmwF9Vg3uGK5QgpopU9QR7t40Fj8m1spyvoGXFE5zpezZ3ziq6BkUL5wfxMYxojowpJDzZTVKsfO3SRKusUF9pP3wKu4NorKkegHW387nODyPTFUTwAmPHygKpFkQIYTdOskbBBzIMiRCiF5yTQ2iGVkcu4fs7BoDz2O6Qb2NZmmocqxGdst2e1IwJti40LQq0GoeFVf1O1vo0jP0R9VQh5jr9hOurRAPw59CDisZFGW8DJjNsiFi5joRdMHzN5yHTmiLRa2PB6xpOD1gQ0OVXtKWTBcA5ueQWmJeLrAsh20DwqhshdXl84R6pDGn1obRNZIarNT322ctyxQ4Y5H4cA3QruCBaeFsmlcxGY3bd8i7cbbvdo2xIIIVzKxQlzTPBdCIWRXuZcbL1MYLK99Odsw5WgEcvB5vaG4TJFSTtzfZXFy4H8ads'), -(2, '2', 2, 'b', 'XcTIan6Sk2JTT98F41uOn9BVdIapLVCu1fOfbVu8GC0q6q8dGQoF7BQU4GiTlj5DgXnp0E9mJX5SwD2BCNWri6jvODz8Gp4AMgEUZxLOjjFmt1VkgPrU67YIrmNCwre1b0SNJ90mvU5yFOoF3FWB3U2uT04wonF4wuwSWrWY9SExpormD7KOuLLYAjaGTd0bWH6ttDoVQLRkFofUYMz5cLJcSntWdMAU872qudaMG624AwCec5sOLm9b6QhHY3eusgV9pGHbXm7XmI6RF7lqSVDzxGzvyahYNMvkc6Cf6ccFK3fFUFO3WZkY5fT1ad3QTIqsP8WmyZEzol4GAiuzZAHvB2szeq1keaSzEeSoI6YPJXFevyRFzlVGJN7OxErxHnYd8TPPOyhQI0PwpQ7MY1cX9cWiqrxTl8lcDp23kntMsbmouacyEsHeFkagozm8muqnEM4w3qQhXNIOkV8pkoD0s2rxo5tytlBbW0OpgnKp6UxLAp7QqfmWXcOLIePdL3bOVI2WJfBXrgsnfVlnNukoH22rn4Vb3pvcsIyT4x8loFZzeVmXfR4xLeT73Vs5KDYYOGZOWdzh5KVWdvGTcpVU2fSNYl1GeDps45o7mTj2ycllkewLbGD84QNVP67aDujad7gLmt8jYrzwxS04AX7k2tz7tBE4gEqOefBwXyCBy1t9j7vSA9tg8ZupGMsy0QNzw1vRCo5jmNt3f4AjwWqBGYIIjYaS27vZwKOGdTTEqpbebWW45sBkxe9DrvrDYUi11wLMtr1sxKNzvZgfS65ROvjdXYJfkVXWtiqo8jpwf1KNdvTDJscQUFgh9e9XfCMAZTUOoBtQmQhDVQe4CON8JGVm4pDnKf7acwhAzxZU8X7HZblEQeYCKIA07MalK4f0XBzEL5rHmhLOry1a6uPFmaqx2DAHPegthCqcvgeNCXA48nrXXwgG04TLvNU4Xk3Lwwhug24btNMauk5w0cYPMl0DZ3CmnMleYe2u0pndVLsOY1PlKOLs8nrZEp6VKXrb3ZdkcZZ6c9h88dXIAkrrGoHh5cB2RtCTyZyBS0Y8akHDODUVh7LIYkd9vjZ4W9sPqxxnbGQfYIMWCm7zGLbhhOrf8GBN1dBdQvEZYWOsqrvGd2z1C8WiGXvrTjdUXnudsT1XYCniHyqpAVPLyQGZ3CSWaswmOi1bjeDOSN2t3fH50pyznZPmFbJfL8R1QFV3mCPCxkKc4o3eI24hOkX4MPepi6HlBadwgFbY69KDjKs9fphhUA2SYxvHWr3igc5Wp9ZmyBW88c1BxykzK8xbJseGrdavV4uSl96L0GnSpRhbJuKfX1QUDU42yImShSgdyXVci4O3lXVrJYqFHFrTd2jl2spp3V2VJqu3noUxrFZVmBCPOvg3Mqx0uAefGXtBI3T9vNJSrgFVNO4xFOa03oOlG1bRvT1I4bk7sBBAiVyQ0c445CxVPhhUuExt44BocoXFUDYh6EZGEw0OU56znN7wWqUaegqZpOMtRYZk5MpSIFauHyDXIVv17A6OHTN1zsW5hHIiWdQ8g5T362HvHiMLH3IhK1yL4jf29V5GqkKMkMb7kKPWTEn6ICkJQ4CBZSSKbEQhDZZoch6LHvI4HbOAIM3aTLR8O9hPeudAPJ9OgzvlZhfVLlK4QJRb8ADYfYCI3AyZb4xF7mEUQLUbZ9EiIkfHNBl8fzzyqhMeTY6oxK4sAatyu0Ku67CgfJR4AxOLHUKd0vVTcQ4eswNVGBIapEKbMexGrmL4FtV0c5rcu5xa6PiEVDNLvkD5KcxMvxbgDPnxhunvW5c5aQeSuiHYOVkiURaTDnP4JIcgDwH4MpcJfZtbwZezcE5XJwVDDAzlACaLZV642JQdQ7VSXTdLuJfHNheAtnaTdLPLawjktf1JpMZU6DveZVUTGUcgvN1hbPBTgxRMIXy2sVJJPrFXv9pjRItkDw8ivGX6972kheAex0HZML789Ks2eG6mI9Gp1JN2lw4hc78YYwBvDyi2vLoDP9Vcn32Cd9Ca6Rq9Pmi5nbUXUqbi3QNqjo5W1h1ekjL6rSG9ExJtZLCR3jwfSn9gdemwiMRi7M6eCnyvlKzVtPxOYGA223k2wjynuWuGHUOT7TrQ42wmDjXMfp0mhbCJxsivHULCC81hAozkgd1BaNFJ4cIAH1BgJJvunlB7pAcnyDqvN2sBvupw9As8uLUB0ochRf5E9o2qrm3R7cGDTM6RpGJ5D4DO48BViras5HIIOAf5ebrsfBskkK9fHe3sRbI1miceFOfXKMAlt1gkUIX7I7givW1bRuiIz5QXunwS7GY8xjLIdHpSwF94zy1JFgZP5wgkJs9fpMbrrbdHi1rILa5Rl9AnmsFiq1jONgT5DoucvFJ0MyXM2UyvODEACRwFzSI0EFMqCTVVPZwxjl6XTYB064Pk6ZNF7Hkl1a7VieyPxNoYE6Ngik4lslJg80djZwNm3PXOHTAJHiG7hszqYD5lYnxtnqInF2NIWRFtVRXzR1eJpKP0tJzR4x5FOCYg0tNm57meCAIjwanu7fMBsbrqDOMM1txXOuxcR3S1ohi9JlRyWapfSjjbaByKP7AtCB55pUhVrY0asrInRIW8OUZH1ti9rj9eSVLORpw0Pa5wqNhcnqFMDJgw9vo721WkwGHEpETAX1Pk7GE8adIwClJIYm9zYDYofkvfhrIDtqFrvmEF3Rq5n5K4hbprEoHogKzHemGkBYw6luv2qfN2vQS4QQICwXranq0fUY25f6Uzuu1IHgho2cVHSsurt4y9BhB6s1ZMwGwymykpt0mVmXXbt13U482VW45umJGOWcieCi7TjqmrNhwgZyScviPwfVhlg9CG4SW2NKc3yp9PoB1t8ffXMJBKgEmZ7ODbZ3ya00TQmamoQ1hqeifsdh5Kgck5ZxiaTMmrhIKC7cKx83P1AnT2t3PgFVV466YG1hX7Shyc51ykA1PoGcK50Irh0zDoZpc941oQSsCHoHDFneg50dxJZUMO7KYY0kApEsbnkAnXH74giY7TW96f1uvpgpEGB2vscWoEKpeswScNaIPwJJCOzWUC5tsfbZSdQqLTOq26d2H0dKYbaxi3LZvxGFQs4PgMszQiglc3cprfpsKKJmwPXnKm1lw8XtfImvlZvbSv4XyAaoSPDbCBPnI0C3hDoMfEG89WkGi4maOxeVccRWnYR4pWJIlAKb5JbwiK4FhoXnSdk5WN8XaYiqhHtSqob8tMW89OfENwXgvEg3PMkscbP16Fk9YsXylW73JZJncFQYL5evKZv21YoUAxEohqIlbR7Qjda4XHfDaYohURcP51Bs4W2vlcJihCehZ4HGb5KiWwWq8CrzKqXoDxEgA8hKjYMSiTj8osUhM0kTMTk79LGErZ90mOj6BvPIsWYnHiy4AyHDzuh7DFejzMnWmx0gEI88pNn4zvuwAAaPn9TANmZmsTmPhtS7dIbMoXKC2kbryesKLPDkxjBQDRoHkbkHPuBYxOciKimIGf6irMhj06rAZLxNYftaujnwxE4EoerhLYuHk7K2FEFiGw49xv3Ytqw99UGmLBiRkxIE2LtXpcNzoxcsQWEFqSs0MLHUvkHEgVtuuSw014qjvHAdZcqDFqforUf8HPa5yp7kxI5umQVHaKQl08yEvvhF1mFXKdLFsMHt1GOUMqyxRveYbCGJEWfwfeYeweMC7GyhHRoInzfhmaBkdnq0d7u4YQQt3cz82PfxVE5z7sl4WirUm4m7CzGCWMfbjdl3aGPvD1x73zREaHQBnPpw5HAThR0uXuwZEbHeXzz8esCsjAxiYvyR2C8H3mS9q4M2J8hDQOFFQMutM15m6Eclh6LVwvl4n3HFhsfRBy2ZZyKDS30A93PQHijIdp8J2KRN4ntTTBbEchsCm1Bvub58l7vhdxZTJWnN8VFIqlJhjNzvws4qeLXFdavHDvpW85rEmdnm624EkGMKb0sP9OinlKujpg48e1jBEuojxDNbklBcSaIiQNRGcHKezAe414KOlImg2TNbMAb9Y6nhbIb5SiMcgRYh5TAJMky7dlVJiMcTjzJ85hkzd961igKU81bB9Vecuj6cPQDqyjDKaPTxZMUMUluVcBGPHSVdiH7v4z967MBUaBPLSquVwPvxlt2lhN57vCukko6QVZkpKwbm1AM1KNCytRYe1S7lreye6Wwb0lrYma97rySUMbJQgucxONLkTgINxWrLfYSEF0QHxUL4SAatew6PGaxHccNXuQ2Tr2LcLSHgpvwdM32Axe7pvb1nBLvVO7MyweIH1NN089GhFUxUGl9Pcnax13GpZyjG8Bz58cynLQAz5OyshIbsRy6893aBOiYt5Fj8AEHjld5spPdHrEl6ec9O5o6n5hDx9EdjTuJIL4csC4taQqfjinqW9BuFrBoYGO2KmhjjQGLAvu6F0zTtSDLPvxWipTJU8ltiYJo0BsUQVfihyHGUEDWfNgnjtKosRydmLuQypdRNiYhBSajqGupS7jj5brvbrmJFuesbitd5qKIRBrAd2wTPzUOPre5WQziMK4dobCjffZlQualudKv60iz4aqE5NbGMgW8OAXTzN6MaHpaGpls6QNcnrgIhexb1E2jf1bDbVsbm6QK4CqOdwonbp8WZtEWzzbCFiUdwj0DfS880RtDYrQyNUBidXcgpKTEOpWK0Q9y9lJfUffREZKoiV1PPRYPjvCLBlqZ4YKbtxEo6DgjPnNFg4J0gHVa4fv3bATVmf2wK8wnjLo7sj29FsXOpKvGCRQpR4aBOzDdAGFJxOMO8Mj1UJTmRChf0TL1GxioCpkZrWRiqx8B8nVKTbS44KrIxqAc7vZIZLnMndSMWHI8KYzODdfZ5SDMBTTAJdPIgk2oOaeZ7drz8ho4N45vF2EfBd9l2YYxo7yOYv9j8rk4SWBbbmQMey5uy7dAHd7mUCFM2OH0sMi8AMT9ffGxonnizZf7qdoUA1okdUKiCW9lIo5CWn4ZlwizP4Li1Z0TQwqC6nW2e8nyMvePQBbMiEIaRc0K4LQGFr7PX3XoZ2BYI5VW5jHaoCzq5FbjLmx1HyiVkVdCHjxrn33CCntzp7ayMxatewEubeBTO0AbdnFqAg38rcblEppRCTz02O1un2BUKYI8MU0jyjaRLMvskhqKiNG1xA6K4QGPCBfAbHfejmonuG1IrVdm7HQWlAew2cxOUgi0NEsABlwuC0jVrHIq6RBu4I0EkY77J6zytmQNXYcqlLRVnsChKOmWsDv8xEhkbfQGsAAo9OB0oZoW5e0fIWz9DvA8RmBdg59Oxps8IB6g4sr111RrNiV11ilIDoUg8AV4uGGI80ANcpIEX9G4cFuY2Ny4uBqXVR8O7KQo3ICFHbIBwRsXNclcRP6m5nymyOFvICqq7h6x7O71jMAdmCBxmTP7g6mu5CV7riPLiqh1PBEWYncSztU4Q5TUloaQshdLImc52lOblcHkQJMhMbGKtYueXrPH0FPN1zGv0g7lkA29jNAigcWTEqVljSNbTlESpo6Gaf1zoYsiyDFS1fjoU5AO1Stb0SqhvqtYtIbxDKQAuNWavYJGd0A7wcBCMIQHmye7rgYaNYMimQymPIayusvgzL0f9zpLtEiRKLGMJY92F4BHBzKXQK6tJvxLV9uSeJcdDoLJPcNi68fdFUcrufAHIzEajDjlUrh5X3nETxdgyU3L4Yp5kUYfm9YTBCUYMZovEDbJRG2zYQHg36JtR6YyztyCzokTJXHmnT8GJPQVuJSl35IO7tgKERO3Guwy6cTtvr8aoSZk5XBubN7ty9URnNEfegkK2cXv3irpUfGqtlvFlk0daKQSXO99V3OPhj95GdZfeDXWyqOT806adHTqbeRIRR9bbDUW3ZDVf7IzExpA28JrQOE3rrgk3dGF4n5wisgNMVNSWwhpRSU0OZcNFSw0ZqtSz9XoPa4imdBe2WKvoSyUwYLGjbXNsvNd0rLeItBhNRxhy6tMwQqRaIdN6yGz04VFMsGvJOMenAgt5XR0EzQEt2LS6zpgT9FaBz9MRdIMshZUs5Tki4y1aqDTI479IDFfB8JFslcaGl6XKswef0xt3S74ufccCpwsu9ksn8cGcRemMYmnas3ObMTQVjyF7WKPizJJAsJj43rri51EnGH0k8fDKwWyAegutZgWsy9HUchQ0RuZSYI4Ect8OL29zGKiCtHIJv041TRcYxnConTY8jaPco13gock3zw3xb5khJQBe9AOG9OOOcgEBwjnmgI6S6fSOB5CSLaulZUTF00KbTvU0M4omiuUFMH93kU1JQQ7KIIjjjziUYebG0O19KopV4oyir16Saoyw9gpLChGEeIGmobSBpOmfivFlUBlkun7iloLaTqLOaBjAaJxxKEwHBwXHO9QH6Fp1gugBP77YPVIzIETaBtRSYLKL1t8s70NZeAzWJIk8jcBHbzhISSyTLfD8vmkGZwQNSQdI2BAxixA6MfPFeppv3NqSN6DcNkQVYOhocKa3kRnv7nc1gctNaYrMO113wbhlTLzEc7Ji4yRge7rJ2rWZcDjLYEWhZCwwZU4U1ARQqZJ3g4v5Z99W3ni0YnPuhpyGd929J4Ap8gikJLF7oYCaFrZ9oMbME1cLtw6GIIyfpSfUM6CfZAKXFl6TY7hepkrTXacYLFAMEde52YeNZ32J6pdR6otgrrhkpnPtXjI5voNu3YgwCeZoK6KZoc8kJ17P5rPTqqKxNTmS0rUI9l9CIL5DunJBdsWetHQHWf6LwThz671AgogPllGhShafHUFYFpRM1mNVIZC2LAwLwEqVW5G0YLXcW358kYXxzZ4XRvDcQfxtXqWyw9sM4j0z63daSxZrI3f0GljKdFe9GLBrYrj3deNeyqqsdTFTUVoNHjOoRBdNFHM0uuOK2JvBh0elBiTKPfcFXrUL6iSDBcEjrKTp354zeK6YmGHLfPYcLDtE3lpHsdjQncoXQox9C96X65RWqAZ29GPGS7lAAmUgKgvY9c64LHr56jAzBIIpDpabNTh0COMJhFvybmqkSV7oSkEEZeY1GCZDbhRuPUrWIahI6YwcM4gZgOSSwwUdbyaQjO2ynZffX3dZi5U9WtHGmHQNwJlUlaheo5ZPRcgcopnbxxwKSlA442obfGBCj1EkTjlwCMF9l7UIqdDSeRsT4D0QQpJrUG9AoNujQWSOUtW8lehlUJekbQqWTTfGvCiJeXpVqL4qHI2nstv4ttE3X0W8DtIcMfCSAeKpam1KDzyKOud8t89RfikSX7Q80xKYxgcFaSPqtfGbbGGc58FGi3BkW7DHHkkLRIufLJ33RvUt7ZgZmM23uBnqBRYp53zXbuRfSrAcsf3GMyWnqEfmty4Wx6diCyOnUP7xsUKIbwBcZWLuFVPTQ4rT7BXcghbsOca9jdUMQ0TGRhrTj5oDl5apYRbtAuddOjmF4XqUOHVQYAaL1yicIrdUqjZx5rbCbCL9bw3kz08lXh868vyIqnQQhKBSjhboppEMa7UfJBYWU5VKuQwFreuaYphUjE5xutjeuBNoanSqWNLu9AaeKcg7DGkKFmFsmySTsgGq48eAi5XIA1gQ1oqlWhOEeppUc4Y2R5UZuyAPBcmKCJ1BNMlRwPYO5iIdAvG3z6Xj19YxUaRvwFGtA6WLt8eUtMgzC2cNgIGLVDGWTF8ssd3X5FXyTSs3pOPpvo8BYGvo2bKqBK8zkaFZ46nCiBA3rkv5PIOwouUuRvcvuOTqqNb1mmcNB9f1yJxylO0ZJQN7h2gGyeKZPycjAHBmJb00g8NL3FcDbWwara17CjwoI1eqdLe1rIDR9IrjBcBEAbUJhExeIVacZgPQvOJeYZwgGiwZQAsBZMLyOA2sNH5EIt0suHLlsmXMSQFyDZb9I2vzozzpw1V80HPEQgrwYdiGyjRUFxm3ifuWGCicn9R9wDWHzsh2cSmIOzL7wyA1YKyLu8wA0UJfhDp0NFhCjxPHCK0etBkN0amvM2ikoczNanK7vJ37kGLnz8tBpc2n12CVZJc1qJnfVsitk9D6XDLXXQgOP6PoMZre2x5t7L2Y0cOlJoUzy1RjdvXucX9KypIQZ7CD9szNmCglwgxzIgrB2RqIEQWRQCkVuywUH7Z3p8CudyGHGDxs6fcOC9Wjy92D95RcNkZYZK1MWU1du7GGW6mSbvSVba3Faa74oBlxEm4RyC'); - --- long string is required for TOAST testing. It should be random, bcs 'to TOAST or not to TOAST' decision happens after compression of values diff --git a/tests/e2e/pg2ch/snapshottsv1/check_db_test.go b/tests/e2e/pg2ch/snapshottsv1/check_db_test.go deleted file mode 100644 index d7435b5a..00000000 --- a/tests/e2e/pg2ch/snapshottsv1/check_db_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package snapshottsv1 - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - client2 "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotOnly - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func testSnapshot(t *testing.T, source *postgres.PgSource, target model.ChDestination) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "CH target Native", Port: target.NativePort}, - helpers.LabeledPort{Label: "CH target HTTP", Port: target.HTTPPort}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, source, &target, TransferType) - transfer.TypeSystemVersion = 1 - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(client2.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.Background(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, source, target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparatorV1))) -} - -func TestSnapshot(t *testing.T) { - target := Target - - testSnapshot(t, Source, target) -} diff --git a/tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql b/tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/snapshottsv1/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql b/tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql deleted file mode 100644 index e0b5db3c..00000000 --- a/tests/e2e/pg2ch/snapshottsv1/dump/pg/dump.sql +++ /dev/null @@ -1,160 +0,0 @@ --- needs to be sure there is db1 -create table __test ( - id bigint not null, - aid serial, - - -- numeric - f float, - d double precision, - de decimal(10,2), --- ti tinyint, --- mi mediumint, - i int, - bi bigint, - biu bigint, - b bit(8), - - -- date time - da date, - ts timestamp, - dt timestamp, --- tm time, --- y year, - - -- strings - c char, - str varchar(256), - t text, --- bb blob, - - -- binary --- bin binary(10), --- vbin varbinary(100), - - -- other --- e enum ("e1", "e2"), --- se set('a', 'b', 'c'), --- j json, - primary key (aid, str, id) -- test multi pk and reverse order keys -); - -insert into __test values ( - 1, - 0, - 1.45e-10, - 3.14e-100, - 2.5, --- -124, -- ti --- 32765, -- mi - -8388605, - 2147483642, - 9223372036854775804, - - b'10101111', - - '2005-03-04', - now(), - now(), --- now(), --- '2099', -- year - - '1', - 'hello, friend of mine', - 'okay, now bye-bye' --- 'this it actually text but blob', -- blob --- 'a\0deadbeef', -- bin --- 'cafebabe', -- vbin --- "e1", -- e --- 'a', -- se --- '{"yandex is the best place to work at": ["wish i", "would stay", 4.15, {"here after":"the "}, ["i", ["n", ["t", "e r n s h i"], "p"]]]}' -) -, -( - 2, - 1, - 1.34e-10, - null, - null, --- -12, -- ti --- 1123, -- mi - -1294129412, - 112412412421941041, - 129491244912401240, - - b'10000001', - - '1999-03-04', - now(), - null, --- now(), --- '1971', -- year - - '2', - 'another hello', - 'okay, another bye' --- 'another blob', -- blob --- 'cafebabeda', -- bin --- '\0\0\0\0\1', -- vbin --- "e2", -- e --- 'b', -- se --- '{"simpler": ["than", 13e-10, {"it": {"could": "be"}}]}' -) -, -( - 3, - 4, - 5.34e-10, - null, - 123, --- -122, -- ti --- -1123, -- mi - 294129412, - -784124124219410491, - 129491098649360240, - - b'10000010', - - '1999-03-05', - null, - now(), --- now(), --- '1972', -- year - - 'c', - 'another another hello', - 'okay, another another bye' --- 'another another blob but looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg', -- blob --- 'caafebabee', -- bin --- '\0\0\0\0\1abcd124edb', -- vbin --- "e1", -- e --- 'c', -- se --- '{"simpler": ["than", 13e-10, {"it": {"could": ["be", "no", "ideas ", " again"], "sorry": null}}]}' -) -; - -insert into __test (str, id) values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) values ('nvaapsijfapfn', 201, now()), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But Ill be back, this is public promise', 422, now()), - ('Remember me, my name is hazzus', 333, now()); - -insert into __test (str, id, f, d, de) values ('100', 100, 'NaN'::real, 'NaN'::double precision, 'NaN'::numeric); -insert into __test (str, id, f, d) values - ('101', 101, '+Inf'::real, '+Inf'::double precision), - ('102', 102, '-Inf'::real, '-Inf'::double precision); diff --git a/tests/e2e/pg2ch/tables_inclusion/check_tables_inclusion_test.go b/tests/e2e/pg2ch/tables_inclusion/check_tables_inclusion_test.go deleted file mode 100644 index b10e1a74..00000000 --- a/tests/e2e/pg2ch/tables_inclusion/check_tables_inclusion_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package tables - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/clickhouse/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotOnly - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump/pg")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func testSnapshot(t *testing.T, source *postgres.PgSource, target model.ChDestination) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "CH target Native", Port: target.NativePort}, - helpers.LabeledPort{Label: "CH target HTTP", Port: target.HTTPPort}, - )) - }() - source.DBTables = []string{"public.__test_1", "public.__test_2", "public.__test_3"} - transfer := helpers.MakeTransfer(helpers.TransferID, source, &target, TransferType) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{"public.__test_1", "public.__test_2"}} - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) -} - -func TestSnapshot(t *testing.T) { - target := Target - - testSnapshot(t, Source, target) -} diff --git a/tests/e2e/pg2ch/tables_inclusion/dump/ch/dump.sql b/tests/e2e/pg2ch/tables_inclusion/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/tables_inclusion/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/tables_inclusion/dump/pg/dump.sql b/tests/e2e/pg2ch/tables_inclusion/dump/pg/dump.sql deleted file mode 100644 index 6e569c39..00000000 --- a/tests/e2e/pg2ch/tables_inclusion/dump/pg/dump.sql +++ /dev/null @@ -1,37 +0,0 @@ --- needs to be sure there is db1 -create table __test_1 -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test_1 (id, val1, val2) -values (1, 1, 'a'), - (2, 2, 'b'); - -create table __test_2 -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test_2 (id, val1, val2) -values (1, 2, 'b'), - (2, 2, 'c'); - -create table __test_3 -( - id int, - val1 int, - val2 varchar, - primary key (id) -); - -insert into __test_3 (id, val1, val2) -values (1, 3, 'c'), - (2, 2, 'd'); - diff --git a/tests/e2e/pg2ch/timestamp/check_db_test.go b/tests/e2e/pg2ch/timestamp/check_db_test.go deleted file mode 100644 index aafff348..00000000 --- a/tests/e2e/pg2ch/timestamp/check_db_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/e2e/pg2ch" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - databaseName = "public" - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump/pg"), pgrecipe.WithDBTables("public.__test")) - Target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - - Source.DBTables = []string{"public.__test"} - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - err := tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator))) -} diff --git a/tests/e2e/pg2ch/timestamp/dump/ch/dump.sql b/tests/e2e/pg2ch/timestamp/dump/ch/dump.sql deleted file mode 100644 index 5af5a873..00000000 --- a/tests/e2e/pg2ch/timestamp/dump/ch/dump.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE public; diff --git a/tests/e2e/pg2ch/timestamp/dump/pg/dump.sql b/tests/e2e/pg2ch/timestamp/dump/pg/dump.sql deleted file mode 100644 index 90186404..00000000 --- a/tests/e2e/pg2ch/timestamp/dump/pg/dump.sql +++ /dev/null @@ -1,8 +0,0 @@ -create table __test( - id serial primary key , - val text, - created_at timestamp default CURRENT_TIMESTAMP not null -); - -insert into __test (val) -values ('1'), ('2'), ('3'); diff --git a/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go b/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go deleted file mode 100644 index b81b4bdc..00000000 --- a/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go +++ /dev/null @@ -1,366 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - debeziumcommon "github.com/doublecloud/tross/pkg/debezium/common" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - "github.com/doublecloud/tross/pkg/debezium/testutil" - kafka2 "github.com/doublecloud/tross/pkg/providers/kafka" - "github.com/doublecloud/tross/pkg/providers/logbroker" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/tests/helpers" - "github.com/golang/mock/gomock" - "github.com/segmentio/kafka-go" - "github.com/stretchr/testify/require" -) - -var ( - Source = pgcommon.PgSource{ - Hosts: []string{"localhost"}, - User: os.Getenv("PG_LOCAL_USER"), - Password: server.SecretString(os.Getenv("PG_LOCAL_PASSWORD")), - Database: os.Getenv("PG_LOCAL_DATABASE"), - Port: helpers.GetIntFromEnv("PG_LOCAL_PORT"), - SlotID: "testslot", - } -) - -var testCases []debeziumcommon.KeyValue - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Source.WithDefaults() -} - -//--------------------------------------------------------------------------------------------------------------------- - -// fill 't' by giant random string -var update1Stmt = `UPDATE public.basic_types SET t = 'LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\u003c4SaNJTHkL@1?6YcDf\u003eHI[862bUb4gT@k\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^\u003e\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4\u003cI_@d]\u003eF1e]hj_XJII862[N\u003cj=bYA\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\u003e0UDDBb8h]65C\u003efC\u003c[02jRT]bJ\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\u003eH2X\\]_\u003cEE3@?U2_L67UV8FNQecS2Y=@6\u003ehb1\\3F66UE[W9\u003c]?HH\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJ\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\u003e@A\u003e5\u003cK\\d4QM:7:41B^_c\\FCI=\u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[T\u003cIQI4S_g\u003e;gf[BF_EN\u003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\u003c\u003e[=0W3Of;6;RFY=Q\\OK\\7[\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\u003cOK6gV=EMGC?\\F\u003cXaa_\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;\u003eMZG\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\u003e7cU]M[\u003c72c\u003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:\u003ea5a;j\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\u003c84==_9FJbjbEhQeOV\u003eWDP4MV^W1_]=TeAa66jLObKG\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\u003ebGaJ2f;VB\u003eG\\3\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\u003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\u003ek\u003ePUHD6\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\u003e=\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@f\u003ciDV?6i0WVXj\u003c@ZPd5d\\5B]O?7h=C=8O:L:IR8I\u003e^6\u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLE\u003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\u003e7UBbR58G?[X_O1b\\:[65\u003eP9Z6\u003c]S8=a\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\u003e^GMUPS2]b\u003e]DN?aUKNL^@RV\u003cFTBh:Q[Q3E5VHbK?5=RTKI\u003eggZZ\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\u003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[H\u003cUb4EE^\u003ckWO7\u003eR8fD9JQHR\u003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5\u003cBA\\3IVT@gG\\4\u003cRRS459YROd=_H1OM=a_hd\u003cSMLOd=S6^:eG\u003ejPgQ4_^d\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\u003cDa;\\Ni[AC\u003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9\u003eT12E?FZ9cYCLQbH[2O\u003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihE\u003ehMVaDF\u003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\u003eHga5eW[E8\u003c9jdYO7\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Q\u003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NR\u003eTK07=]7Ecdej\u003cUj\u003cDe1H\u003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_I\u003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\u003cSV@b[GVEU3Xh;R7\u003cXeTNgN\u003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhR\u003e]@GIYf[L55g\u003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\u003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\u003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9He\u003e1L[3\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\u003eCLdAe]6L2AD0aYHc5\u003e=fM7h\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]\u003eKE\u003cea\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\u003eLW5VIfJL:eQ4K3a1^WN5T=\\X=\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1C\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\u003eTQQWYJ_@FAX\\]9jh\u003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6\u003eBgES\u003e5EaeOFeG:i\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\u003eRQ]5Z9jA@Y9V1ZI6TDkC\u003eNZ_f_DR\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\u003eFW\u003cJ6X?IiJ\u003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;f\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHI\u003eI]gBS\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34\u003eh_2@i3kd02G\u003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\u003c^U7Hk]7Q6P:QZS;Ge@:\u003c\u003cfT6PK7j4?;cdC@c5GI:gS[W\u003cf26;\u003cBG7fMXFTWJcbB\\9QT\u003eh3HdV8Pb3Rh\u003e^?Ue:7RP[=jT4AE\u003ebiL_1dYW1\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\u003cA9LXQbECIc2M\u003c^I\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;c\u003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\u003c_F9P\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\u003e=R4U3W1G;\u003chN\\WFO_=DD\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5T\u003eY?bFOMZO\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\u003efaaP8P4KDVSCiQ=2\u003c=Ef:\u003eP\u003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_\u003c@5Z8fDPJAE8DcGUIb8C\u003c_L7XhP=\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\u003e]LW\u003ee^b\u003e?0G9Ie\u003c\u003c@UT4e9\u003cGM_jME7[6TFEN:\u003c\\H\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\u003cL42d\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\u003eEJQi8j;]L5CILgXdR_\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\u003cKHA:\\[CW7SRYVhE1[MD\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8k\u003eQb]UVVZ:18fe_8M?\\?\u003e\u003eLf4QSG@jO@\u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\u003cR]Ofg:TNGW0L\u003ePOC_CP\u003e^PI[aZ:KY^V@Q;;ME_k\\K0\u003eYP]1D5QSc51SfZ]FIP1Y6\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\u003e0]RJGeQiJG5\\=O8TRG5U\u003eLGa\u003eRi2K\u003c3=1TVHN=FhTJYajbIP\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\u003e93HU2ig?7\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1J\u003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^G\u003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1a\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\u003cA\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\u003e@^\u003e[4\u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVde\u003cUVVNH2EJ^=ALOFKUX:^\u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2S\u003cX?9bC7Ebc5V5E]\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SC\u003e\u003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\u003ceM8?j]NZai4\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\u003eI=?@f\u003cG349NMId8[T^@Sf\u003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@d\u003cc\u003cMhS3K;b\u003eZbHAf[GKME9igTY7iVFba\u003e4D;WFVb=dQ4Abj2\u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\u003e@TM9eO\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\u003c@=gPHLhQFDC@:T\u003cREdY\u003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1ad\u003cIiK1O7fbD[7[\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\u003e=FFMHPSBf8:\\XRZ91D:2D[1Y\u003eX\\bfj4BEQZe:1A\u003cQj^@7SAK]C_NCM\\0\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\u003e2\u003e4X:9JYPXk\u003eX_?;DAfL\u003ec?HF\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\u003e1\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aE\u003cY^MPd\u003e6M^iNNe=P6i6Lf::P6ebjX;\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\u003c93\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\u003c8@j5e\u003eVA76=g2=gD4V1eYF0bZd0EZ\u003cMk2M4g[Z=baJ]cVY\u003c[D=U2RUdBNdW=69=8UB4E1@\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;D\u003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4ia\u003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fa\u003e:Vj=BR7EW0_hV4=]DaSeQ\u003c?8]?9X4GbZF41h;FS\u003c9Pa=^SQT\u003cL:GAIP3XX[\\4RKJVLFabj20Oc\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\W\u003cHg9FWd\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\u003eS\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\u003e67\u003eW\u003cQNZRKDH@]_j^M_AV9g4\u003chIF\u003eaSDhbj9GMdjh=F=j:\u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaS\u003eO]caAKi\u003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\u003cWkC\u003c^KSgbI7?aGVaRkbA2?_Raf^\u003e9DID]07\u003cS431;BaRhX:hNJj]\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\u003cN?J\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\u003e8]\u003eU2:HGATaUBPG\u003c\\c0aX@_D;_EOK=]Sjk=1:VGK\u003e=4P^K\\OD\\D008D\u003cgY[GfMjeM\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNf\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\u003ceaB7BV;\u003eIRKK' WHERE i=1;` - -// TOASTed update -var update2Stmt = `UPDATE public.basic_types SET bl=false WHERE bl=true;` - -// update with pkey change -var update3Stmt = `UPDATE public.basic_types SET i=2 WHERE i=1;` -var deleteStmt = `DELETE FROM public.basic_types WHERE 1=1;` -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - -- 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -var canonizedDebeziumInsertK = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":1}}` -var canonizedDebeziumInsertV = `{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"before"},{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"fullfillment.public.basic_types.Envelope"},"payload":{"before":null,"after":{"bl":true,"b":true,"b8":"rw==","vb":"rg==","si":-32768,"ss":1,"int":-8388605,"aid":0,"id":1,"bid":3372036854775807,"oid_":null,"real_":1.45E-10,"d":3.14E-100,"c":"1","str":"varchar_example","character_":"abcd","character_varying_":"varc","timestamptz_":"2004-10-19T08:23:54Z","tst":"2004-10-19T09:23:54Z","timetz_":"08:51:02.746572Z","time_with_time_zone_":"08:51:02.746572Z","iv":90000000000,"ba":"yv66vg==","j":"{\"k1\": \"v1\"}","jb":"{\"k2\": \"v2\"}","x":"bar","uid":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","pt":{"x":23.4,"y":-44.5,"wkb":"AQEAAABmZmZmZmY3QAAAAAAAQEbA","srid":null},"it":"192.168.100.128/25","int4range_":"[3,7)","int8range_":"[3,7)","numrange_":"[1.9,1.91)","tsrange_":"[\"2010-01-02 10:00:00\",\"2010-01-02 11:00:00\")","tstzrange_":"[\"2010-01-01 06:00:00+00\",\"2010-01-01 10:00:00+00\")","daterange_":"[2000-01-10,2000-01-21)","f":1.45E-10,"i":1,"t":"text_example","date_":10599,"time_":14706000000,"time1":14706100,"time6":14706123456,"timetz__":"17:30:25Z","timetz1":"17:30:25.5Z","timetz6":"17:30:25.575401Z","timestamp1":1098181434900,"timestamp6":1098181434987654,"timestamp":1098181434000000,"numeric_":{"scale":0,"value":"EAAAAAAAAAAAAAAAAA=="},"numeric_5":"MDk=","numeric_5_2":"ME8=","decimal_":{"scale":0,"value":"AeJA"},"decimal_5":"MDk=","decimal_5_2":"ME8=","hstore_":"{\"a\":\"1\",\"b\":\"2\"}","inet_":"192.168.1.5","cidr_":"10.1.0.0/16","macaddr_":"08:00:2b:01:02:03","citext_":"Tom"},"source":{"version":"1.1.2.Final","connector":"postgresql","name":"fullfillment","ts_ms":1643136761176,"snapshot":"false","db":"pguser","schema":"public","table":"basic_types","txId":558,"lsn":24901344,"xmin":null},"op":"c","ts_ms":1643136761897,"transaction":null}}` -var canonizedDebeziumUpdate1K = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":1}}` -var canonizedDebeziumUpdate1V = `{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"before"},{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"fullfillment.public.basic_types.Envelope"},"payload":{"before":null,"after":{"bl":true,"b":true,"b8":"rw==","vb":"rg==","si":-32768,"ss":1,"int":-8388605,"aid":0,"id":1,"bid":3372036854775807,"oid_":null,"real_":1.45E-10,"d":3.14E-100,"c":"1","str":"varchar_example","character_":"abcd","character_varying_":"varc","timestamptz_":"2004-10-19T08:23:54Z","tst":"2004-10-19T09:23:54Z","timetz_":"08:51:02.746572Z","time_with_time_zone_":"08:51:02.746572Z","iv":90000000000,"ba":"yv66vg==","j":"{\"k1\": \"v1\"}","jb":"{\"k2\": \"v2\"}","x":"bar","uid":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","pt":{"x":23.4,"y":-44.5,"wkb":"AQEAAABmZmZmZmY3QAAAAAAAQEbA","srid":null},"it":"192.168.100.128/25","int4range_":"[3,7)","int8range_":"[3,7)","numrange_":"[1.9,1.91)","tsrange_":"[\"2010-01-02 10:00:00\",\"2010-01-02 11:00:00\")","tstzrange_":"[\"2010-01-01 06:00:00+00\",\"2010-01-01 10:00:00+00\")","daterange_":"[2000-01-10,2000-01-21)","f":1.45E-10,"i":1,"t":"LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\\u003c4SaNJTHkL@1?6YcDf\\u003eHI[862bUb4gT@k\\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\\\DEhJcS9^=Did^\\u003e\\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\\\8ii=aKaZVZ\\\\Ue_1?e_DEfG?f2AYeWIU_GS1\\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\\\XYTSG:?[VZ4E4\\u003cI_@d]\\u003eF1e]hj_XJII862[N\\u003cj=bYA\\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\\u003e0UDDBb8h]65C\\u003efC\\u003c[02jRT]bJ\\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\\\ALdBODQL729fBcY9;=bhjM8C\\\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\\u003eH2X\\\\]_\\u003cEE3@?U2_L67UV8FNQecS2Y=@6\\u003ehb1\\\\3F66UE[W9\\u003c]?HH\\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\\\cSEJL5M7\\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\\\]SJ?O=a]H:hL[4^EJacJ\\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\\u003e@A\\u003e5\\u003cK\\\\d4QM:7:41B^_c\\\\FCI=\\u003eOehJ7=[EBg3_dTB4[L7\\\\^ePVVfi48\\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\\\UT4Ie6YBd[T\\u003cIQI4S_g\\u003e;gf[BF_EN\\u003c68:QZ@?09jTEG:^K]QG0\\\\DfMVAAk_L6gA@M0P\\\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\\u003eBH_GUW3;h\\\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\\u003c\\u003e[=0W3Of;6;RFY=Q\\\\OK\\\\7[\\\\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\\u003cOK6gV=EMGC?\\\\F\\u003cXaa_\\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\\\QN=hE5WKY\\\\\\\\jVc6E;ZBbTX\\\\_1;\\u003eMZG\\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\\u003e7cU]M[\\u003c72c\\u003e3gSEdHc6\\\\@2CBI7T9=OGDG16d\\\\Bk^:\\u003ea5a;j\\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\\\75H=Z2QG\\\\eGQP1eUdgEM34?\\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\\u003c84==_9FJbjbEhQeOV\\u003eWDP4MV^W1_]=TeAa66jLObKG\\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\\\9@ILE68[MiF3c[?O8\\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\\u003ebGaJ2f;VB\\u003eG\\\\3\\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\\u003e3038WY6g@;\\\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\\u003ek\\u003ePUHD6\\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\\u003e=\\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\\\QYXCQSZDTFDd0J1JhDIi=@f\\u003ciDV?6i0WVXj\\u003c@ZPd5d\\\\5B]O?7h=C=8O:L:IR8I\\u003e^6\\u003ejFgN?1G05Y^ThdQ:=^B\\\\h^fGE3Taga_A]CP^ZPcHCLE\\u003c2OHa9]T49i7iRheH\\\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\\u003e7UBbR58G?[X_O1b\\\\:[65\\u003eP9Z6\\u003c]S8=a\\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\\u003e^GMUPS2]b\\u003e]DN?aUKNL^@RV\\u003cFTBh:Q[Q3E5VHbK?5=RTKI\\u003eggZZ\\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\\u003eSJd2@=U3GeKc\\\\NZaUeD7R@Kd6^1P=?8V8:fE[H\\u003cUb4EE^\\u003ckWO7\\u003eR8fD9JQHR\\u003cP\\\\7eQbA]L8aaNS2M@QTNF;V@O_[5\\u003cBA\\\\3IVT@gG\\\\4\\u003cRRS459YROd=_H1OM=a_hd\\u003cSMLOd=S6^:eG\\u003ejPgQ4_^d\\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\\u003ecPfK[\\\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\\u003cDa;\\\\Ni[AC\\u003eCVGc_\\\\_=1eeMj;TcOg:;8N1C?PAjaT=9\\u003eT12E?FZ9cYCLQbH[2O\\u003e4bMT8LJ[XSiAT0VI?18Hdb\\\\EHS]8UAFY8cB@C[k1CiBgihE\\u003ehMVaDF\\u003c\\\\iidT??BG6TWJDWJWU\\\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\\u003eHga5eW[E8\\u003c9jdYO7\\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\\\17IM?A7F3JBDcK25RIbjLHE^G0Q\\u003ceXie_FG3WNJZh[3;5e^O\\\\]k96]O7C\\\\00Yf5Bc\\\\BK]2NR\\u003eTK07=]7Ecdej\\u003cUj\\u003cDe1H\\u003ce91;U^=8DK\\\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\\\9Na1^d4YgDgdUS2_I\\u003c:c8^JIa]NEgU558f6f:S\\\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\\u003cSV@b[GVEU3Xh;R7\\u003cXeTNgN\\u003cdaBSW=3dY9WIOB^:EK6P2=\\\\Z7E=3cIgYZOFhR\\u003e]@GIYf[L55g\\u003cUiIFXP[eTSCPA23WjUf\\\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\\u003cK^_XXbkXaNB^JAHfkfjA\\\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\\u003eM934TQN3\\\\]k=Fk?W]Tg[_]JhcUW?b9He\\u003e1L[3\\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\\u003eCLdAe]6L2AD0aYHc5\\u003e=fM7h\\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\\\WgS7F]TO8G\\\\K4ZJ0]\\u003eKE\\u003cea\\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\\u003eLW5VIfJL:eQ4K3a1^WN5T=\\\\X=\\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\\\_5H1C\\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\\u003eTQQWYJ_@FAX\\\\]9jh\\u003ebZKLBhJ4JO6F]ZhBFV\\\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\\\ET4dQGWajP7A0[?M\\\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\\\aFM9e\\\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\\\E@AUCbX6\\u003eBgES\\u003e5EaeOFeG:i\\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\\u003cYgVEcjFcQD\\\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\\u003e[9X01E@[WeF5T_2Q9c\\\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\\\BSiEbcHI\\\\_@\\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\\u003eRQ]5Z9jA@Y9V1ZI6TDkC\\u003eNZ_f_DR\\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\\u003eFW\\u003cJ6X?IiJ\\u003c549XOhWM^ZE\\\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\\\gUkj1DZX7H]5;f\\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\\\@9@;gHHI\\u003eI]gBS\\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\\\D?b34\\u003eh_2@i3kd02G\\u003c5MQUCjUcI1\\\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\\u003c^U7Hk]7Q6P:QZS;Ge@:\\u003c\\u003cfT6PK7j4?;cdC@c5GI:gS[W\\u003cf26;\\u003cBG7fMXFTWJcbB\\\\9QT\\u003eh3HdV8Pb3Rh\\u003e^?Ue:7RP[=jT4AE\\u003ebiL_1dYW1\\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\\u003cA9LXQbECIc2M\\u003c^I\\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\\\Y?:fIPFMied[4B^FU;c\\u003e\\\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\\u003c_F9P\\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\\u003e=R4U3W1G;\\u003chN\\\\WFO_=DD\\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\\\JU6^agiJY]=5T\\u003eY?bFOMZO\\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\\\Y5?3iRg4\\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\\u003efaaP8P4KDVSCiQ=2\\u003c=Ef:\\u003eP\\u003cDNX^FW1AMcaVHe6\\\\PY4N?AQKNeFX9fcLIP?_\\u003c@5Z8fDPJAE8DcGUIb8C\\u003c_L7XhP=\\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\\u003e]LW\\u003ee^b\\u003e?0G9Ie\\u003c\\u003c@UT4e9\\u003cGM_jME7[6TFEN:\\u003c\\\\H\\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\\u003cL42d\\\\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\\u003eEJQi8j;]L5CILgXdR_\\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\\u003cKHA:\\\\[CW7SRYVhE1[MD\\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\\\cV=SLT]iM=Xa5XCZG8k\\u003eQb]UVVZ:18fe_8M?\\\\?\\u003e\\u003eLf4QSG@jO@\\u003c57iZ]UIgVRaOEi1UZ@ch\\\\]1BEHSDgcP1iN\\\\[8:W^\\\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\\u003cR]Ofg:TNGW0L\\u003ePOC_CP\\u003e^PI[aZ:KY^V@Q;;ME_k\\\\K0\\u003eYP]1D5QSc51SfZ]FIP1Y6\\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\\u003e0]RJGeQiJG5\\\\=O8TRG5U\\u003eLGa\\u003eRi2K\\u003c3=1TVHN=FhTJYajbIP\\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\\u003e93HU2ig?7\\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\\\FG1J\\u003eh^?RKUT[e4T\\\\6]ZG6OXgN_Oi\\\\@D8A^G\\u003eQVa1?J\\\\:NDfT7U0=9Y9WLYU=iiF?\\\\]MBGCCW]3@H[eNEe[MSe94R^AP\\\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\\\h71TY29]HTS@VBA\\\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\\\3Me2UC4dS\\\\NFEIMdbSFaZi1a\\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\\u003cA\\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\\u003e@^\\u003e[4\\u003e=^kM;eO@R\\\\\\\\Id]Gb2\\\\cbYC5j5CZ9QggPI\\\\ETVde\\u003cUVVNH2EJ^=ALOFKUX:^\\u003e5Z^NK88511BWWh:4iNN\\\\[_=?:XdbaW5fEcJ0Rf2S\\u003cX?9bC7Ebc5V5E]\\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\\\ICcTX=hbfHGJ\\\\2T91SC\\u003e\\u003e5EVE[XS:DDRX;;DH8;CPS\\\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\\u003ceM8?j]NZai4\\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\\u003eI=?@f\\u003cG349NMId8[T^@Sf\\u003c5O?SCB5FPNS_^Ok:R4C6Q\\\\iXLRK\\\\:Eg@d\\u003cc\\u003cMhS3K;b\\u003eZbHAf[GKME9igTY7iVFba\\u003e4D;WFVb=dQ4Abj2\\u003eJNSSLP;:V:11V?5jK\\\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\\u003e@TM9eO\\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\\u003c@=gPHLhQFDC@:T\\u003cREdY\\u003caWB]VFgMC_YS1U7J64jMHB\\\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\\\\\D:eMNPiWe1ad\\u003cIiK1O7fbD[7[\\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\\u003e=FFMHPSBf8:\\\\XRZ91D:2D[1Y\\u003eX\\\\bfj4BEQZe:1A\\u003cQj^@7SAK]C_NCM\\\\0\\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\\u003e2\\u003e4X:9JYPXk\\u003eX_?;DAfL\\u003ec?HF\\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\\u003e1\\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\\\K25Zb\\\\=BHROPTbhJNeHVgA[_CTfG\\\\A8\\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\\u003cMCXc2X^EOV7cHAb6\\\\QTPc1ZgZ2;\\\\RFh4YUg[BZ5aE\\u003cY^MPd\\u003e6M^iNNe=P6i6Lf::P6ebjX;\\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\\u003c93\\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\\u003c8@j5e\\u003eVA76=g2=gD4V1eYF0bZd0EZ\\u003cMk2M4g[Z=baJ]cVY\\u003c[D=U2RUdBNdW=69=8UB4E1@\\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\\\;D\\u003c@44QYE[fO:AjN^cbcEMjH=\\\\ajM1CZA8^EhD3B4ia\\u003e?\\\\2XSf25dJAU@@7ASaQ\\\\TfYghk0fa\\u003e:Vj=BR7EW0_hV4=]DaSeQ\\u003c?8]?9X4GbZF41h;FS\\u003c9Pa=^SQT\\u003cL:GAIP3XX[\\\\4RKJVLFabj20Oc\\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\\\W\\u003cHg9FWd\\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\\u003eS\\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\\\Y^4_\\\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\\u003e67\\u003eW\\u003cQNZRKDH@]_j^M_AV9g4\\u003chIF\\u003eaSDhbj9GMdjh=F=j:\\u003c^Wj3C8jGDgY;VBOS8N\\\\P0UNhbe:a4FT[EW2MVIaS\\u003eO]caAKi\\u003cNa1]WfgMiB6YW]\\\\9H:jjHN]@D3[BcgX\\\\aJI\\\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\\u003cWkC\\u003c^KSgbI7?aGVaRkbA2?_Raf^\\u003e9DID]07\\u003cS431;BaRhX:hNJj]\\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\\u003cN?J\\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\\u003e8]\\u003eU2:HGATaUBPG\\u003c\\\\c0aX@_D;_EOK=]Sjk=1:VGK\\u003e=4P^K\\\\OD\\\\D008D\\u003cgY[GfMjeM\\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\\\\\OAQGLQWYhNhhAZPeNf\\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\\u003ceaB7BV;\\u003eIRKK","date_":10599,"time_":14706000000,"time1":14706100,"time6":14706123456,"timetz__":"17:30:25Z","timetz1":"17:30:25.5Z","timetz6":"17:30:25.575401Z","timestamp1":1098181434900,"timestamp6":1098181434987654,"timestamp":1098181434000000,"numeric_":{"scale":0,"value":"EAAAAAAAAAAAAAAAAA=="},"numeric_5":"MDk=","numeric_5_2":"ME8=","decimal_":{"scale":0,"value":"AeJA"},"decimal_5":"MDk=","decimal_5_2":"ME8=","hstore_":"{\"a\":\"1\",\"b\":\"2\"}","inet_":"192.168.1.5","cidr_":"10.1.0.0/16","macaddr_":"08:00:2b:01:02:03","citext_":"Tom"},"source":{"version":"1.1.2.Final","connector":"postgresql","name":"fullfillment","ts_ms":1643136777184,"snapshot":"false","db":"pguser","schema":"public","table":"basic_types","txId":559,"lsn":24914760,"xmin":null},"op":"u","ts_ms":1643136777241,"transaction":null}}` -var canonizedDebeziumUpdate2K = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":1}}` -var canonizedDebeziumUpdate2V = `{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"before"},{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"fullfillment.public.basic_types.Envelope"},"payload":{"before":null,"after":{"bl":false,"b":true,"b8":"rw==","vb":"rg==","si":-32768,"ss":1,"int":-8388605,"aid":0,"id":1,"bid":3372036854775807,"oid_":null,"real_":1.45E-10,"d":3.14E-100,"c":"1","str":"varchar_example","character_":"abcd","character_varying_":"varc","timestamptz_":"2004-10-19T08:23:54Z","tst":"2004-10-19T09:23:54Z","timetz_":"08:51:02.746572Z","time_with_time_zone_":"08:51:02.746572Z","iv":90000000000,"ba":"yv66vg==","j":"{\"k1\": \"v1\"}","jb":"{\"k2\": \"v2\"}","x":"bar","uid":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","pt":{"x":23.4,"y":-44.5,"wkb":"AQEAAABmZmZmZmY3QAAAAAAAQEbA","srid":null},"it":"192.168.100.128/25","int4range_":"[3,7)","int8range_":"[3,7)","numrange_":"[1.9,1.91)","tsrange_":"[\"2010-01-02 10:00:00\",\"2010-01-02 11:00:00\")","tstzrange_":"[\"2010-01-01 06:00:00+00\",\"2010-01-01 10:00:00+00\")","daterange_":"[2000-01-10,2000-01-21)","f":1.45E-10,"i":1,"t":"__debezium_unavailable_value","date_":10599,"time_":14706000000,"time1":14706100,"time6":14706123456,"timetz__":"17:30:25Z","timetz1":"17:30:25.5Z","timetz6":"17:30:25.575401Z","timestamp1":1098181434900,"timestamp6":1098181434987654,"timestamp":1098181434000000,"numeric_":{"scale":0,"value":"EAAAAAAAAAAAAAAAAA=="},"numeric_5":"MDk=","numeric_5_2":"ME8=","decimal_":{"scale":0,"value":"AeJA"},"decimal_5":"MDk=","decimal_5_2":"ME8=","hstore_":"{\"a\":\"1\",\"b\":\"2\"}","inet_":"192.168.1.5","cidr_":"10.1.0.0/16","macaddr_":"08:00:2b:01:02:03","citext_":"Tom"},"source":{"version":"1.1.2.Final","connector":"postgresql","name":"fullfillment","ts_ms":1643136788597,"snapshot":"false","db":"pguser","schema":"public","table":"basic_types","txId":560,"lsn":24915608,"xmin":null},"op":"u","ts_ms":1643136788636,"transaction":null}}` -var canonizedDebeziumUpdate30K = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":1}}` -var canonizedDebeziumUpdate30V = `{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"before"},{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"fullfillment.public.basic_types.Envelope"},"payload":{"before":{"bl":null,"b":null,"b8":null,"vb":null,"si":null,"ss":0,"int":null,"aid":0,"id":null,"bid":0,"oid_":null,"real_":null,"d":null,"c":null,"str":null,"character_":null,"character_varying_":null,"timestamptz_":null,"tst":null,"timetz_":null,"time_with_time_zone_":null,"iv":null,"ba":null,"j":null,"jb":null,"x":null,"uid":null,"pt":null,"it":null,"int4range_":null,"int8range_":null,"numrange_":null,"tsrange_":null,"tstzrange_":null,"daterange_":null,"f":null,"i":1,"t":null,"date_":null,"time_":null,"time1":null,"time6":null,"timetz__":null,"timetz1":null,"timetz6":null,"timestamp1":null,"timestamp6":null,"timestamp":null,"numeric_":null,"numeric_5":null,"numeric_5_2":null,"decimal_":null,"decimal_5":null,"decimal_5_2":null,"hstore_":null,"inet_":null,"cidr_":null,"macaddr_":null,"citext_":null},"after":null,"source":{"version":"1.1.2.Final","connector":"postgresql","name":"fullfillment","ts_ms":1643136800841,"snapshot":"false","db":"pguser","schema":"public","table":"basic_types","txId":563,"lsn":25011512,"xmin":null},"op":"d","ts_ms":1643136801203,"transaction":null}}` -var canonizedDebeziumUpdate31K = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":1}}` -var canonizedDebeziumUpdate32K = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":2}}` -var canonizedDebeziumUpdate32V = `{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"before"},{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"fullfillment.public.basic_types.Envelope"},"payload":{"before":null,"after":{"bl":false,"b":true,"b8":"rw==","vb":"rg==","si":-32768,"ss":1,"int":-8388605,"aid":0,"id":1,"bid":3372036854775807,"oid_":null,"real_":1.45E-10,"d":3.14E-100,"c":"1","str":"varchar_example","character_":"abcd","character_varying_":"varc","timestamptz_":"2004-10-19T08:23:54Z","tst":"2004-10-19T09:23:54Z","timetz_":"08:51:02.746572Z","time_with_time_zone_":"08:51:02.746572Z","iv":90000000000,"ba":"yv66vg==","j":"{\"k1\": \"v1\"}","jb":"{\"k2\": \"v2\"}","x":"bar","uid":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","pt":{"x":23.4,"y":-44.5,"wkb":"AQEAAABmZmZmZmY3QAAAAAAAQEbA","srid":null},"it":"192.168.100.128/25","int4range_":"[3,7)","int8range_":"[3,7)","numrange_":"[1.9,1.91)","tsrange_":"[\"2010-01-02 10:00:00\",\"2010-01-02 11:00:00\")","tstzrange_":"[\"2010-01-01 06:00:00+00\",\"2010-01-01 10:00:00+00\")","daterange_":"[2000-01-10,2000-01-21)","f":1.45E-10,"i":2,"t":"__debezium_unavailable_value","date_":10599,"time_":14706000000,"time1":14706100,"time6":14706123456,"timetz__":"17:30:25Z","timetz1":"17:30:25.5Z","timetz6":"17:30:25.575401Z","timestamp1":1098181434900,"timestamp6":1098181434987654,"timestamp":1098181434000000,"numeric_":{"scale":0,"value":"EAAAAAAAAAAAAAAAAA=="},"numeric_5":"MDk=","numeric_5_2":"ME8=","decimal_":{"scale":0,"value":"AeJA"},"decimal_5":"MDk=","decimal_5_2":"ME8=","hstore_":"{\"a\":\"1\",\"b\":\"2\"}","inet_":"192.168.1.5","cidr_":"10.1.0.0/16","macaddr_":"08:00:2b:01:02:03","citext_":"Tom"},"source":{"version":"1.1.2.Final","connector":"postgresql","name":"fullfillment","ts_ms":1643136800841,"snapshot":"false","db":"pguser","schema":"public","table":"basic_types","txId":563,"lsn":25011512,"xmin":null},"op":"c","ts_ms":1643136801204,"transaction":null}}` -var canonizedDebeziumDeleteK = `{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"i"}],"optional":false,"name":"fullfillment.public.basic_types.Key"},"payload":{"i":2}}` -var canonizedDebeziumDeleteV = `{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"before"},{"type":"struct","fields":[{"type":"boolean","optional":true,"field":"bl"},{"type":"boolean","optional":true,"field":"b"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"b8"},{"type":"bytes","optional":true,"name":"io.debezium.data.Bits","version":1,"parameters":{"length":"8"},"field":"vb"},{"type":"int16","optional":true,"field":"si"},{"type":"int16","optional":false,"field":"ss"},{"type":"int32","optional":true,"field":"int"},{"type":"int32","optional":false,"field":"aid"},{"type":"int64","optional":true,"field":"id"},{"type":"int64","optional":false,"field":"bid"},{"type":"int64","optional":true,"field":"oid_"},{"type":"float","optional":true,"field":"real_"},{"type":"double","optional":true,"field":"d"},{"type":"string","optional":true,"field":"c"},{"type":"string","optional":true,"field":"str"},{"type":"string","optional":true,"field":"character_"},{"type":"string","optional":true,"field":"character_varying_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"timestamptz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTimestamp","version":1,"field":"tst"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz_"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"time_with_time_zone_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroDuration","version":1,"field":"iv"},{"type":"bytes","optional":true,"field":"ba"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"j"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"jb"},{"type":"string","optional":true,"name":"io.debezium.data.Xml","version":1,"field":"x"},{"type":"string","optional":true,"name":"io.debezium.data.Uuid","version":1,"field":"uid"},{"type":"struct","fields":[{"type":"double","optional":false,"field":"x"},{"type":"double","optional":false,"field":"y"},{"type":"bytes","optional":true,"field":"wkb"},{"type":"int32","optional":true,"field":"srid"}],"optional":true,"name":"io.debezium.data.geometry.Point","version":1,"doc":"Geometry (POINT)","field":"pt"},{"type":"string","optional":true,"field":"it"},{"type":"string","optional":true,"field":"int4range_"},{"type":"string","optional":true,"field":"int8range_"},{"type":"string","optional":true,"field":"numrange_"},{"type":"string","optional":true,"field":"tsrange_"},{"type":"string","optional":true,"field":"tstzrange_"},{"type":"string","optional":true,"field":"daterange_"},{"type":"double","optional":true,"field":"f"},{"type":"int32","optional":false,"field":"i"},{"type":"string","optional":true,"field":"t"},{"type":"int32","optional":true,"name":"io.debezium.time.Date","version":1,"field":"date_"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time_"},{"type":"int32","optional":true,"name":"io.debezium.time.Time","version":1,"field":"time1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTime","version":1,"field":"time6"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz__"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz1"},{"type":"string","optional":true,"name":"io.debezium.time.ZonedTime","version":1,"field":"timetz6"},{"type":"int64","optional":true,"name":"io.debezium.time.Timestamp","version":1,"field":"timestamp1"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp6"},{"type":"int64","optional":true,"name":"io.debezium.time.MicroTimestamp","version":1,"field":"timestamp"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"numeric_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"numeric_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"numeric_5_2"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"scale"},{"type":"bytes","optional":false,"field":"value"}],"optional":true,"name":"io.debezium.data.VariableScaleDecimal","version":1,"doc":"Variable scaled decimal","field":"decimal_"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"0","connect.decimal.precision":"5"},"field":"decimal_5"},{"type":"bytes","optional":true,"name":"org.apache.kafka.connect.data.Decimal","version":1,"parameters":{"scale":"2","connect.decimal.precision":"5"},"field":"decimal_5_2"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"hstore_"},{"type":"string","optional":true,"field":"inet_"},{"type":"string","optional":true,"field":"cidr_"},{"type":"string","optional":true,"field":"macaddr_"},{"type":"string","optional":true,"field":"citext_"}],"optional":true,"name":"fullfillment.public.basic_types.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"fullfillment.public.basic_types.Envelope"},"payload":{"before":{"bl":null,"b":null,"b8":null,"vb":null,"si":null,"ss":0,"int":null,"aid":0,"id":null,"bid":0,"oid_":null,"real_":null,"d":null,"c":null,"str":null,"character_":null,"character_varying_":null,"timestamptz_":null,"tst":null,"timetz_":null,"time_with_time_zone_":null,"iv":null,"ba":null,"j":null,"jb":null,"x":null,"uid":null,"pt":null,"it":null,"int4range_":null,"int8range_":null,"numrange_":null,"tsrange_":null,"tstzrange_":null,"daterange_":null,"f":null,"i":2,"t":null,"date_":null,"time_":null,"time1":null,"time6":null,"timetz__":null,"timetz1":null,"timetz6":null,"timestamp1":null,"timestamp6":null,"timestamp":null,"numeric_":null,"numeric_5":null,"numeric_5_2":null,"decimal_":null,"decimal_5":null,"decimal_5_2":null,"hstore_":null,"inet_":null,"cidr_":null,"macaddr_":null,"citext_":null},"after":null,"source":{"version":"1.1.2.Final","connector":"postgresql","name":"fullfillment","ts_ms":1643136813333,"snapshot":"false","db":"pguser","schema":"public","table":"basic_types","txId":564,"lsn":25012328,"xmin":null},"op":"d","ts_ms":1643136813526,"transaction":null}}` - -//--------------------------------------------------------------------------------------------------------------------- - -var index = 0 -var tt *testing.T - -func callbackFunc(_ interface{}, msgs ...interface{}) error { - //---------------------------------------------------------------------- - // filter only 'basic_types' table - - finalMsgs := make([]kafka.Message, 0) - for _, el := range msgs { - switch v := el.(type) { - case kafka.Message: - finalMsgs = append(finalMsgs, v) - } - } - if len(finalMsgs) == 0 { - return nil - } - - //---------------------------------------------------------------------- - // check - - for _, v := range finalMsgs { - var value *string = nil - if v.Value != nil { - tmp := string(v.Value) - value = &tmp - } - //-------------------------------------------------------------------------------- - fmt.Printf("msg key:%s\n", string(v.Key)) - if value == nil { - fmt.Println("msg val:nil") - } else { - fmt.Printf("msg val:%s\n", *value) - } - //-------------------------------------------------------------------------------- - testutil.CheckCanonizedDebeziumEvent2(tt, string(v.Key), value, testCases[index]) - index++ - } - - //---------------------------------------------------------------------- - - return nil -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestReplication(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - - //------------------------------------------------------------------------------ - // init testSuite - - testSuite := []debeziumcommon.ChangeItemCanon{ - { - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumInsertK, - DebeziumVal: &canonizedDebeziumInsertV, - }}, - }, - { - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumUpdate1K, - DebeziumVal: &canonizedDebeziumUpdate1V, - }}, - }, - { - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumUpdate2K, - DebeziumVal: &canonizedDebeziumUpdate2V, - }}, - }, - { - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumUpdate30K, - DebeziumVal: &canonizedDebeziumUpdate30V, - }, { - DebeziumKey: canonizedDebeziumUpdate31K, - DebeziumVal: nil, - }, { - DebeziumKey: canonizedDebeziumUpdate32K, - DebeziumVal: &canonizedDebeziumUpdate32V, - }}, - }, - { - DebeziumEvents: []debeziumcommon.KeyValue{{ - DebeziumKey: canonizedDebeziumDeleteK, - DebeziumVal: &canonizedDebeziumDeleteV, - }, { - DebeziumKey: canonizedDebeziumDeleteK, - DebeziumVal: nil, - }}, - }, - } - - testSuite = testutil.FixTestSuite(t, testSuite, "fullfillment", "pguser", "pg") - - for _, canons := range testSuite { - testCases = append(testCases, canons.DebeziumEvents...) - } - - //------------------------------------------------------------------------------ - // start replication - - tt = t - - dst := &kafka2.KafkaDestination{ - Connection: &kafka2.KafkaConnectionOptions{ - TLS: logbroker.DefaultTLS, - Brokers: []string{"my_broker_0"}, - }, - Auth: &kafka2.KafkaAuth{ - Enabled: true, - Mechanism: "SHA-512", - User: "user1", - Password: "qwert12345", - }, - TopicPrefix: "fullfillment", - FormatSettings: server.SerializationFormat{ - Name: server.SerializationFormatDebezium, - Settings: map[string]string{ - debeziumparameters.DatabaseDBName: "pguser", - debeziumparameters.AddOriginalTypes: "false", - debeziumparameters.SourceType: "pg", - }, - }, - ParralelWriterCount: 10, - // declare 'FormatSettings' explicitly, bcs here are not honest kafka-target, - // and here are we forced to create transfer after creating sink - in real workflow, - // FillDependentFields() called earlier than creating sink - AddSystemTables: false, - } - - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - producer := kafka2.NewMockwriter(ctrl) - producer.EXPECT().WriteMessages(gomock.Any(), gomock.Any()).AnyTimes().Do(callbackFunc) - producer.EXPECT().Close().AnyTimes() - - client := kafka2.NewMockclient(ctrl) - client.EXPECT().BuildWriter([]string{"my_broker_0"}, gomock.Any(), gomock.Any(), gomock.Any()).Return(producer) - client.EXPECT().CreateTopicIfNotExist([]string{"my_broker_0"}, "fullfillment.public.basic_types", gomock.Any(), gomock.Any(), gomock.Any()) - - sink, err := kafka2.NewSinkImpl( - dst, - solomon.NewRegistry(nil).WithTags(map[string]string{"ts": time.Now().String()}), - logger.Log, - client, - false, - ) - require.NoError(t, err) - - target := server.MockDestination{SinkerFactory: func() abstract.Sinker { return sink }} - transfer := helpers.MakeTransfer("fake", &Source, &target, abstract.TransferTypeIncrementOnly) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //----------------------------------------------------------------------------------------------------------------- - // execute SQL statements - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), update1Stmt) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), update2Stmt) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), update3Stmt) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), deleteStmt) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - - for { - if index == len(testCases) { - break - } - time.Sleep(time.Second) - } -} diff --git a/tests/e2e/pg2kafkamock/debezium_replication/init_source/dump.sql b/tests/e2e/pg2kafkamock/debezium_replication/init_source/dump.sql deleted file mode 100644 index 5d4c4e75..00000000 --- a/tests/e2e/pg2kafkamock/debezium_replication/init_source/dump.sql +++ /dev/null @@ -1,106 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - -- MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); diff --git a/tests/e2e/pg2mysql/snapshot/check_db_test.go b/tests/e2e/pg2mysql/snapshot/check_db_test.go deleted file mode 100644 index 4cc5f9aa..00000000 --- a/tests/e2e/pg2mysql/snapshot/check_db_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package light - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - - Source = *pgrecipe.RecipeSource() - Target = *helpers.RecipeMysqlTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "pg source", Port: Source.Port}, - helpers.LabeledPort{Label: "mysql target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Snapshot", Snapshot) - }) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - _ = helpers.Activate(t, transfer) - helpers.CheckRowsCount(t, Target, Target.Database, "__test", 16) -} diff --git a/tests/e2e/pg2mysql/snapshot/dump/type_check.sql b/tests/e2e/pg2mysql/snapshot/dump/type_check.sql deleted file mode 100644 index 0d41867d..00000000 --- a/tests/e2e/pg2mysql/snapshot/dump/type_check.sql +++ /dev/null @@ -1,90 +0,0 @@ -create table __test ( - id bigint not null, - aid serial, - f float, - d double precision, - de decimal(10,2), - i int, - bi bigint, - biu bigint, - b bit(8), - da date, - ts timestamp, - dt timestamp, - c char, - str varchar(256), - t text, - primary key (aid, id) -- test multi pk and reverse order keys -); - -insert into __test values ( - 1, -- id - 0, -- aid - 1.45e-10, -- f - 3.14e-100, -- d - 2.5, -- de - -8388605, -- i - 2147483642, -- bi - 9223372036854775804, --biu - b'10101111', -- b - '2005-03-04', -- da - now(), -- ts - now(), -- dt - '1', -- c - 'hello, friend of mine', -- str - 'okay, now bye-bye' -- t -) -, -( - 2, -- id - 1, -- aid - 1.34e-10, -- f - null, -- d - null, -- de - -1294129412, -- i - 112412412421941041, -- bi - 129491244912401240, --biu - b'10000001', -- b - '1999-03-04', -- da - now(), -- ts - null, -- dt - '2', -- c - 'another hello', -- str - 'okay, another bye' -- t -) -, -( - 3, -- id - 4, -- aid - 5.34e-10, -- f - null, -- d - 123, -- de - 294129412, -- i - -784124124219410491, -- bi - 129491098649360240, --biu - b'10000010', -- b - '1999-03-05', -- da - null, -- ts - now(), -- dt - 'c', -- c - 'another another hello', -- str - 'okay, another another bye' -- t -) -; - -insert into __test (str, id) values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) values ('nvaapsijfapfn', 201, now()), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But Ill be back, this is public promise', 422, now()), - ('Remember me, my name is hazzus', 333, now()); - -insert into __test (str, id, f, d, de) values ('100', 100, 'NaN'::real, 'NaN'::double precision, 'NaN'::numeric); -insert into __test (str, id, f, d) values - ('101', 101, '+Inf'::real, '+Inf'::double precision), - ('102', 102, '-Inf'::real, '-Inf'::double precision); diff --git a/tests/e2e/pg2pg/access/check_db_test.go b/tests/e2e/pg2pg/access/check_db_test.go deleted file mode 100644 index 2e5dffcb..00000000 --- a/tests/e2e/pg2pg/access/check_db_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package snapshot - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" - "golang.org/x/net/context" -) - -var ( - tablesA = []abstract.TableDescription{ - { - Schema: "public", - Name: "t_accessible", - Filter: abstract.NoFilter, - EtaRow: 0, - Offset: 0, - }, - { - Schema: "public", - Name: "t_empty", - Filter: abstract.NoFilter, - EtaRow: 0, - Offset: 0, - }, - } - tablesIA = []abstract.TableDescription{ - { - Schema: "public", - Name: "t_inaccessible", - Filter: abstract.NoFilter, - EtaRow: 0, - Offset: 0, - }, - { - Schema: "public", - Name: "t_empty", - Filter: abstract.NoFilter, - EtaRow: 0, - Offset: 0, - }, - } -) - -func descsToPgNames(descs []abstract.TableDescription) []string { - result := make([]string, 0) - for _, d := range descs { - result = append(result, d.Fqtn()) - } - return result -} - -var ( - SourceA = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix(""), pgrecipe.WithDBTables(descsToPgNames(tablesA)...), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.User = "blockeduser" - pg.Password = "sim-sim@OPEN" - })) - SourceIAForDump = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix(""), pgrecipe.WithDBTables(descsToPgNames(tablesIA)...)) - SourceIA = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix(""), pgrecipe.WithDBTables(descsToPgNames(tablesIA)...), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.User = "blockeduser" - pg.Password = "sim-sim@OPEN" - })) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -var ( - sourceATID = helpers.TransferID + "A" - sourceIATID = helpers.TransferID + "IA" - sourceIAForDumpTID = helpers.TransferID + "IAForDump" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - - Target.Cleanup = server.DisabledCleanup - helpers.InitSrcDst(sourceATID, &SourceA, &Target, abstract.TransferTypeSnapshotOnly) - helpers.InitSrcDst(sourceIATID, &SourceIA, &Target, abstract.TransferTypeSnapshotOnly) - helpers.InitSrcDst(sourceIAForDumpTID, &SourceIAForDump, &Target, abstract.TransferTypeSnapshotOnly) -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source A", Port: SourceA.Port}, - helpers.LabeledPort{Label: "PG source IA for dump", Port: SourceIAForDump.Port}, - helpers.LabeledPort{Label: "PG source IA", Port: SourceIA.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("Upload_accessible", UploadTestAccessible) - t.Run("Upload_inaccessible", UploadTestInaccessible) -} - -func UploadTestAccessible(t *testing.T) { - transfer := helpers.MakeTransfer(sourceATID, &SourceA, &Target, abstract.TransferTypeSnapshotOnly) - - pgdump, err := postgres.ExtractPgDumpSchema(transfer) - require.NoError(t, err) - require.NoError(t, postgres.ApplyPgDumpPreSteps(pgdump, transfer, helpers.EmptyRegistry())) - - require.NoError(t, tasks.Upload(context.TODO(), coordinator.NewFakeClient(), *transfer, nil, tasks.UploadSpec{Tables: tablesA}, helpers.EmptyRegistry())) -} - -func UploadTestInaccessible(t *testing.T) { - transferForDump := helpers.MakeTransfer(sourceIAForDumpTID, &SourceIAForDump, &Target, abstract.TransferTypeSnapshotOnly) - pgdump, err := postgres.ExtractPgDumpSchema(transferForDump) - require.NoError(t, err) - require.NoError(t, postgres.ApplyPgDumpPreSteps(pgdump, transferForDump, helpers.EmptyRegistry())) - - transfer := helpers.MakeTransfer(sourceIATID, &SourceIA, &Target, abstract.TransferTypeSnapshotOnly) - err = tasks.Upload(context.TODO(), coordinator.NewFakeClient(), *transfer, nil, tasks.UploadSpec{Tables: tablesIA}, helpers.EmptyRegistry()) - require.Error(t, err) - require.Contains(t, err.Error(), "Missing tables in source (pg)") - require.Contains(t, err.Error(), `"public"."t_inaccessible"`) -} diff --git a/tests/e2e/pg2pg/access/dump/dump.sql b/tests/e2e/pg2pg/access/dump/dump.sql deleted file mode 100644 index 4298309e..00000000 --- a/tests/e2e/pg2pg/access/dump/dump.sql +++ /dev/null @@ -1,16 +0,0 @@ --- This test checks access is properly checked at Upload - -CREATE USER blockeduser PASSWORD 'sim-sim@OPEN'; - -CREATE TABLE t_accessible(i INT PRIMARY KEY, t TEXT); -INSERT INTO t_accessible VALUES (1, 'a'), (2, 'b'), (3, 'c'); -GRANT SELECT ON TABLE t_accessible TO blockeduser; - -CREATE TABLE t_empty(LIKE t_accessible); -GRANT SELECT ON TABLE t_empty TO blockeduser; - -CREATE TABLE t_inaccessible(LIKE t_accessible); -INSERT INTO t_inaccessible SELECT * FROM t_accessible; -REVOKE SELECT ON TABLE t_inaccessible FROM blockeduser; - -CREATE TYPE custom_enum AS ENUM('a', 'b', 'c'); diff --git a/tests/e2e/pg2pg/all_types/check_db_test.go b/tests/e2e/pg2pg/all_types/check_db_test.go deleted file mode 100644 index dd97c622..00000000 --- a/tests/e2e/pg2pg/all_types/check_db_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package alltypes - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgStorage "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/canon/postgres" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestAllDataTypes(t *testing.T) { - Source := pgrecipe.RecipeSource(pgrecipe.WithPrefix("")) - Source.WithDefaults() - Target := pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) - conn, err := pgStorage.MakeConnPoolFromDst(Target, logger.Log) - require.NoError(t, err) - // TODO: Allow to optionally transit extensions as part of transfer - _, err = conn.Exec(context.Background(), ` -create extension if not exists hstore; -create extension if not exists ltree; -create extension if not exists citext; -`) - require.NoError(t, err) - - helpers.InitSrcDst(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) - - tableCase := func(tableName string) func(t *testing.T) { - return func(t *testing.T) { - t.Run("initial data", func(t *testing.T) { - conn, err := pgStorage.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), postgres.TableSQLs[tableName]) - require.NoError(t, err) - }) - - Source.DBTables = []string{tableName} - transfer := helpers.MakeTransfer( - t.Name(), - Source, - Target, - abstract.TransferTypeSnapshotAndIncrement, - ) - transfer.DataObjects = &server.DataObjects{IncludeObjects: []string{tableName}} - worker := helpers.Activate(t, transfer) - - conn, err := pgStorage.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - _, err = conn.Exec(context.Background(), postgres.TableSQLs[tableName]) - require.NoError(t, err) - srcStorage, err := pgStorage.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - dstStorage, err := pgStorage.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - tid, err := abstract.ParseTableID(tableName) - require.NoError(t, err) - require.NoError(t, helpers.WaitEqualRowsCount(t, tid.Namespace, tid.Name, srcStorage, dstStorage, time.Second*30)) - worker.Close(t) - hashQuery := fmt.Sprintf(` -SELECT md5(array_agg(md5((t.*)::varchar))::varchar) - FROM ( - SELECT * - FROM %s - ORDER BY 1 - ) AS t -; -`, tableName) - var srcHash string - require.NoError(t, srcStorage.Conn.QueryRow(context.Background(), hashQuery).Scan(&srcHash)) - var dstHash string - require.NoError(t, srcStorage.Conn.QueryRow(context.Background(), hashQuery).Scan(&dstHash)) - require.Equal(t, srcHash, dstHash) - } - } - t.Run("array_types", tableCase("public.array_types")) - t.Run("date_types", tableCase("public.date_types")) - t.Run("geom_types", tableCase("public.geom_types")) - t.Run("numeric_types", tableCase("public.numeric_types")) - t.Run("text_types", tableCase("public.text_types")) - t.Run("wtf_types", tableCase("public.wtf_types")) -} diff --git a/tests/e2e/pg2pg/bytea_key/check_db_test.go b/tests/e2e/pg2pg/bytea_key/check_db_test.go deleted file mode 100644 index 4ed56642..00000000 --- a/tests/e2e/pg2pg/bytea_key/check_db_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package byteakey - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source"), pgrecipe.WithDBTables("public.test")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestByteaKey(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - dstConn, err := pgcommon.MakeConnPoolFromDst(&Target, logger.Log) - require.NoError(t, err) - defer dstConn.Close() - - _, err = srcConn.Exec(context.Background(), `SELECT pg_create_logical_replication_slot('testslot', 'wal2json')`) - require.NoError(t, err) - defer func() { - _, err := srcConn.Exec(context.Background(), `SELECT pg_drop_replication_slot('testslot')`) - require.NoError(t, err) - }() - - _, err = srcConn.Exec(context.Background(), `INSERT INTO test VALUES ('\xdeadbeef', 'a')`) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), `UPDATE test SET value = 'b'`) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), `INSERT INTO test VALUES ('\xB16B00B5', 'b')`) - require.NoError(t, err) - - worker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - worker.Start() - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - err = worker.Stop() - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/bytea_key/init_source/dump.sql b/tests/e2e/pg2pg/bytea_key/init_source/dump.sql deleted file mode 100644 index 534ac185..00000000 --- a/tests/e2e/pg2pg/bytea_key/init_source/dump.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE test ( - id BYTEA PRIMARY KEY, - value TEXT -); diff --git a/tests/e2e/pg2pg/bytea_key/init_target/dump.sql b/tests/e2e/pg2pg/bytea_key/init_target/dump.sql deleted file mode 100644 index c0d6aaeb..00000000 --- a/tests/e2e/pg2pg/bytea_key/init_target/dump.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE test ( - id BYTEA PRIMARY KEY, - value TEXT -); \ No newline at end of file diff --git a/tests/e2e/pg2pg/debezium/all_datatypes/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes/check_db_test.go deleted file mode 100644 index 18dd7ced..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes/check_db_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 2, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '12:13:14-04', - '12:13:14.5-04', - '12:13:14.456789-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2,c=>"another hstore value = which > needs '' quoting \" and escaping"', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithPriorityComparators(pgDebeziumTimeAsStringComparator))) -} - -func pgDebeziumTimeAsStringComparator(lVal interface{}, lSchema abstract.ColSchema, rVal interface{}, rSchema abstract.ColSchema, _ bool) (comparable bool, result bool, err error) { - lS, lSOk := lVal.(string) - rS, rSOk := rVal.(string) - castsToString := lSOk && rSOk - - switch { - case lSchema.OriginalType == "pg:time with time zone" && rSchema.OriginalType == "pg:time with time zone": - if !castsToString { - return false, false, nil - } - return true, helpers.TimeWithPrecision(lS, 0) == helpers.TimeWithPrecision(rS, 0), nil - } - - return false, false, nil -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes/init_source/dump.sql deleted file mode 100644 index 97323de3..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes/init_source/dump.sql +++ /dev/null @@ -1,213 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '12:13:14-04', - '12:13:14.5-04', - '12:13:14.456789-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2,c=>"hstore value = which > needs '' quoting \" and escaping"', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_arr/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_arr/check_db_test.go deleted file mode 100644 index 5483583a..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_arr/check_db_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - 2, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - --- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, --- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, --- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, --- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, --- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - // require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 1)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_source/dump.sql deleted file mode 100644 index d52ff934..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_source/dump.sql +++ /dev/null @@ -1,170 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - i int PRIMARY KEY, - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[] - --- ARR_HSTORE_ HSTORE[], --- ARR_INET_ INET[], --- ARR_CIDR_ CIDR[], --- ARR_MACADDR_ MACADDR[], --- -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) --- -- LTREE - should be in special table, I suppose --- ARR_CITEXT_ CITEXT[] -); - -INSERT INTO public.basic_types VALUES ( - 1, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - - -- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, - -- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, - -- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, - -- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, - -- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_arr/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/check_db_test.go deleted file mode 100644 index fd74e5d3..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/check_db_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 2, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_source/dump.sql deleted file mode 100644 index a3039795..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_source/dump.sql +++ /dev/null @@ -1,213 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/check_db_test.go deleted file mode 100644 index d6976dfe..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/check_db_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - 2, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - --- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, --- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, --- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, --- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, --- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_source/dump.sql deleted file mode 100644 index e8fe9042..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_source/dump.sql +++ /dev/null @@ -1,170 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - i int PRIMARY KEY, - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[] - --- ARR_HSTORE_ HSTORE[], --- ARR_INET_ INET[], --- ARR_CIDR_ CIDR[], --- ARR_MACADDR_ MACADDR[], --- -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) --- -- LTREE - should be in special table, I suppose --- ARR_CITEXT_ CITEXT[] -); - -INSERT INTO public.basic_types VALUES ( - 1, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - - -- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, - -- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, - -- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, - -- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, - -- '{"Tom","Tom"}' -- CITEXT_ CITEXT - ); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_nohomo_arr/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde/check_db_test.go deleted file mode 100644 index 10545689..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde/check_db_test.go +++ /dev/null @@ -1,202 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 2, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - Source.DBTables = []string{"public.basic_types"} -} - -//--------------------------------------------------------------------------------------------------------------------- - -func serdeUdf(t *testing.T, items []abstract.ChangeItem) abstract.TransformerResult { - newChangeItems := make([]abstract.ChangeItem, 0) - errors := make([]abstract.TransformerError, 0) - for i := range items { - if items[i].IsSystemTable() { - continue - } - currJSON := items[i].ToJSONString() - fmt.Printf("changeItem dump:%s\n", currJSON) - outChangeItem, err := abstract.UnmarshalChangeItem([]byte(currJSON)) - if err != nil { - errors = append(errors, abstract.TransformerError{ - Input: items[i], - Error: err, - }) - } else { - newChangeItems = append(newChangeItems, *outChangeItem) - } - } - return abstract.TransformerResult{ - Transformed: newChangeItems, - Errors: errors, - } -} - -func anyTablesUdf(table abstract.TableID, schema abstract.TableColumns) bool { - return true -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - serdeTransformer := helpers.NewSimpleTransformer(t, serdeUdf, anyTablesUdf) - helpers.AddTransformer(t, transfer, serdeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_source/dump.sql deleted file mode 100644 index a3039795..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_source/dump.sql +++ /dev/null @@ -1,213 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/check_db_test.go deleted file mode 100644 index 8e2ad5e0..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/check_db_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - 2, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - --- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, --- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, --- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, --- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, --- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -//--------------------------------------------------------------------------------------------------------------------- - -func serdeUdf(t *testing.T, items []abstract.ChangeItem) abstract.TransformerResult { - newChangeItems := make([]abstract.ChangeItem, 0) - errors := make([]abstract.TransformerError, 0) - for i := range items { - if items[i].IsSystemTable() { - continue - } - currJSON := items[i].ToJSONString() - fmt.Printf("changeItem dump:%s\n", currJSON) - outChangeItem, err := abstract.UnmarshalChangeItem([]byte(currJSON)) - if err != nil { - errors = append(errors, abstract.TransformerError{ - Input: items[i], - Error: err, - }) - } else { - newChangeItems = append(newChangeItems, *outChangeItem) - } - } - return abstract.TransformerResult{ - Transformed: newChangeItems, - Errors: errors, - } -} - -func anyTablesUdf(table abstract.TableID, schema abstract.TableColumns) bool { - return true -} - -//--------------------------------------------------------------------------------------------------------------------- - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - serdeTransformer := helpers.NewSimpleTransformer(t, serdeUdf, anyTablesUdf) - helpers.AddTransformer(t, transfer, serdeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_source/dump.sql deleted file mode 100644 index d52ff934..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_source/dump.sql +++ /dev/null @@ -1,170 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - i int PRIMARY KEY, - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[] - --- ARR_HSTORE_ HSTORE[], --- ARR_INET_ INET[], --- ARR_CIDR_ CIDR[], --- ARR_MACADDR_ MACADDR[], --- -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) --- -- LTREE - should be in special table, I suppose --- ARR_CITEXT_ CITEXT[] -); - -INSERT INTO public.basic_types VALUES ( - 1, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - - -- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, - -- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, - -- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, - -- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, - -- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_arr/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/check_db_test.go deleted file mode 100644 index d35cacb0..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/check_db_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - 2, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - --- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, --- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, --- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, --- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, --- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithPriorityComparators(helpers.PgDebeziumIgnoreTemporalAccuracyForArraysComparator))) - require.Equal(t, 2, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_source/dump.sql deleted file mode 100644 index d52ff934..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_source/dump.sql +++ /dev/null @@ -1,170 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - i int PRIMARY KEY, - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[] - --- ARR_HSTORE_ HSTORE[], --- ARR_INET_ INET[], --- ARR_CIDR_ CIDR[], --- ARR_MACADDR_ MACADDR[], --- -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) --- -- LTREE - should be in special table, I suppose --- ARR_CITEXT_ CITEXT[] -); - -INSERT INTO public.basic_types VALUES ( - 1, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - - -- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, - -- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, - -- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, - -- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, - -- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_embedded/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/check_db_test.go deleted file mode 100644 index 18d1a2a2..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/check_db_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumcommon "github.com/doublecloud/tross/pkg/debezium/common" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - 2, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - --- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, --- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, --- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, --- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, --- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "false", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - originalTypes := map[abstract.TableID]map[string]*debeziumcommon.OriginalTypeInfo{ - {Namespace: "public", Name: "basic_types"}: { - "i": {OriginalType: "pg:integer"}, - "arr_bl": {OriginalType: "pg:boolean[]"}, - "arr_si": {OriginalType: "pg:smallint[]"}, - "arr_int": {OriginalType: "pg:integer[]"}, - "arr_id": {OriginalType: "pg:bigint[]"}, - "arr_oid_": {OriginalType: "pg:oid[]"}, - "arr_real_": {OriginalType: "pg:real[]"}, - "arr_d": {OriginalType: "pg:double precision[]"}, - "arr_c": {OriginalType: "pg:character(1)[]"}, - "arr_str": {OriginalType: "pg:character varying(256)[]"}, - "arr_character_": {OriginalType: "pg:character(4)[]"}, - "arr_character_varying_": {OriginalType: "pg:character varying(5)[]"}, - "arr_timestamptz_": {OriginalType: "pg:timestamp with time zone[]"}, - "arr_tst": {OriginalType: "pg:timestamp with time zone[]"}, - "arr_timetz_": {OriginalType: "pg:time with time zone[]"}, - "arr_time_with_time_zone_": {OriginalType: "pg:time with time zone[]"}, - "arr_uid": {OriginalType: "pg:uuid[]"}, - "arr_it": {OriginalType: "pg:inet[]"}, - "arr_f": {OriginalType: "pg:double precision[]"}, - "arr_i": {OriginalType: "pg:integer[]"}, - "arr_t": {OriginalType: "pg:text[]"}, - "arr_date_": {OriginalType: "pg:date[]"}, - "arr_time_": {OriginalType: "pg:time without time zone[]"}, - "arr_time1": {OriginalType: "pg:time(1) without time zone[]"}, - "arr_time6": {OriginalType: "pg:time(6) without time zone[]"}, - "arr_timetz__": {OriginalType: "pg:time with time zone[]"}, - "arr_timetz1": {OriginalType: "pg:time(1) with time zone[]"}, - "arr_timetz6": {OriginalType: "pg:time(6) with time zone[]"}, - "arr_timestamp1": {OriginalType: "pg:timestamp(1) without time zone[]"}, - "arr_timestamp6": {OriginalType: "pg:timestamp(6) without time zone[]"}, - "arr_timestamp": {OriginalType: "pg:timestamp without time zone[]"}, - "arr_numeric_": {OriginalType: "pg:numeric[]"}, - "arr_numeric_5": {OriginalType: "pg:numeric(5,0)[]"}, - "arr_numeric_5_2": {OriginalType: "pg:numeric(5,2)[]"}, - "arr_decimal_": {OriginalType: "pg:numeric[]"}, - "arr_decimal_5": {OriginalType: "pg:numeric(5,0)[]"}, - "arr_decimal_5_2": {OriginalType: "pg:numeric(5,2)[]"}, - }, - } - receiver := debezium.NewReceiver(originalTypes, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithPriorityComparators(helpers.PgDebeziumIgnoreTemporalAccuracyForArraysComparator))) - require.Equal(t, 2, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_source/dump.sql deleted file mode 100644 index d52ff934..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_source/dump.sql +++ /dev/null @@ -1,170 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - i int PRIMARY KEY, - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[] - --- ARR_HSTORE_ HSTORE[], --- ARR_INET_ INET[], --- ARR_CIDR_ CIDR[], --- ARR_MACADDR_ MACADDR[], --- -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) --- -- LTREE - should be in special table, I suppose --- ARR_CITEXT_ CITEXT[] -); - -INSERT INTO public.basic_types VALUES ( - 1, - - -- ----------------------------------------------------------------------------------------------------------------- - - '{true,true}', -- ARR_bl boolean[], - -- '{1,1}' -- ARR_b bit(1)[], - -- [io.debezium.relational.TableSchemaBuilder] - -- org.apache.kafka.connect.errors.DataException: Invalid Java object for schema with type BOOLEAN: class java.util.ArrayList for field: "arr_b" - - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - '{1,2}', -- ARR_si smallint[], - '{1,2}', -- ARR_int integer[], - '{1,2}', -- ARR_id bigint[], - '{1,2}', -- ARR_oid_ oid[], - - '{1.45e-10,1.45e-10}', -- ARR_real_ real[], - '{3.14e-100,3.14e-100}', -- ARR_d double precision[], - - '{"1", "1"}', -- ARR_c char[], - '{"varchar_example", "varchar_example"}', -- ARR_str varchar(256)[], - - '{"abcd","abcd"}', -- ARR_CHARACTER_ CHARACTER(4)[], - '{"varc","varc"}', -- ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - '{"2004-10-19 10:23:54+02","2004-10-19 10:23:54+02"}', -- ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - '{"2004-10-19 11:23:54+02","2004-10-19 11:23:54+02"}', -- ARR_tst TIMESTAMP WITH TIME ZONE[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIMETZ_ TIMETZ[], - '{"00:51:02.746572-08","00:51:02.746572-08"}', -- ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - - '{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}', -- ARR_uid uuid[], - '{"192.168.100.128/25","192.168.100.128/25"}', -- ARR_it inet[], - - - '{"1.45e-10","1.45e-10"}', -- ARR_f float[], - '{1,1}', -- ARR_i int[], - '{"text_example","text_example"}', -- ARR_t text[], - - '{"January 8, 1999", "January 8, 1999"}', -- DATE_ DATE, - - '{"04:05:06", "04:05:06"}', -- TIME_ TIME, - '{"04:05:06.1", "04:05:06.1"}', -- TIME1 TIME(1), - '{"04:05:06.123456", "04:05:06.123456"}', -- TIME6 TIME(6), - - '{"2020-05-26 13:30:25-04", "2020-05-26 13:30:25-04"}', -- TIMETZ__ TIME WITH TIME ZONE, - '{"2020-05-26 13:30:25.5-04", "2020-05-26 13:30:25.5-04"}', -- TIMETZ1 TIME(1) WITH TIME ZONE, - '{"2020-05-26 13:30:25.575401-04", "2020-05-26 13:30:25.575401-04"}', -- TIMETZ6 TIME(6) WITH TIME ZONE, - - '{"2004-10-19 10:23:54.9", "2004-10-19 10:23:54.9"}', -- TIMESTAMP1 TIMESTAMP(1), - '{"2004-10-19 10:23:54.987654", "2004-10-19 10:23:54.987654"}', -- TIMESTAMP6 TIMESTAMP(6), - '{"2004-10-19 10:23:54", "2004-10-19 10:23:54"}', -- TIMESTAMP TIMESTAMP, - - '{"1267650600228229401496703205376","12676506002282294.01496703205376"}', -- NUMERIC_ NUMERIC, - '{"12345","12345"}', -- NUMERIC_5 NUMERIC(5), - '{"123.67","123.67"}', -- NUMERIC_5_2 NUMERIC(5,2), - - '{"123456","123456"}', -- DECIMAL_ DECIMAL, - '{"12345","12345"}', -- DECIMAL_5 DECIMAL(5), - '{"123.67","123.67"}' -- DECIMAL_5_2 DECIMAL(5,2), - - -- '{"a=>1,b=>2","a=>1,b=>2"}', -- HSTORE_ HSTORE, - -- '{"192.168.1.5", "192.168.1.5"}', -- INET_ INET, - -- '{"10.1/16","10.1/16"}', -- CIDR_ CIDR, - -- '{"08:00:2b:01:02:03","08:00:2b:01:02:03"}', -- MACADDR_ MACADDR, - -- '{"Tom","Tom"}' -- CITEXT_ CITEXT -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_arr_external/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go deleted file mode 100644 index 3a096e86..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/check_db_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 2, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.Equal(t, 2, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_source/dump.sql deleted file mode 100644 index a3039795..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_source/dump.sql +++ /dev/null @@ -1,213 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go deleted file mode 100644 index a8565e34..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/check_db_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source"), pgrecipe.WithDBTables("public.basic_types", "public.basic_types_arr")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Runtime = &abstract.LocalRuntime{ - ShardingUpload: abstract.ShardUploadParams{ - ProcessCount: 1, - }, - } - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), `INSERT INTO public.basic_types (i) VALUES (2);`) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), `INSERT INTO public.basic_types_arr (i) VALUES (2);`) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types_arr", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.Equal(t, 4, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_source/dump.sql deleted file mode 100644 index 8c1e2ceb..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_source/dump.sql +++ /dev/null @@ -1,209 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types (i) VALUES (1); - -CREATE TABLE public.basic_types_arr -( - i int PRIMARY KEY, - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_bl boolean[], - -- ARR_b bit(1)[], - -- ARR_b8 bit(8)[], - -- ARR_vb varbit(8)[], - - ARR_si smallint[], - -- ARR_ss smallserial[], - ARR_int integer[], - -- ARR_aid serial[], - ARR_id bigint[], - -- ARR_bid bigserial[], - ARR_oid_ oid[], - - ARR_real_ real[], - ARR_d double precision[], - - ARR_c char[], - ARR_str varchar(256)[], - - ARR_CHARACTER_ CHARACTER(4)[], - ARR_CHARACTER_VARYING_ CHARACTER VARYING(5)[], - ARR_TIMESTAMPTZ_ TIMESTAMPTZ[], -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - ARR_tst TIMESTAMP WITH TIME ZONE[], - ARR_TIMETZ_ TIMETZ[], - ARR_TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE[], - -- ARR_iv interval[], - -- ARR_ba bytea[], - - -- ARR_j json[], - -- ARR_jb jsonb[], - -- ARR_x xml[], - - ARR_uid uuid[], - -- ARR_pt point[], - ARR_it inet[], - -- ARR_INT4RANGE_ INT4RANGE[], - -- ARR_INT8RANGE_ INT8RANGE[], - -- ARR_NUMRANGE_ NUMRANGE[], - -- ARR_TSRANGE_ TSRANGE[], - -- ARR_TSTZRANGE_ TSTZRANGE[], - -- ARR_DATERANGE_ DATERANGE[], - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - ARR_f float[], - ARR_i int[], - ARR_t text[], - - -- ---------------------------------------------------------------------------------------------------------------- - - ARR_DATE_ DATE[], - ARR_TIME_ TIME[], - ARR_TIME1 TIME(1)[], -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - ARR_TIME6 TIME(6)[], - - ARR_TIMETZ__ TIME WITH TIME ZONE[], - ARR_TIMETZ1 TIME(1) WITH TIME ZONE[], - ARR_TIMETZ6 TIME(6) WITH TIME ZONE[], - - ARR_TIMESTAMP1 TIMESTAMP(1)[], - ARR_TIMESTAMP6 TIMESTAMP(6)[], - ARR_TIMESTAMP TIMESTAMP[], - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - ARR_NUMERIC_ NUMERIC[], - ARR_NUMERIC_5 NUMERIC(5)[], - ARR_NUMERIC_5_2 NUMERIC(5,2)[], - - --DECIMAL - -- The types decimal and numeric are equivalent - ARR_DECIMAL_ DECIMAL[], - ARR_DECIMAL_5 DECIMAL(5)[], - ARR_DECIMAL_5_2 DECIMAL(5,2)[] - --- ARR_HSTORE_ HSTORE[], --- ARR_INET_ INET[], --- ARR_CIDR_ CIDR[], --- ARR_MACADDR_ MACADDR[], --- -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) --- -- LTREE - should be in special table, I suppose --- ARR_CITEXT_ CITEXT[] -); - -INSERT INTO public.basic_types_arr (i) VALUES (1); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_embedded_nulls/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go deleted file mode 100644 index bb015f60..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/check_db_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumcommon "github.com/doublecloud/tross/pkg/debezium/common" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 2, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "false", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - originalTypes := map[abstract.TableID]map[string]*debeziumcommon.OriginalTypeInfo{ - abstract.TableID{Namespace: "public", Name: "basic_types"}: { - "i": {OriginalType: "pg:integer"}, - "bl": {OriginalType: "pg:boolean"}, - "b": {OriginalType: "pg:bit(1)"}, - "b8": {OriginalType: "pg:bit(8)"}, - "vb": {OriginalType: "pg:bit varying(8)"}, - "si": {OriginalType: "pg:smallint"}, - "ss": {OriginalType: "pg:smallint"}, - "int": {OriginalType: "pg:integer"}, - "aid": {OriginalType: "pg:integer"}, - "id": {OriginalType: "pg:bigint"}, - "bid": {OriginalType: "pg:bigint"}, - "oid_": {OriginalType: "pg:oid"}, - "real_": {OriginalType: "pg:real"}, - "d": {OriginalType: "pg:double precision"}, - "c": {OriginalType: "pg:character(1)"}, - "str": {OriginalType: "pg:character varying(256)"}, - "character_": {OriginalType: "pg:character(4)"}, - "character_varying_": {OriginalType: "pg:character varying(5)"}, - "timestamptz_": {OriginalType: "pg:timestamp with time zone"}, - "tst": {OriginalType: "pg:timestamp with time zone"}, - "timetz_": {OriginalType: "pg:time with time zone"}, - "time_with_time_zone_": {OriginalType: "pg:time with time zone"}, - "iv": {OriginalType: "pg:interval"}, - "ba": {OriginalType: "pg:bytea"}, - "j": {OriginalType: "pg:json"}, - "jb": {OriginalType: "pg:jsonb"}, - "x": {OriginalType: "pg:xml"}, - "uid": {OriginalType: "pg:uuid"}, - "pt": {OriginalType: "pg:point"}, - "it": {OriginalType: "pg:inet"}, - "int4range_": {OriginalType: "pg:int4range"}, - "int8range_": {OriginalType: "pg:int8range"}, - "numrange_": {OriginalType: "pg:numrange"}, - "tsrange_": {OriginalType: "pg:tsrange"}, - "tstzrange_": {OriginalType: "pg:tstzrange"}, - "daterange_": {OriginalType: "pg:daterange"}, - "f": {OriginalType: "pg:double precision"}, - "t": {OriginalType: "pg:text"}, - "date_": {OriginalType: "pg:date"}, - "time_": {OriginalType: "pg:time without time zone"}, - "time1": {OriginalType: "pg:time(1) without time zone"}, - "time6": {OriginalType: "pg:time(6) without time zone"}, - "timetz__": {OriginalType: "pg:time with time zone"}, - "timetz1": {OriginalType: "pg:time with time zone"}, - "timetz6": {OriginalType: "pg:time with time zone"}, - "timestamp1": {OriginalType: "pg:timestamp(1) without time zone"}, - "timestamp6": {OriginalType: "pg:timestamp(6) without time zone"}, - "timestamp": {OriginalType: "pg:timestamp without time zone"}, - "numeric_": {OriginalType: "pg:numeric"}, - "numeric_5": {OriginalType: "pg:numeric"}, - "numeric_5_2": {OriginalType: "pg:numeric"}, - "decimal_": {OriginalType: "pg:numeric"}, - "decimal_5": {OriginalType: "pg:numeric"}, - "decimal_5_2": {OriginalType: "pg:numeric"}, - "money_": {OriginalType: "pg:money"}, - "hstore_": {OriginalType: "pg:hstore"}, - "inet_": {OriginalType: "pg:inet"}, - "cidr_": {OriginalType: "pg:cidr"}, - "macaddr_": {OriginalType: "pg:macaddr"}, - "citext_": {OriginalType: "pg:citext"}, - }, - } - receiver := debezium.NewReceiver(originalTypes, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) - require.Equal(t, 2, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_source/dump.sql deleted file mode 100644 index a3039795..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_source/dump.sql +++ /dev/null @@ -1,213 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, - ss smallserial, - int integer, - aid serial, - id bigint, - bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - 1, - -8388605, - 0, - 1, - 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_external/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go deleted file mode 100644 index bc7a5725..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/check_db_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -var insertStmt = ` -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, - -- 1, - -8388605, - -- 0, - 1, - -- 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 2, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "false", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - transfer.Src.(*pgcommon.PgSource).PreSteps.Table = false - transfer.Src.(*pgcommon.PgSource).PreSteps.PrimaryKey = false - transfer.Dst.(*pgcommon.PgDestination).MaintainTables = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithoutCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - helpers.Activate(t, transfer) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "basic_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 2)) - require.Equal(t, 2, serde.CountOfProcessedMessage) -} - -// Target schema: -// create table if not exists "public"."basic_types" ( -// "i" integer, -// "bl" boolean, -// "b" boolean, -// "b8" bytea, -// "vb" bytea, -// "si" smallint, -// "int" integer, -// "id" bigint, -// "oid_" bigint, -// "real_" double precision, -// "d" double precision, -// "c" text, -// "str" text, -// "character_" text, -// "character_varying_" text, -// "timestamptz_" text, -// "tst" text, -// "timetz_" text, -// "time_with_time_zone_" text, -// "iv" bigint, -// "ba" bytea, -// "j" text, -// "jb" text, -// "x" text, -// "uid" text, -// "pt" text, -// "it" text, -// "int4range_" text, -// "int8range_" text, -// "numrange_" text, -// "tsrange_" text, -// "tstzrange_" text, -// "daterange_" text, -// "f" double precision, -// "t" text, -// "date_" integer, -// "time_" bigint, -// "time1" integer, -// "time6" bigint, -// "timetz__" text, -// "timetz1" text, -// "timetz6" text, -// "timestamp1" bigint, -// "timestamp6" bigint, -// "timestamp" bigint, -// "numeric_" double precision, -// "numeric_5" text, -// "numeric_5_2" text, -// "decimal_" double precision, -// "decimal_5" text, -// "decimal_5_2" text, -// "money_" text, -// "hstore_" text, -// "inet_" text, -// "cidr_" text, -// "macaddr_" text, -// "citext_" text, -// primary key (i) -//) diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_source/dump.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_source/dump.sql deleted file mode 100644 index 56bf4cd2..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_source/dump.sql +++ /dev/null @@ -1,213 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; - -CREATE TABLE public.basic_types -( - bl boolean, - b bit(1), - b8 bit(8), - vb varbit(8), - - si smallint, --- ss smallserial, - int integer, --- aid serial, - id bigint, --- bid bigserial, - oid_ oid, - - real_ real, - d double precision, - - c char, - str varchar(256), - - CHARACTER_ CHARACTER(4), - CHARACTER_VARYING_ CHARACTER VARYING(5), - TIMESTAMPTZ_ TIMESTAMPTZ, -- timestamptz is accepted as an abbreviation for timestamp with time zone; this is a PostgreSQL extension - tst TIMESTAMP WITH TIME ZONE, - TIMETZ_ TIMETZ, - TIME_WITH_TIME_ZONE_ TIME WITH TIME ZONE, - iv interval, - ba bytea, - - j json, - jb jsonb, - x xml, - - uid uuid, - pt point, - it inet, - INT4RANGE_ INT4RANGE, - INT8RANGE_ INT8RANGE, - NUMRANGE_ NUMRANGE, - TSRANGE_ TSRANGE, - TSTZRANGE_ TSTZRANGE, - DATERANGE_ DATERANGE, - -- ENUM - - -- add, from our /Users/timmyb32r/arc/arcadia/transfer_manager/go/tests/e2e/pg2pg/debezium/replication/dump/type_check.sql: - f float, - i int PRIMARY KEY, - t text, - - -- ---------------------------------------------------------------------------------------------------------------- - - DATE_ DATE, - TIME_ TIME, - TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - TIME6 TIME(6), - - TIMETZ__ TIME WITH TIME ZONE, - TIMETZ1 TIME(1) WITH TIME ZONE, - TIMETZ6 TIME(6) WITH TIME ZONE, - - TIMESTAMP1 TIMESTAMP(1), - TIMESTAMP6 TIMESTAMP(6), - TIMESTAMP TIMESTAMP, - - --NUMERIC(precision) # selects a scale of 0 - --NUMERIC(precision, scale) - -- 'numeric' type - it's bignum - -- precision - digits in the whole number, that is, the number of digits to both sides of the decimal point - -- scale - count of decimal digits in the fractional part, to the right of the decimal point - -- - -- example: So the number 23.5141 has a precision of 6 and a scale of 4. Integers can be considered to have a scale of zero - -- In addition to ordinary numeric values, the numeric type has several special values: - -- Infinity - -- -Infinity - -- NaN - NUMERIC_ NUMERIC, - NUMERIC_5 NUMERIC(5), - NUMERIC_5_2 NUMERIC(5,2), - - --DECIMAL - -- The types decimal and numeric are equivalent - DECIMAL_ DECIMAL, - DECIMAL_5 DECIMAL(5), - DECIMAL_5_2 DECIMAL(5,2), - - --MONEY - -- The money type stores a currency amount with a fixed fractional precision - -- [local] =# CREATE TABLE money_example (cash money); - -- [local] =# INSERT INTO money_example VALUES ('$99.99'); - -- [local] =# INSERT INTO money_example VALUES (99.99); - -- [local] =# INSERT INTO money_example VALUES (99.98996998); - MONEY_ MONEY, - - HSTORE_ HSTORE, - INET_ INET, - CIDR_ CIDR, - MACADDR_ MACADDR, - -- MACADDR8 not supported by postgresql 9.6 (which is in our recipes) - -- LTREE - should be in special table, I suppose - CITEXT_ CITEXT -); - -INSERT INTO public.basic_types VALUES ( - true, - b'1', - b'10101111', - b'10101110', - - -32768, --- 1, - -8388605, --- 0, - 1, --- 3372036854775807, - 2, - - 1.45e-10, - 3.14e-100, - - '1', - 'varchar_example', - - 'abcd', - 'varc', - '2004-10-19 10:23:54+02', - '2004-10-19 11:23:54+02', - '00:51:02.746572-08', - '00:51:02.746572-08', - interval '1 day 01:00:00', - decode('CAFEBABE', 'hex'), - - '{"k1": "v1"}', - '{"k2": "v2"}', - 'bar', - - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - point(23.4, -44.5), - '192.168.100.128/25', - '[3,7)'::int4range, - '[3,7)'::int8range, - numrange(1.9,1.91), - '[2010-01-02 10:00, 2010-01-02 11:00)', - '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange, - daterange('2000-01-10'::date, '2000-01-20'::date, '[]'), - - 1.45e-10, - 1, - 'text_example', - - -- ---------------------------------------------------------------------------------------------------------------- - - -- DATE_ DATE, - 'January 8, 1999', - - -- TIME_ TIME, - -- TIME1 TIME(1), -- precision: This is a fractional digits number placed in the seconds’ field. This can be up to six digits. HH:MM:SS.pppppp - -- TIME6 TIME(6), - '04:05:06', - '04:05:06.1', - '04:05:06.123456', - - -- TIMETZ__ TIME WITH TIME ZONE, - -- TIMETZ1 TIME(1) WITH TIME ZONE, - -- TIMETZ6 TIME(6) WITH TIME ZONE, - '2020-05-26 13:30:25-04', - '2020-05-26 13:30:25.5-04', - '2020-05-26 13:30:25.575401-04', - - -- TIMESTAMP1 TIMESTAMP(1), - -- TIMESTAMP6 TIMESTAMP(6), - -- TIMESTAMP TIMESTAMP, - '2004-10-19 10:23:54.9', - '2004-10-19 10:23:54.987654', - '2004-10-19 10:23:54', - - -- - -- NUMERIC_ NUMERIC, - -- NUMERIC_5 NUMERIC(5), - -- NUMERIC_5_2 NUMERIC(5,2), - 1267650600228229401496703205376, - 12345, - 123.67, - - -- DECIMAL_ DECIMAL, - -- DECIMAL_5 DECIMAL(5), - -- DECIMAL_5_2 DECIMAL(5,2), - 123456, - 12345, - 123.67, - - -- MONEY_ MONEY, - 99.98, - - -- HSTORE_ HSTORE, - 'a=>1,b=>2', - - -- INET_ INET, - '192.168.1.5', - - -- CIDR_ CIDR, - '10.1/16', - - -- MACADDR_ MACADDR, - '08:00:2b:01:02:03', - - -- CITEXT_ CITEXT - 'Tom' -); diff --git a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_target/init.sql b/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_target/init.sql deleted file mode 100644 index ace79d51..00000000 --- a/tests/e2e/pg2pg/debezium/all_datatypes_serde_via_debezium_not_enriched/init_target/init.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION ltree; -CREATE EXTENSION citext; diff --git a/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/check_db_test.go b/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/check_db_test.go deleted file mode 100644 index 004d5438..00000000 --- a/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/check_db_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget() -) - -var insertStmt = ` -INSERT INTO public.user_table VALUES (5,123,ARRAY ['VALUE_THREE','VALUE_ONE']::my_enum_type[]); -INSERT INTO public.user_table VALUES (6,321,null); - --- TODO: for this cases we need to update wal2json --- INSERT INTO public.double_precision_values VALUES (4,'-Infinity'); --- INSERT INTO public.double_precision_values VALUES (5,'Infinity'); --- INSERT INTO public.double_precision_values VALUES (6,'NaN'); -` - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), insertStmt) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "user_table", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 6)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams().WithPriorityComparators(helpers.PgDebeziumIgnoreTemporalAccuracyForArraysComparator))) - require.Equal(t, 6, serde.CountOfProcessedMessage) -} diff --git a/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/init_source/dump.sql b/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/init_source/dump.sql deleted file mode 100644 index 1ae70822..00000000 --- a/tests/e2e/pg2pg/debezium/double_precision_nan_inf_and_enum_arr_via_debezium/init_source/dump.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TYPE my_enum_type AS ENUM ( - 'VALUE_ONE', - 'VALUE_TWO', - 'VALUE_THREE' - ); - -CREATE TABLE public.user_table -( - i int PRIMARY KEY, - d double precision, - enum_arr my_enum_type[] - -); - -INSERT INTO public.user_table VALUES (1, 0,ARRAY []::my_enum_type[]); -INSERT INTO public.user_table VALUES (2, 'Infinity',ARRAY ['VALUE_ONE']::my_enum_type[]); -INSERT INTO public.user_table VALUES (3, 'NaN',ARRAY ['VALUE_TWO','VALUE_THREE']::my_enum_type[]); -INSERT INTO public.user_table VALUES (4, '-Infinity', ARRAY ['VALUE_TWO']::my_enum_type[]); - diff --git a/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go b/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go deleted file mode 100644 index cd339d07..00000000 --- a/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/check_db_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/debezium" - debeziumparameters "github.com/doublecloud/tross/pkg/debezium/parameters" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/doublecloud/tross/tests/helpers/serde" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshotAndIncrement(t *testing.T) { - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - )) - defer require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - - //--- - - emitter, err := debezium.NewMessagesEmitter(map[string]string{ - debeziumparameters.DatabaseDBName: "public", - debeziumparameters.TopicPrefix: "my_topic", - debeziumparameters.AddOriginalTypes: "true", - debeziumparameters.SourceType: "pg", - }, "1.1.2.Final", false, logger.Log) - require.NoError(t, err) - receiver := debezium.NewReceiver(nil, nil) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - transfer.Src.(*pgcommon.PgSource).NoHomo = true - - debeziumSerDeTransformer := helpers.NewSimpleTransformer(t, serde.MakeDebeziumSerDeUdfWithCheck(emitter, receiver), serde.AnyTablesUdf) - helpers.AddTransformer(t, transfer, debeziumSerDeTransformer) - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //--- - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - _, err = srcConn.Exec(context.Background(), ` - INSERT INTO public.numeric_types VALUES - ( - 2, - - -32768, -- t_smallint - -2147483648, -- t_integer - -9223372036854775808, -- t_bigint - - -0.01 - ); - - INSERT INTO public.numeric_types VALUES - ( - 3, - - 32767, -- t_smallint - 2147483647, -- t_integer - 9223372036854775807, -- t_bigint - - 0.01 - ); - `) - require.NoError(t, err) - - //--- - - require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "numeric_types", helpers.GetSampleableStorageByModel(t, Target), 60*time.Second, 4)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/init_source/dump.sql b/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/init_source/dump.sql deleted file mode 100644 index 06f7eb11..00000000 --- a/tests/e2e/pg2pg/debezium/num_limits_serde_via_debezium_embedded/init_source/dump.sql +++ /dev/null @@ -1,32 +0,0 @@ -create table if not exists public.numeric_types -( - i int PRIMARY KEY, - - t_smallint smallint, - t_integer integer, - t_bigint bigint, - - t_numeric_18_2 numeric(18,2) -); - -INSERT INTO public.numeric_types VALUES -( - 0, - - -32768, -- t_smallint - -2147483648, -- t_integer - -9223372036854775808, - - -0.01 -); - -INSERT INTO public.numeric_types VALUES -( - 1, - - 32767, -- t_smallint - 2147483647, -- t_integer - 9223372036854775807, -- t_bigint - - 0.01 -); diff --git a/tests/e2e/pg2pg/drop_tables/drop_test.go b/tests/e2e/pg2pg/drop_tables/drop_test.go deleted file mode 100644 index 96a3b96b..00000000 --- a/tests/e2e/pg2pg/drop_tables/drop_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - srcAll = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("SRC_")) - srcFilter = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("SRC_"), pgrecipe.WithDBTables("public.ids_1", "public.ids_2")) - srcNoViewAll = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump_1"), pgrecipe.WithPrefix("NOVIEW_SRC_")) - srcNoViewFilter = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump_1"), pgrecipe.WithPrefix("NOVIEW_SRC_"), pgrecipe.WithDBTables("public.items_1", "public.ids_1")) - - dstAllR = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("FULL_")) - dstFilterR = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("FILTER_")) - dstAllSR = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("FULL_S_")) - dstNoViewAllR = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("NOVIEW_FULL_")) - dstNoViewFilterR = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("NOVIEW_FILTER_")) - dstSelectiveR = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("FULL_SELECTIVE_")) -) - -const ( - existsT1Query = `SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'ids_1');` - existsT2Query = `SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'ids_2');` - existsT3Query = `SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'items_2');` - existsV1Query = `SELECT EXISTS(SELECT 1 FROM information_schema.views WHERE table_schema = 'public' AND table_name = 'spb_items_1_2020');` - existsS1Query = `SELECT EXISTS(SELECT 1 FROM information_schema.sequences WHERE sequence_schema = 'public' AND sequence_name = 'ids_1_seq');` - existsS2Query = `SELECT EXISTS(SELECT 1 FROM information_schema.sequences WHERE sequence_schema = 'public' AND sequence_name = 'items_1_seq');` - existsS3Query = `SELECT EXISTS(SELECT 1 FROM information_schema.sequences WHERE sequence_schema = 'public' AND sequence_name = 'ids_2_seq');` - existsS4Query = `SELECT EXISTS(SELECT 1 FROM information_schema.sequences WHERE sequence_schema = 'public' AND sequence_name = 'items_2_seq');` -) - -func init() { - _ = os.Setenv("YC", "1") -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source all", Port: srcAll.Port}, - helpers.LabeledPort{Label: "PG source noview", Port: srcNoViewAll.Port}, - helpers.LabeledPort{Label: "PG target all", Port: dstAllR.Port}, - helpers.LabeledPort{Label: "PG target filter", Port: dstFilterR.Port}, - helpers.LabeledPort{Label: "PG target filter snapshot", Port: dstAllSR.Port}, - helpers.LabeledPort{Label: "PG target noview", Port: dstNoViewAllR.Port}, - helpers.LabeledPort{Label: "PG target noview filter", Port: dstNoViewFilterR.Port}, - helpers.LabeledPort{Label: "PG target selective", Port: dstSelectiveR.Port}, - )) - }() - - srcAll.WithDefaults() - srcFilter.WithDefaults() - srcNoViewAll.WithDefaults() - srcNoViewFilter.WithDefaults() - dstAllR.WithDefaults() - dstFilterR.WithDefaults() - dstAllSR.WithDefaults() - dstNoViewAllR.WithDefaults() - dstNoViewFilterR.WithDefaults() - dstSelectiveR.WithDefaults() - - t.Run("DROP cleanup policy test", func(t *testing.T) { - t.Run("Drop all tables", DropAll) - t.Run("Drop filtered tables", DropFilter) - t.Run("Drop all tables in snapshot", DropAllSnapshotOnly) - t.Run("Drop all tables with no VIEW at source", DropNoViewAll) - t.Run("Drop filtered tables with no VIEW at source", DropNoViewFilter) - t.Run("Drop selective tables with dependent VIEW", DropSelective) - }) -} - -func DropAll(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &srcAll, &dstAllR, abstract.TransferTypeSnapshotAndIncrement) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.CleanupSinker(tables) - require.NoError(t, err) - - conn, err := postgres.MakeConnPoolFromDst(&dstAllR, logger.Log) - require.NoError(t, err) - defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var exists bool - require.NoError(t, conn.QueryRow(ctx, existsS1Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS2Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS3Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS4Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT1Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT2Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT3Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsV1Query).Scan(&exists)) - require.False(t, exists) -} - -func DropFilter(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &srcFilter, &dstFilterR, abstract.TransferTypeSnapshotAndIncrement) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.CleanupSinker(tables) - require.Error(t, err) - require.Contains(t, err.Error(), "cannot drop table ids_1 because other objects depend on it") - - conn, err := postgres.MakeConnPoolFromDst(&dstFilterR, logger.Log) - require.NoError(t, err) - defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var exists bool - require.NoError(t, conn.QueryRow(ctx, existsS1Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS2Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS3Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS4Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT1Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT2Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT3Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsV1Query).Scan(&exists)) - require.True(t, exists) -} - -func DropAllSnapshotOnly(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &srcAll, &dstAllSR, abstract.TransferTypeSnapshotOnly) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.CleanupSinker(tables) - require.NoError(t, err) - - conn, err := postgres.MakeConnPoolFromDst(&dstAllSR, logger.Log) - require.NoError(t, err) - defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var exists bool - require.NoError(t, conn.QueryRow(ctx, existsS1Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS2Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS3Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS4Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT1Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT2Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT3Query).Scan(&exists)) - require.False(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsV1Query).Scan(&exists)) - require.False(t, exists) -} - -func DropNoViewAll(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &srcNoViewAll, &dstNoViewAllR, abstract.TransferTypeSnapshotAndIncrement) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - // must not drop VIEW in target when it is absent in source - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.CleanupSinker(tables) - require.Error(t, err) - require.Contains(t, err.Error(), "failed dependent VIEWs check") - - conn, err := postgres.MakeConnPoolFromDst(&dstNoViewAllR, logger.Log) - require.NoError(t, err) - defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var exists bool - require.NoError(t, conn.QueryRow(ctx, existsS1Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS2Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS3Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS4Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT1Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT2Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT3Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsV1Query).Scan(&exists)) - require.True(t, exists) -} - -func DropNoViewFilter(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &srcNoViewFilter, &dstNoViewFilterR, abstract.TransferTypeSnapshotAndIncrement) - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - logger.Log.Infof("got tables: %v", tables) - - // must not drop VIEW in target when it is absent in source - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.CleanupSinker(tables) - require.Error(t, err) - require.Contains(t, err.Error(), "failed dependent VIEWs check") - - conn, err := postgres.MakeConnPoolFromDst(&dstNoViewFilterR, logger.Log) - require.NoError(t, err) - defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var exists bool - require.NoError(t, conn.QueryRow(ctx, existsS1Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS2Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS3Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsS4Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT1Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT2Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsT3Query).Scan(&exists)) - require.True(t, exists) - require.NoError(t, conn.QueryRow(ctx, existsV1Query).Scan(&exists)) - require.True(t, exists) -} - -func DropSelective(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &srcAll, &dstSelectiveR, abstract.TransferTypeSnapshotAndIncrement) - tables := abstract.TableMap{ - abstract.TableID{Namespace: "public", Name: "items_1"}: *new(abstract.TableInfo), - } - logger.Log.Infof("got tables: %v", tables) - - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err := snapshotLoader.CleanupSinker(tables) - require.NoError(t, err) - - dstStorage, err := postgres.NewStorage(dstSelectiveR.ToStorageParams()) - require.NoError(t, err) - defer dstStorage.Close() - - tablesAfterCleanup, err := server.FilteredTableList(dstStorage, transfer) - require.NoError(t, err) - - _, items1Exists := tablesAfterCleanup[abstract.TableID{Namespace: "public", Name: "items_1"}] - require.False(t, items1Exists) - - _, items1ViewExists := tablesAfterCleanup[abstract.TableID{Namespace: "public", Name: "spb_items_1_2020"}] - require.False(t, items1ViewExists) - - _, items2Exists := tablesAfterCleanup[abstract.TableID{Namespace: "public", Name: "items_2"}] - require.True(t, items2Exists) -} diff --git a/tests/e2e/pg2pg/drop_tables/dump/snapshot.sql b/tests/e2e/pg2pg/drop_tables/dump/snapshot.sql deleted file mode 100644 index c0241082..00000000 --- a/tests/e2e/pg2pg/drop_tables/dump/snapshot.sql +++ /dev/null @@ -1,41 +0,0 @@ -create table ids_1 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -create sequence ids_1_seq as int increment by 1 -owned by ids_1.id; - -create table items_1 ( - id int not null primary key, - item_id int not null references ids_1(id), - ts timestamp, - city varchar(100) -); -create sequence items_1_seq as int increment by 1 -owned by items_1.id; - -create table ids_2 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -create sequence ids_2_seq as int increment by 1 -owned by ids_2.id; - -create table items_2 ( - id int not null primary key, - item_id int not null references ids_2(id), - city varchar(100) -); -create sequence items_2_seq as int increment by 1 -owned by items_2.id; - -create view spb_items_1_2020 as - select * - from items_1 - where city = 'spb' and ts >= timestamp '2020-01-01 00:00:00'; diff --git a/tests/e2e/pg2pg/drop_tables/dump_1/snapshot.sql b/tests/e2e/pg2pg/drop_tables/dump_1/snapshot.sql deleted file mode 100644 index 842a0167..00000000 --- a/tests/e2e/pg2pg/drop_tables/dump_1/snapshot.sql +++ /dev/null @@ -1,18 +0,0 @@ -create table ids_1 ( - id int not null primary key, - - name varchar(40) not null, - description varchar(100) -); - -create sequence ids_1_seq as int increment by 1 -owned by ids_1.id; - -create table items_1 ( - id int not null primary key, - item_id int not null references ids_1(id), - ts timestamp, - city varchar(100) -); -create sequence items_1_seq as int increment by 1 -owned by items_1.id; diff --git a/tests/e2e/pg2pg/enum_with_fallbacks/check_db_test.go b/tests/e2e/pg2pg/enum_with_fallbacks/check_db_test.go deleted file mode 100644 index edd1a9bd..00000000 --- a/tests/e2e/pg2pg/enum_with_fallbacks/check_db_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package usertypes - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Target.Cleanup = server.DisabledCleanup - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func loadSnapshot(t *testing.T) { - Source.PreSteps.Constraint = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - _ = helpers.Activate(t, transfer) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -// This test is kind of tricky -// -// We haven't options to turn-off CopyUpload behaviour, but we need to test behaviour on homo-inserts (who runs after COPY insert failed) -// -// So, this test initializes 'dst' table by the same table_schema, that in the 'src'. -// And except this, initialization put in 'dst' one row (which is the same as one in 'src'). -// This leads to next behaviour: when COPY upload starts, COPY failed bcs of rows collision, and fallback into inserts - which successfully finished bcs of my fix. -// -// If run this test on trunk (before my fix) - it's failed. - -func TestUserTypes(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - loadSnapshot(t) -} diff --git a/tests/e2e/pg2pg/enum_with_fallbacks/init_dst/init.sql b/tests/e2e/pg2pg/enum_with_fallbacks/init_dst/init.sql deleted file mode 100644 index 2cbb1577..00000000 --- a/tests/e2e/pg2pg/enum_with_fallbacks/init_dst/init.sql +++ /dev/null @@ -1,6 +0,0 @@ -create type mcae as enum ('STRING', 'NUMBER', 'ENUM'); - -CREATE TABLE enums(i INT PRIMARY KEY, e mcae); - -INSERT INTO enums(i, e) VALUES -(1, 'STRING'); diff --git a/tests/e2e/pg2pg/enum_with_fallbacks/init_src/init.sql b/tests/e2e/pg2pg/enum_with_fallbacks/init_src/init.sql deleted file mode 100644 index 9bf5808b..00000000 --- a/tests/e2e/pg2pg/enum_with_fallbacks/init_src/init.sql +++ /dev/null @@ -1,8 +0,0 @@ -create type mcae as enum ('STRING', 'NUMBER', 'ENUM'); - -CREATE TABLE enums(i INT PRIMARY KEY, e mcae); - -INSERT INTO enums(i, e) VALUES -(1, 'STRING'), -(2, 'NUMBER'), -(3, 'ENUM'); diff --git a/tests/e2e/pg2pg/insufficient_privileges/check_db_test.go b/tests/e2e/pg2pg/insufficient_privileges/check_db_test.go deleted file mode 100644 index 42d22c75..00000000 --- a/tests/e2e/pg2pg/insufficient_privileges/check_db_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package insufficientprivileges - -import ( - "context" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func init() { - _ = pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) // to init test container -} - -func TestSnapshotWithEmptyTableListFails(t *testing.T) { - var emptyIncludeTables, emptyExcludeTables []string - transfer := helpers.MakeTransfer(helpers.TransferID, newSource(emptyIncludeTables, emptyExcludeTables), newTarget(), abstract.TransferTypeSnapshotOnly) - err := tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.Error(t, err) - require.Contains(t, err.Error(), "permission denied") -} - -func TestSnapshotWithAccessToPublicTableWorks(t *testing.T) { - includeTables := []string{"public.promiscuous"} - var emptyExcludeTables []string - for _, transferType := range []abstract.TransferType{abstract.TransferTypeSnapshotOnly, abstract.TransferTypeSnapshotAndIncrement} { - transfer := helpers.MakeTransfer(helpers.TransferID, newSource(includeTables, emptyExcludeTables), newTarget(), transferType) - err := tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - } -} - -func TestSnapshotWithInsufficientPermissionsToSpecificTableFails(t *testing.T) { - includeTables := []string{"public.promiscuous", "public.secret"} - var emptyExcludeTables []string - for _, transferType := range []abstract.TransferType{abstract.TransferTypeSnapshotOnly, abstract.TransferTypeSnapshotAndIncrement} { - transfer := helpers.MakeTransfer(helpers.TransferID, newSource(includeTables, emptyExcludeTables), newTarget(), transferType) - err := tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.Error(t, err) - require.Contains(t, err.Error(), "Tables not found") - require.Contains(t, err.Error(), "public.secret") - } -} - -func TestAddTableInSource(t *testing.T) { - emptyIncludeTables := []string{} - excludeTables := []string{"public.secret"} // Activation will fail with error if we don't exclude this table - - // Activate - transfer := helpers.MakeTransfer(helpers.TransferID, newSource(emptyIncludeTables, excludeTables), newTarget(), abstract.TransferTypeSnapshotAndIncrement) - err := tasks.ActivateDelivery(context.Background(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - // Replication, first try - transfer.Dst.(*postgres.PgDestination).CopyUpload = false // :( - wrkr := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - defer wrkr.Stop() - runChannel := make(chan error) - go func() { runChannel <- wrkr.Run() }() - - // Wait until replication has started and transfers one row - insertOneRow(t, helpers.GetIntFromEnv("SOURCE_PG_LOCAL_PORT"), "public.promiscuous", tableRow{100, "100"}) - for rowCount(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.promiscuous") < 4 { - time.Sleep(time.Second) - } - require.Equal(t, 4, rowCount(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.promiscuous")) - - // Add table with one row to the source database and create an empty one in the target - createTable(t, helpers.GetIntFromEnv("SOURCE_PG_LOCAL_PORT"), "public.secret2") - createTable(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.secret2") - insertOneRow(t, helpers.GetIntFromEnv("SOURCE_PG_LOCAL_PORT"), "public.secret2", tableRow{100, "100"}) - - // Expect replication to fail now - err = <-runChannel - require.Error(t, err) - require.False(t, abstract.IsFatal(err)) - require.Equal(t, 0, rowCount(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.secret2")) - - // Replication, second try - wrkr = local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - defer wrkr.Stop() - - err = wrkr.Run() - require.Error(t, err) - require.False(t, abstract.IsFatal(err)) - require.Equal(t, 0, rowCount(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.secret2")) - - // Give access to the source table secret2 to loser and check that replication works after that - grantPrivileges(t, helpers.GetIntFromEnv("SOURCE_PG_LOCAL_PORT"), "public.secret2", "loser") - wrkr = local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - defer wrkr.Stop() - wrkr.Start() // Use asynchronous Start() instead of synchronous Run() to avoid blocking - for rowCount(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.secret2") == 0 { - time.Sleep(time.Second) - } - require.Equal(t, 1, rowCount(t, helpers.GetIntFromEnv("TARGET_PG_LOCAL_PORT"), "public.secret2")) -} diff --git a/tests/e2e/pg2pg/insufficient_privileges/init_source/init.sql b/tests/e2e/pg2pg/insufficient_privileges/init_source/init.sql deleted file mode 100644 index 29cb6f29..00000000 --- a/tests/e2e/pg2pg/insufficient_privileges/init_source/init.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE USER loser WITH PASSWORD '123'; - -CREATE TABLE public.promiscuous ( - id integer PRIMARY KEY, - value text -); -INSERT INTO promiscuous VALUES (1, '1'), (2, '2'), (3, '3'); - -CREATE TABLE public.secret ( - id integer PRIMARY KEY, - value text -); - -REVOKE ALL ON ALL TABLES IN SCHEMA public FROM public, loser; -GRANT ALL PRIVILEGES ON TABLE public.promiscuous TO loser; -ALTER ROLE loser WITH REPLICATION; - -INSERT INTO public.secret VALUES (11, '11'), (22, '22'), (33, '33'); diff --git a/tests/e2e/pg2pg/insufficient_privileges/util.go b/tests/e2e/pg2pg/insufficient_privileges/util.go deleted file mode 100644 index 0bf1a6c1..00000000 --- a/tests/e2e/pg2pg/insufficient_privileges/util.go +++ /dev/null @@ -1,82 +0,0 @@ -package insufficientprivileges - -import ( - "context" - "fmt" - "testing" - - "github.com/doublecloud/tross/internal/core/log" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/jackc/pgx/v4" - "github.com/stretchr/testify/require" -) - -func newSource(includeTables, excludeTables []string) *pgcommon.PgSource { - source := pgrecipe.RecipeSource() - source.User = "loser" - source.Password = "123" - source.DBTables = includeTables - source.ExcludedTables = excludeTables - source.SlotID = "" - return source -} - -func newTarget() *pgcommon.PgDestination { - return pgrecipe.RecipeTarget() -} - -type tableRow struct { - id int - value string -} - -func makeConnConfig(dbPort int) *pgx.ConnConfig { - config, _ := pgx.ParseConfig("") - config.Port = uint16(dbPort) - - source := pgrecipe.RecipeSource() - config.Host = "localhost" - config.Database = source.Database - config.User = source.User - config.Password = string(source.Password) - config.PreferSimpleProtocol = true - - return config -} - -func exec(t *testing.T, dbPort int, query string, params ...interface{}) { - var logger log.Logger = nil - connPool, err := pgcommon.NewPgConnPool(makeConnConfig(dbPort), logger) - require.NoError(t, err) - - _, err = connPool.Exec(context.Background(), query, params...) - require.NoError(t, err) -} - -func queryRow(t *testing.T, dbPort int, query string, outValue interface{}) { - var logger log.Logger = nil - connPool, err := pgcommon.NewPgConnPool(makeConnConfig(dbPort), logger) - require.NoError(t, err) - - err = connPool.QueryRow(context.Background(), query).Scan(outValue) - require.NoError(t, err) -} - -func createTable(t *testing.T, dbPort int, tableName string) { - exec(t, dbPort, fmt.Sprintf(`CREATE TABLE %s (id INTEGER PRIMARY KEY, value TEXT)`, tableName)) -} - -func insertOneRow(t *testing.T, dbPort int, tableName string, row tableRow) { - exec(t, dbPort, fmt.Sprintf(`INSERT INTO %s VALUES ($1, $2)`, tableName), row.id, row.value) -} - -func rowCount(t *testing.T, dbPort int, tableName string) int { - var rowCount int - queryRow(t, dbPort, fmt.Sprintf(`SELECT COUNT(*) FROM %s`, tableName), &rowCount) - return rowCount -} - -func grantPrivileges(t *testing.T, dbPort int, tableName, userName string) { - exec(t, dbPort, fmt.Sprintf(`GRANT ALL PRIVILEGES ON TABLE %s TO %s;`, tableName, userName)) -} diff --git a/tests/e2e/pg2pg/jsonb/check_db_test.go b/tests/e2e/pg2pg/jsonb/check_db_test.go deleted file mode 100644 index c65dd0c3..00000000 --- a/tests/e2e/pg2pg/jsonb/check_db_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package usertypes - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func loadSnapshot(t *testing.T) { - Source.PreSteps.Constraint = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - tables, err := tasks.ObtainAllSrcTables(transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - snapshotLoader := tasks.NewSnapshotLoader(coordinator.NewFakeClient(), "test-operation", transfer, helpers.EmptyRegistry()) - err = snapshotLoader.UploadTables(context.TODO(), tables.ConvertToTableDescriptions(), true) - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func checkReplicationWorks(t *testing.T) { - transfer := server.Transfer{ - ID: "test_id", - Src: &Source, - Dst: &Target, - Type: abstract.TransferTypeSnapshotAndIncrement, - } - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - worker := local.NewLocalWorker(coordinator.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - worker.Start() - defer worker.Stop() - - _, err = srcConn.Exec(context.Background(), `INSERT INTO testtable VALUES (5, '{"k5": {"k55": {"val55": 5}}}')`) - require.NoError(t, err) - require.NoError(t, helpers.WaitStoragesSynced(t, Source, Target, 50, helpers.NewCompareStorageParams())) -} - -func TestUserTypes(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - Target.CopyUpload = false - loadSnapshot(t) - checkReplicationWorks(t) -} diff --git a/tests/e2e/pg2pg/jsonb/init_source/init.sql b/tests/e2e/pg2pg/jsonb/init_source/init.sql deleted file mode 100644 index 82a49914..00000000 --- a/tests/e2e/pg2pg/jsonb/init_source/init.sql +++ /dev/null @@ -1,14 +0,0 @@ -BEGIN; -create table testtable ( - id integer primary key, - val jsonb -); -insert into testtable (id, val) values (1, '{"key1": "v1"}'); -insert into testtable (id, val) values (2, '{"key2": 2}'); -insert into testtable (id, val) values (3, '{"key3": "''"}'); -insert into testtable (id, val) values (4, '{"key4": "\""}'); -COMMIT; - -BEGIN; -SELECT pg_create_logical_replication_slot('testslot', 'wal2json'); -COMMIT; diff --git a/tests/e2e/pg2pg/jsonb/init_target/init.sql b/tests/e2e/pg2pg/jsonb/init_target/init.sql deleted file mode 100644 index e39b079c..00000000 --- a/tests/e2e/pg2pg/jsonb/init_target/init.sql +++ /dev/null @@ -1,6 +0,0 @@ -BEGIN; -create table testtable ( - id integer primary key, - val jsonb -); -COMMIT; diff --git a/tests/e2e/pg2pg/multiindex/check_db_test.go b/tests/e2e/pg2pg/multiindex/check_db_test.go deleted file mode 100644 index 2385f030..00000000 --- a/tests/e2e/pg2pg/multiindex/check_db_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package multiindex - -import ( - "context" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source"), pgrecipe.WithDBTables("public.test")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestMultiindexBasic(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - dstConn, err := pgcommon.MakeConnPoolFromDst(&Target, logger.Log) - require.NoError(t, err) - defer dstConn.Close() - - _, err = srcConn.Exec(context.Background(), `SELECT pg_create_logical_replication_slot('testslot', 'wal2json')`) - require.NoError(t, err) - defer func() { - _, err := srcConn.Exec(context.Background(), `SELECT pg_drop_replication_slot('testslot')`) - require.NoError(t, err) - }() - - _, err = srcConn.Exec(context.Background(), ` - INSERT INTO test VALUES (1, 777, 'a'); -- {1: (777, 'a')} - DELETE FROM test WHERE aid = 1; -- {} - INSERT INTO test VALUES (2, 777, 'b'); -- {2: (777, 'b')} - -- Target database is here - INSERT INTO test VALUES (3, 888, 'c'); -- {2: (777, 'b'), 3: (888, 'c')} - `) - require.NoError(t, err) - defer func() { - _, err := srcConn.Exec(context.Background(), `DELETE FROM test;`) - require.NoError(t, err) - }() - - _, err = dstConn.Exec(context.Background(), `INSERT INTO test VALUES (2, 777, 'b')`) - require.NoError(t, err) - defer func() { - _, err := dstConn.Exec(context.Background(), `DELETE FROM test;`) - require.NoError(t, err) - }() - - worker := local.NewLocalWorker(coordinator.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - worker.Start() - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - err = worker.Stop() - require.NoError(t, err) - - var aid, bid int - var value string - err = dstConn.QueryRow(context.Background(), `SELECT aid, bid, value FROM test WHERE aid = 2`).Scan(&aid, &bid, &value) - require.NoError(t, err) - require.Equal(t, 2, aid) - require.Equal(t, 777, bid) - require.Equal(t, "b", value) - - err = dstConn.QueryRow(context.Background(), `SELECT aid, bid, value FROM test WHERE aid = 3`).Scan(&aid, &bid, &value) - require.NoError(t, err) - require.Equal(t, 3, aid) - require.Equal(t, 888, bid) - require.Equal(t, "c", value) -} - -func TestMultiindexPkeyChange(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - Target.PerTransactionPush = true // in per table mode result depends on collapse and so may flap - - transfer := server.Transfer{ - ID: "test_id", - Src: &Source, - Dst: &Target, - } - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - dstConn, err := pgcommon.MakeConnPoolFromDst(&Target, logger.Log) - require.NoError(t, err) - defer dstConn.Close() - - _, err = srcConn.Exec(context.Background(), `SELECT pg_create_logical_replication_slot('testslot', 'wal2json')`) - require.NoError(t, err) - defer func() { - _, err := srcConn.Exec(context.Background(), `SELECT pg_drop_replication_slot('testslot')`) - require.NoError(t, err) - }() - - _, err = srcConn.Exec(context.Background(), ` - INSERT INTO test VALUES (1, 777, 'a'); -- {1: (777, 'a')} - UPDATE test SET aid = 2, bid = 888 WHERE aid = 1; -- {2: (888, 'a')} - UPDATE test SET bid = 999 WHERE aid = 2; -- {2: (999, 'a')} - INSERT INTO test VALUES (3, 888, 'b'); -- {2: (999, 'a'), 3: (888, 'b')} - -- Target database is here - `) - require.NoError(t, err) - defer func() { - _, err := srcConn.Exec(context.Background(), `DELETE FROM test;`) - require.NoError(t, err) - }() - - _, err = dstConn.Exec(context.Background(), ` - INSERT INTO test VALUES (2, 999, 'a'); - INSERT INTO test VALUES (3, 888, 'b'); - `) - require.NoError(t, err) - defer func() { - _, err := dstConn.Exec(context.Background(), `DELETE FROM test;`) - require.NoError(t, err) - }() - - worker := local.NewLocalWorker(coordinator.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - err = worker.Run() - require.Error(t, err) - require.Contains(t, strings.ToLower(err.Error()), "duplicate key value violates unique constraint") - err = worker.Stop() - require.NoError(t, err) -} diff --git a/tests/e2e/pg2pg/multiindex/init_source/dump.sql b/tests/e2e/pg2pg/multiindex/init_source/dump.sql deleted file mode 100644 index da656228..00000000 --- a/tests/e2e/pg2pg/multiindex/init_source/dump.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE test ( - aid integer PRIMARY KEY, - bid integer, - value text -); -CREATE UNIQUE INDEX uindex ON test (bid); diff --git a/tests/e2e/pg2pg/multiindex/init_target/dump.sql b/tests/e2e/pg2pg/multiindex/init_target/dump.sql deleted file mode 100644 index da656228..00000000 --- a/tests/e2e/pg2pg/multiindex/init_target/dump.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE test ( - aid integer PRIMARY KEY, - bid integer, - value text -); -CREATE UNIQUE INDEX uindex ON test (bid); diff --git a/tests/e2e/pg2pg/namesake_tables/check_db_test.go b/tests/e2e/pg2pg/namesake_tables/check_db_test.go deleted file mode 100644 index 42dd6772..00000000 --- a/tests/e2e/pg2pg/namesake_tables/check_db_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package snapshot - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithDBTables("public.__test")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - helpers.Activate(t, transfer) - helpers.CheckRowsCount(t, Target, "public", "__test", 1) -} diff --git a/tests/e2e/pg2pg/namesake_tables/dump/type_check.sql b/tests/e2e/pg2pg/namesake_tables/dump/type_check.sql deleted file mode 100644 index 1b76ce26..00000000 --- a/tests/e2e/pg2pg/namesake_tables/dump/type_check.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE public.__test ( - id INT PRIMARY KEY, - valA TEXT -); - -INSERT INTO public.__test (id,valA) VALUES (1,'blablabla'); - --- - -CREATE SCHEMA public2; - -CREATE TABLE public2.__test ( - id INT PRIMARY KEY, - valB TEXT -); diff --git a/tests/e2e/pg2pg/null_temporals_tsv_1/check_db_test.go b/tests/e2e/pg2pg/null_temporals_tsv_1/check_db_test.go deleted file mode 100644 index dabf9a62..00000000 --- a/tests/e2e/pg2pg/null_temporals_tsv_1/check_db_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package usertypes - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump")) - Target = pgrecipe.RecipeTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotOnly) - transfer.TypeSystemVersion = 1 - - _ = helpers.Activate(t, transfer) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/null_temporals_tsv_1/dump/dump.sql b/tests/e2e/pg2pg/null_temporals_tsv_1/dump/dump.sql deleted file mode 100644 index 6b3458dd..00000000 --- a/tests/e2e/pg2pg/null_temporals_tsv_1/dump/dump.sql +++ /dev/null @@ -1,7 +0,0 @@ -create table testtable ( - id integer primary key, - val1 timestamp without time zone, - val2 timestamp with time zone, - val3 date -); -insert into testtable values (1, NULL, NULL, NULL); diff --git a/tests/e2e/pg2pg/partitioned_tables/all_parts/dump/initial.sql b/tests/e2e/pg2pg/partitioned_tables/all_parts/dump/initial.sql deleted file mode 100644 index 13ea4f11..00000000 --- a/tests/e2e/pg2pg/partitioned_tables/all_parts/dump/initial.sql +++ /dev/null @@ -1,105 +0,0 @@ -CREATE TABLE measurement_inherited ( - id int not null, - logdate date not null, - unitsales int, - PRIMARY KEY (id, logdate) -); - -CREATE TABLE measurement_inherited_y2006m02 ( - CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -) INHERITS (measurement_inherited); - -CREATE TABLE measurement_inherited_y2006m03 ( - CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -) INHERITS (measurement_inherited); - -CREATE TABLE measurement_inherited_y2006m04 ( - CHECK ( logdate >= DATE '2006-04-01' AND logdate < DATE '2006-05-01' ) -) INHERITS (measurement_inherited); - -ALTER TABLE measurement_inherited_y2006m02 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_inherited_y2006m03 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_inherited_y2006m04 ADD PRIMARY KEY (id, logdate); - -CREATE RULE measurement_inherited_insert_y2006m02 AS -ON INSERT TO measurement_inherited WHERE - ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m02 VALUES (NEW.*); - -CREATE RULE measurement_inherited_insert_y2006m03 AS -ON INSERT TO measurement_inherited WHERE - ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m03 VALUES (NEW.*); - -CREATE RULE measurement_inherited_insert_y2006m04 AS -ON INSERT TO measurement_inherited WHERE - ( logdate >= DATE '2006-04-01' AND logdate < DATE '2006-05-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m04 VALUES (NEW.*); - -INSERT INTO measurement_inherited(id, logdate, unitsales) -VALUES -(1, '2006-02-02', 1), -(2, '2006-02-02', 1), -(3, '2006-03-03', 1), -(4, '2006-03-03', 1), -(5, '2006-03-03', 1), -(10, '2006-04-03', 1), -(11, '2006-04-03', 1), -(12, '2006-04-03', 1); - ---------------------------------------------------------------------------------- - -CREATE TABLE measurement_declarative ( - id int not null, - logdate date not null, - unitsales int -) PARTITION BY RANGE (logdate); - -CREATE TABLE measurement_declarative_y2006m02 PARTITION OF measurement_declarative - FOR VALUES FROM ('2006-02-01') TO ('2006-03-01'); -CREATE TABLE measurement_declarative_y2006m03 PARTITION OF measurement_declarative - FOR VALUES FROM ('2006-03-01') TO ('2006-04-01'); -CREATE TABLE measurement_declarative_y2006m04 PARTITION OF measurement_declarative - FOR VALUES FROM ('2006-04-01') TO ('2006-05-01'); - -CREATE TABLE measurement_declarative_y2006m05 ( - id int not null, - logdate date not null, - unitsales int -); - ---CREATE TABLE measurement_declarative_y2006m05 --- (LIKE measurement_declarative INCLUDING DEFAULTS INCLUDING CONSTRAINTS); -ALTER TABLE measurement_declarative_y2006m05 ADD CONSTRAINT constraint_y2006m05 - CHECK ( logdate >= DATE '2006-05-01' AND logdate < DATE '2006-06-01' ); - ---ALTER TABLE measurement_declarative ATTACH PARTITION measurement_declarative_y2006m05 --- FOR VALUES FROM ('2006-05-01') TO ('2006-06-01' ); - - -ALTER TABLE measurement_declarative_y2006m02 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m03 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m04 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m05 ADD PRIMARY KEY (id, logdate); - -INSERT INTO measurement_declarative(id, logdate, unitsales) -VALUES -(1, '2006-02-02', 1), -(2, '2006-02-02', 1), -(3, '2006-03-03', 1), -(4, '2006-03-03', 1), -(5, '2006-03-03', 1), -(10, '2006-04-03', 1), -(11, '2006-04-03', 1), -(12, '2006-04-03', 1); - -INSERT INTO measurement_declarative_y2006m05(id, logdate, unitsales) -VALUES -(21, '2006-05-01', 1), -(22, '2006-05-02', 1); - -ALTER TABLE measurement_declarative ATTACH PARTITION measurement_declarative_y2006m05 - FOR VALUES FROM ('2006-05-01') TO ('2006-06-01' ); diff --git a/tests/e2e/pg2pg/partitioned_tables/all_parts/partitioned_tables_test.go b/tests/e2e/pg2pg/partitioned_tables/all_parts/partitioned_tables_test.go deleted file mode 100644 index 1039e97e..00000000 --- a/tests/e2e/pg2pg/partitioned_tables/all_parts/partitioned_tables_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package replication - -import ( - "context" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - - TruncateSource = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.UseFakePrimaryKey = true - })) - TruncateTarget = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) - - DropSource = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.UseFakePrimaryKey = true - })) - DropTarget = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - TruncateTarget.Cleanup = server.Truncate - DropTarget.Cleanup = server.Drop - helpers.InitSrcDst(helpers.TransferID, &TruncateSource, &TruncateTarget, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID - helpers.InitSrcDst(helpers.TransferID, &DropSource, &DropTarget, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: TruncateSource.Port}, - helpers.LabeledPort{Label: "PG target", Port: TruncateTarget.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Verify", Verify) - t.Run("Load", Load) - }) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(TruncateSource.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(TruncateTarget.ToStorageParams()) - require.NoError(t, err) -} - -func Verify(t *testing.T) { - var transfer server.Transfer - transfer.Src = &DropSource - transfer.Dst = &DropTarget - transfer.Type = "SNAPSOT_AND_INCREMENT" - - err := tasks.VerifyDelivery(transfer, logger.Log, helpers.EmptyRegistry()) - require.NoError(t, err) - - dstStorage, err := postgres.NewStorage(DropTarget.ToStorageParams()) - require.NoError(t, err) - - var result bool - err = dstStorage.Conn.QueryRow(context.Background(), ` - SELECT EXISTS - ( - SELECT 1 - FROM pg_tables - WHERE schemaname = 'public' - AND tablename = '_ping' - ); - `).Scan(&result) - require.NoError(t, err) - require.Equal(t, false, result) -} - -func Load(t *testing.T) { - truncateTransfer := helpers.MakeTransfer(helpers.TransferID, &TruncateSource, &TruncateTarget, TransferType) - dropTransfer := helpers.MakeTransfer(helpers.TransferID, &DropSource, &DropTarget, TransferType) - - load(t, dropTransfer, true) - load(t, truncateTransfer, false) -} - -func load(t *testing.T, transfer *server.Transfer, updateSource bool) { - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - if updateSource { - pgSource := transfer.Src.(*postgres.PgSource) - srcStorage, err := postgres.NewStorage(pgSource.ToStorageParams(nil)) - require.NoError(t, err) - pushDataToStorage(t, srcStorage) - } - - //----------------------------------------------------------------------------------------------------------------- - - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_inherited", 10) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_inherited_y2006m02", 3) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_inherited_y2006m03", 4) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_inherited_y2006m04", 3) - - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative", 12) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m02", 3) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m03", 4) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m04", 3) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m05", 2) - - sourceStorage := helpers.GetSampleableStorageByModel(t, transfer.Src) - targetStorage := helpers.GetSampleableStorageByModel(t, transfer.Dst) - - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m02", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m03", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m04", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m02", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m03", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m04", sourceStorage, targetStorage, 60*time.Second)) - helpers.CheckRowsCount(t, transfer.Dst, "public", "measurement_inherited", 10) - helpers.CheckRowsCount(t, transfer.Dst, "public", "measurement_declarative", 12) - compareParams := helpers.NewCompareStorageParams() - compareParams.TableFilter = func(tables abstract.TableMap) []abstract.TableDescription { - return []abstract.TableDescription{ - abstract.TableDescription{ - Name: "measurement_inherited", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m02", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m03", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m04", - Schema: "public", - }, - //skip measurement_declarative because of turned UseFakePrimaryKey option on (limitation of outdated 10.5 PG version) - abstract.TableDescription{ - Name: "measurement_declarative_y2006m02", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_declarative_y2006m03", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_declarative_y2006m04", - Schema: "public", - }, - } - } - require.NoError(t, helpers.CompareStorages(t, transfer.Src, transfer.Dst, compareParams)) -} - -func pushDataToStorage(t *testing.T, storage *postgres.Storage) { - //----------------------------------------------------------------------------------------------------------------- - // update partitioned table created using inheritance directly - _, err := storage.Conn.Exec(context.Background(), ` - insert into measurement_inherited values - (6, '2006-02-02', 1), - (7, '2006-02-02', 1), - (8, '2006-03-02', 1); - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - update measurement_inherited - set logdate = '2006-02-10' - where id = 6; - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - update measurement_inherited - set logdate = '2006-02-20', id = 8 - where id = 7; - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - delete from measurement_inherited - where id = 1; - `) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - // update partitioned table created declarartively - _, err = storage.Conn.Exec(context.Background(), ` - insert into measurement_declarative values - (6, '2006-02-02', 1), - (7, '2006-02-02', 1), - (8, '2006-03-02', 1); - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - update measurement_declarative - set logdate = '2006-02-10' - where id = 6; - `) - require.NoError(t, err) - _, err = storage.Conn.Exec(context.Background(), ` - update measurement_declarative - set logdate = '2006-02-20', id = 8 - where id = 7; - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - delete from measurement_declarative - where id = 1; - `) - require.NoError(t, err) - -} diff --git a/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/dump/initial.sql b/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/dump/initial.sql deleted file mode 100644 index 95021881..00000000 --- a/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/dump/initial.sql +++ /dev/null @@ -1,107 +0,0 @@ -CREATE SCHEMA second_schema; - -CREATE TABLE second_schema.measurement_inherited ( - id int not null, - logdate date not null, - unitsales int, - PRIMARY KEY (id, logdate) -); - -CREATE TABLE measurement_inherited_y2006m02 ( - CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -) INHERITS (second_schema.measurement_inherited); - -CREATE TABLE measurement_inherited_y2006m03 ( - CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -) INHERITS (second_schema.measurement_inherited); - -CREATE TABLE second_schema.measurement_inherited_y2006m04 ( - CHECK ( logdate >= DATE '2006-04-01' AND logdate < DATE '2006-05-01' ) -) INHERITS (second_schema.measurement_inherited); - -ALTER TABLE measurement_inherited_y2006m02 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_inherited_y2006m03 ADD PRIMARY KEY (id, logdate); -ALTER TABLE second_schema.measurement_inherited_y2006m04 ADD PRIMARY KEY (id, logdate); - -CREATE RULE measurement_inherited_insert_y2006m02 AS -ON INSERT TO second_schema.measurement_inherited WHERE - ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m02 VALUES (NEW.*); - -CREATE RULE measurement_inherited_insert_y2006m03 AS -ON INSERT TO second_schema.measurement_inherited WHERE - ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m03 VALUES (NEW.*); - -CREATE RULE measurement_inherited_insert_y2006m04 AS -ON INSERT TO second_schema.measurement_inherited WHERE - ( logdate >= DATE '2006-04-01' AND logdate < DATE '2006-05-01' ) -DO INSTEAD - INSERT INTO second_schema.measurement_inherited_y2006m04 VALUES (NEW.*); - -INSERT INTO second_schema.measurement_inherited(id, logdate, unitsales) -VALUES -(1, '2006-02-02', 1), -(2, '2006-02-02', 1), -(3, '2006-03-03', 1), -(4, '2006-03-03', 1), -(5, '2006-03-03', 1), -(10, '2006-04-03', 1), -(11, '2006-04-03', 1), -(12, '2006-04-03', 1); - ---------------------------------------------------------------------------------- - -CREATE TABLE second_schema.measurement_declarative ( - id int not null, - logdate date not null, - unitsales int -) PARTITION BY RANGE (logdate); - -CREATE TABLE measurement_declarative_y2006m02 PARTITION OF second_schema.measurement_declarative - FOR VALUES FROM ('2006-02-01') TO ('2006-03-01'); -CREATE TABLE measurement_declarative_y2006m03 PARTITION OF second_schema.measurement_declarative - FOR VALUES FROM ('2006-03-01') TO ('2006-04-01'); -CREATE TABLE second_schema.measurement_declarative_y2006m04 PARTITION OF second_schema.measurement_declarative - FOR VALUES FROM ('2006-04-01') TO ('2006-05-01'); - -CREATE TABLE measurement_declarative_y2006m05 ( - id int not null, - logdate date not null, - unitsales int -); - ---CREATE TABLE measurement_declarative_y2006m05 --- (LIKE measurement_declarative INCLUDING DEFAULTS INCLUDING CONSTRAINTS); -ALTER TABLE measurement_declarative_y2006m05 ADD CONSTRAINT constraint_y2006m05 - CHECK ( logdate >= DATE '2006-05-01' AND logdate < DATE '2006-06-01' ); - ---ALTER TABLE measurement_declarative ATTACH PARTITION measurement_declarative_y2006m05 --- FOR VALUES FROM ('2006-05-01') TO ('2006-06-01' ); - - -ALTER TABLE measurement_declarative_y2006m02 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m03 ADD PRIMARY KEY (id, logdate); -ALTER TABLE second_schema.measurement_declarative_y2006m04 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m05 ADD PRIMARY KEY (id, logdate); - -INSERT INTO second_schema.measurement_declarative(id, logdate, unitsales) -VALUES -(1, '2006-02-02', 1), -(2, '2006-02-02', 1), -(3, '2006-03-03', 1), -(4, '2006-03-03', 1), -(5, '2006-03-03', 1), -(10, '2006-04-03', 1), -(11, '2006-04-03', 1), -(12, '2006-04-03', 1); - -INSERT INTO measurement_declarative_y2006m05(id, logdate, unitsales) -VALUES -(21, '2006-05-01', 1), -(22, '2006-05-02', 1); - -ALTER TABLE second_schema.measurement_declarative ATTACH PARTITION public.measurement_declarative_y2006m05 - FOR VALUES FROM ('2006-05-01') TO ('2006-06-01' ); diff --git a/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/partitioned_tables_test.go b/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/partitioned_tables_test.go deleted file mode 100644 index b23d400f..00000000 --- a/tests/e2e/pg2pg/partitioned_tables/all_parts_non_public_schema/partitioned_tables_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package replication - -import ( - "context" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - - TruncateSource = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.UseFakePrimaryKey = true - })) - TruncateTarget = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) - - DropSource = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.UseFakePrimaryKey = true - })) - DropTarget = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - TruncateTarget.Cleanup = server.Truncate - DropTarget.Cleanup = server.Drop - helpers.InitSrcDst(helpers.TransferID, &TruncateSource, &TruncateTarget, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID - helpers.InitSrcDst(helpers.TransferID, &DropSource, &DropTarget, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: TruncateSource.Port}, - helpers.LabeledPort{Label: "PG target", Port: TruncateTarget.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Verify", Verify) - t.Run("Load", Load) - }) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(TruncateSource.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(TruncateTarget.ToStorageParams()) - require.NoError(t, err) -} - -func Verify(t *testing.T) { - var transfer server.Transfer - transfer.Src = &DropSource - transfer.Dst = &DropTarget - transfer.Type = "SNAPSOT_AND_INCREMENT" - - err := tasks.VerifyDelivery(transfer, logger.Log, helpers.EmptyRegistry()) - require.NoError(t, err) - - dstStorage, err := postgres.NewStorage(DropTarget.ToStorageParams()) - require.NoError(t, err) - - var result bool - err = dstStorage.Conn.QueryRow(context.Background(), ` - SELECT EXISTS - ( - SELECT 1 - FROM pg_tables - WHERE schemaname = 'public' - AND tablename = '_ping' - ); - `).Scan(&result) - require.NoError(t, err) - require.Equal(t, false, result) -} - -func Load(t *testing.T) { - truncateTransfer := helpers.MakeTransfer(helpers.TransferID, &TruncateSource, &TruncateTarget, TransferType) - dropTransfer := helpers.MakeTransfer(helpers.TransferID, &DropSource, &DropTarget, TransferType) - - load(t, dropTransfer, true) - load(t, truncateTransfer, false) -} - -func load(t *testing.T, transfer *server.Transfer, updateSource bool) { - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - if updateSource { - pgSource := transfer.Src.(*postgres.PgSource) - srcStorage, err := postgres.NewStorage(pgSource.ToStorageParams(nil)) - require.NoError(t, err) - pushDataToStorage(t, srcStorage) - } - - //----------------------------------------------------------------------------------------------------------------- - - helpers.CheckRowsCount(t, transfer.Src, "second_schema", "measurement_inherited", 10) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_inherited_y2006m02", 3) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_inherited_y2006m03", 4) - helpers.CheckRowsCount(t, transfer.Src, "second_schema", "measurement_inherited_y2006m04", 3) - - helpers.CheckRowsCount(t, transfer.Src, "second_schema", "measurement_declarative", 12) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m02", 3) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m03", 4) - helpers.CheckRowsCount(t, transfer.Src, "second_schema", "measurement_declarative_y2006m04", 3) - helpers.CheckRowsCount(t, transfer.Src, "public", "measurement_declarative_y2006m05", 2) - - sourceStorage := helpers.GetSampleableStorageByModel(t, transfer.Src) - targetStorage := helpers.GetSampleableStorageByModel(t, transfer.Dst) - - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m02", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m03", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "second_schema", "measurement_inherited_y2006m04", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m02", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m03", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "second_schema", "measurement_declarative_y2006m04", sourceStorage, targetStorage, 60*time.Second)) - helpers.CheckRowsCount(t, transfer.Dst, "second_schema", "measurement_inherited", 10) - helpers.CheckRowsCount(t, transfer.Dst, "second_schema", "measurement_declarative", 12) - compareParams := helpers.NewCompareStorageParams() - compareParams.TableFilter = func(tables abstract.TableMap) []abstract.TableDescription { - return []abstract.TableDescription{ - abstract.TableDescription{ - Name: "measurement_inherited", - Schema: "second_schema", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m02", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m03", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m04", - Schema: "second_schema", - }, - //skip measurement_declarative because of turned UseFakePrimaryKey option on (limitation of outdated 10.5 PG version) - abstract.TableDescription{ - Name: "measurement_declarative_y2006m02", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_declarative_y2006m03", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_declarative_y2006m04", - Schema: "second_schema", - }, - } - } - require.NoError(t, helpers.CompareStorages(t, transfer.Src, transfer.Dst, compareParams)) -} - -func pushDataToStorage(t *testing.T, storage *postgres.Storage) { - //----------------------------------------------------------------------------------------------------------------- - // update partitioned table created using inheritance directly - _, err := storage.Conn.Exec(context.Background(), ` - insert into second_schema.measurement_inherited values - (6, '2006-02-02', 1), - (7, '2006-02-02', 1), - (8, '2006-03-02', 1); - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - update second_schema.measurement_inherited - set logdate = '2006-02-10' - where id = 6; - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - update second_schema.measurement_inherited - set logdate = '2006-02-20', id = 8 - where id = 7; - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - delete from second_schema.measurement_inherited - where id = 1; - `) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - // update partitioned table created declarartively - _, err = storage.Conn.Exec(context.Background(), ` - insert into second_schema.measurement_declarative values - (6, '2006-02-02', 1), - (7, '2006-02-02', 1), - (8, '2006-03-02', 1); - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - update second_schema.measurement_declarative - set logdate = '2006-02-10' - where id = 6; - `) - require.NoError(t, err) - _, err = storage.Conn.Exec(context.Background(), ` - update second_schema.measurement_declarative - set logdate = '2006-02-20', id = 8 - where id = 7; - `) - require.NoError(t, err) - - _, err = storage.Conn.Exec(context.Background(), ` - delete from second_schema.measurement_declarative - where id = 1; - `) - require.NoError(t, err) - -} diff --git a/tests/e2e/pg2pg/partitioned_tables/some_parts/dump/initial.sql b/tests/e2e/pg2pg/partitioned_tables/some_parts/dump/initial.sql deleted file mode 100644 index ddfe7058..00000000 --- a/tests/e2e/pg2pg/partitioned_tables/some_parts/dump/initial.sql +++ /dev/null @@ -1,81 +0,0 @@ -CREATE TABLE measurement_inherited ( - id int not null, - logdate date not null, - unitsales int, - PRIMARY KEY (id, logdate) -); - -CREATE TABLE measurement_inherited_y2006m02 ( - CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -) INHERITS (measurement_inherited); - -CREATE TABLE measurement_inherited_y2006m03 ( - CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -) INHERITS (measurement_inherited); - -CREATE TABLE measurement_inherited_y2006m04 ( - CHECK ( logdate >= DATE '2006-04-01' AND logdate < DATE '2006-05-01' ) -) INHERITS (measurement_inherited); - -ALTER TABLE measurement_inherited_y2006m02 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_inherited_y2006m03 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_inherited_y2006m04 ADD PRIMARY KEY (id, logdate); - -CREATE RULE measurement_inherited_insert_y2006m02 AS -ON INSERT TO measurement_inherited WHERE - ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m02 VALUES (NEW.*); - -CREATE RULE measurement_inherited_insert_y2006m03 AS -ON INSERT TO measurement_inherited WHERE - ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m03 VALUES (NEW.*); - -CREATE RULE measurement_inherited_insert_y2006m04 AS -ON INSERT TO measurement_inherited WHERE - ( logdate >= DATE '2006-04-01' AND logdate < DATE '2006-05-01' ) -DO INSTEAD - INSERT INTO measurement_inherited_y2006m04 VALUES (NEW.*); - -INSERT INTO measurement_inherited(id, logdate, unitsales) -VALUES -(1, '2006-02-02', 1), -(2, '2006-02-02', 1), -(3, '2006-03-03', 1), -(4, '2006-03-03', 1), -(5, '2006-03-03', 1), -(10, '2006-04-03', 1), -(11, '2006-04-03', 1), -(12, '2006-04-03', 1); - ---------------------------------------------------------------------------------- - -CREATE TABLE measurement_declarative ( - id int not null, - logdate date not null, - unitsales int -) PARTITION BY RANGE (logdate); - -CREATE TABLE measurement_declarative_y2006m02 PARTITION OF measurement_declarative - FOR VALUES FROM ('2006-02-01') TO ('2006-03-01'); -CREATE TABLE measurement_declarative_y2006m03 PARTITION OF measurement_declarative - FOR VALUES FROM ('2006-03-01') TO ('2006-04-01'); -CREATE TABLE measurement_declarative_y2006m04 PARTITION OF measurement_declarative - FOR VALUES FROM ('2006-04-01') TO ('2006-05-01'); - -ALTER TABLE measurement_declarative_y2006m02 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m03 ADD PRIMARY KEY (id, logdate); -ALTER TABLE measurement_declarative_y2006m04 ADD PRIMARY KEY (id, logdate); - -INSERT INTO measurement_declarative(id, logdate, unitsales) -VALUES -(1, '2006-02-02', 1), -(2, '2006-02-02', 1), -(3, '2006-03-03', 1), -(4, '2006-03-03', 1), -(5, '2006-03-03', 1), -(10, '2006-04-03', 1), -(11, '2006-04-03', 1), -(12, '2006-04-03', 1); diff --git a/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go b/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go deleted file mode 100644 index 612b5f16..00000000 --- a/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package replication - -import ( - "context" - "os" - "strconv" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix(""), pgrecipe.WithDBTables( - "public.measurement_inherited", - "public.measurement_inherited_y2006m02", - "public.measurement_inherited_y2006m04", - "public.measurement_declarative", - "public.measurement_declarative_y2006m02", - "public.measurement_declarative_y2006m04", - ), pgrecipe.WithEdit(func(pg *postgres.PgSource) { - pg.UseFakePrimaryKey = true - })) - dstPort, _ = strconv.Atoi(os.Getenv("DB0_PG_LOCAL_PORT")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Verify", Verify) - t.Run("Load", Load) - }) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Verify(t *testing.T) { - var transfer server.Transfer - transfer.Src = &Source - transfer.Dst = &Target - transfer.Type = "SNAPSOT_AND_INCREMENT" - - err := tasks.VerifyDelivery(transfer, logger.Log, helpers.EmptyRegistry()) - require.NoError(t, err) - - dstStorage, err := postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - var result bool - err = dstStorage.Conn.QueryRow(context.Background(), ` - SELECT EXISTS - ( - SELECT 1 - FROM pg_tables - WHERE schemaname = 'public' - AND tablename = '_ping' - ); - `).Scan(&result) - require.NoError(t, err) - require.Equal(t, false, result) -} - -func Load(t *testing.T) { - Source.PreSteps.Rule = false // if true then all rules will been tried to transfer even rules for excluded partitions - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - srcStorage, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - _, err = srcStorage.Conn.Exec(context.Background(), ` - insert into measurement_inherited values - (6, '2006-02-02', 1), - (7, '2006-02-02', 1), - (8, '2006-03-02', 1); - `) - require.NoError(t, err) - - _, err = srcStorage.Conn.Exec(context.Background(), ` - update measurement_inherited - set logdate = '2006-02-10' - where id = 6; - `) - require.NoError(t, err) - - _, err = srcStorage.Conn.Exec(context.Background(), ` - update measurement_inherited - set logdate = '2006-02-20', id = 8 - where id = 7; - `) - require.NoError(t, err) - - _, err = srcStorage.Conn.Exec(context.Background(), ` - delete from measurement_inherited - where id = 1; - `) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - _, err = srcStorage.Conn.Exec(context.Background(), ` - insert into measurement_declarative values - (6, '2006-02-02', 1), - (7, '2006-02-02', 1), - (8, '2006-03-02', 1); - `) - require.NoError(t, err) - - _, err = srcStorage.Conn.Exec(context.Background(), ` - update measurement_declarative - set logdate = '2006-02-10' - where id = 6; - `) - require.NoError(t, err) - - _, err = srcStorage.Conn.Exec(context.Background(), ` - update measurement_declarative - set logdate = '2006-02-20', id = 8 - where id = 7; - `) - require.NoError(t, err) - - _, err = srcStorage.Conn.Exec(context.Background(), ` - delete from measurement_declarative - where id = 1; - `) - require.NoError(t, err) - - //----------------------------------------------------------------------------------------------------------------- - - helpers.CheckRowsCount(t, Source, "public", "measurement_inherited", 10) - helpers.CheckRowsCount(t, Source, "public", "measurement_inherited_y2006m02", 3) - helpers.CheckRowsCount(t, Source, "public", "measurement_inherited_y2006m03", 4) - helpers.CheckRowsCount(t, Source, "public", "measurement_inherited_y2006m04", 3) - helpers.CheckRowsCount(t, Source, "public", "measurement_declarative", 10) - helpers.CheckRowsCount(t, Source, "public", "measurement_declarative_y2006m02", 3) - helpers.CheckRowsCount(t, Source, "public", "measurement_declarative_y2006m03", 4) - helpers.CheckRowsCount(t, Source, "public", "measurement_declarative_y2006m04", 3) - - sourceStorage := helpers.GetSampleableStorageByModel(t, Source) - targetStorage := helpers.GetSampleableStorageByModel(t, Target) - - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m02", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_inherited_y2006m04", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m02", sourceStorage, targetStorage, 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "measurement_declarative_y2006m04", sourceStorage, targetStorage, 60*time.Second)) - helpers.CheckRowsCount(t, Target, "public", "measurement_inherited", 6) - helpers.CheckRowsCount(t, Target, "public", "measurement_declarative", 6) - compareParams := helpers.NewCompareStorageParams() - compareParams.TableFilter = func(tables abstract.TableMap) []abstract.TableDescription { - return []abstract.TableDescription{ - abstract.TableDescription{ - Name: "measurement_inherited", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m02", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_inherited_y2006m04", - Schema: "public", - }, - //skip measurement_declarative because of turned UseFakePrimaryKey option on (limitation of outdated 10.5 PG version) - abstract.TableDescription{ - Name: "measurement_declarative_y2006m02", - Schema: "public", - }, - abstract.TableDescription{ - Name: "measurement_declarative_y2006m04", - Schema: "public", - }, - } - } - require.NoError(t, helpers.CompareStorages(t, Source, Target, compareParams)) -} diff --git a/tests/e2e/pg2pg/pg_dump/check_db_test.go b/tests/e2e/pg2pg/pg_dump/check_db_test.go deleted file mode 100644 index 596e4c54..00000000 --- a/tests/e2e/pg2pg/pg_dump/check_db_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package pgdump - -import ( - "os" - "strings" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - require.True(t, t.Run("Existence", Existence)) - require.True(t, t.Run("Snapshot", Snapshot)) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - Source.PreSteps.Constraint = true - Source.PreSteps.Trigger = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - // extract schema - itemsSource, err := postgres.ExtractPgDumpSchema(transfer) - require.NoError(t, err) - - // apply on target - require.NoError(t, postgres.ApplyPgDumpPreSteps(itemsSource, transfer, helpers.EmptyRegistry())) - - // make target a source and extract its schema - targetAsSource := postgres.PgSource{ - ClusterID: Target.ClusterID, - Hosts: Target.Hosts, - User: Target.User, - Password: Target.Password, - Database: Target.Database, - Port: Target.Port, - PgDumpCommand: Source.PgDumpCommand, - } - targetAsSource.WithDefaults() - backwardFakeTransfer := helpers.MakeTransfer(helpers.TransferID, &targetAsSource, &Target, abstract.TransferTypeSnapshotOnly) - itemsTarget, err := postgres.ExtractPgDumpSchema(backwardFakeTransfer) - require.NoError(t, err) - - // compare schemas - require.Less(t, 0, len(itemsSource)) - require.Equal(t, len(itemsSource), len(itemsTarget)) - setvalsCount := 0 - for i := 0; i < len(itemsSource); i++ { - require.Equal(t, itemsSource[i].Typ, itemsTarget[i].Typ) - require.Equal(t, itemsSource[i].Body, itemsTarget[i].Body) - if strings.Contains(itemsSource[i].Body, "setval(") { - setvalsCount += 1 - } - } - require.Equal(t, 2, setvalsCount, "The number of setval() calls for SEQUENCEs must be equal to the number of sequences in dump") - - // test extract dump with DBTables - // with custom types - Source.DBTables = []string{"santa.\"Ho-Ho-Ho\""} - transfer = helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - itemsSource, err = postgres.ExtractPgDumpSchema(transfer) - require.NoError(t, err) - - existEnumType := false - for _, i := range itemsSource { - if i.Typ == "TYPE" && i.Name == "my_enum" { - existEnumType = true - break - } - } - require.True(t, existEnumType) - - // without custom types - Source.DBTables = []string{"__test"} - transfer = helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - itemsSource, err = postgres.ExtractPgDumpSchema(transfer) - require.NoError(t, err) - - existEnumType = false - for _, i := range itemsSource { - if i.Typ == "TYPE" { - existEnumType = true - break - } - } - require.False(t, existEnumType) -} diff --git a/tests/e2e/pg2pg/pg_dump/dump/type_check.sql b/tests/e2e/pg2pg/pg_dump/dump/type_check.sql deleted file mode 100644 index 0cc28f6c..00000000 --- a/tests/e2e/pg2pg/pg_dump/dump/type_check.sql +++ /dev/null @@ -1,68 +0,0 @@ --- --- Name: __english_collation; Type: COLLATION; Schema: public; Owner: - --- - -CREATE COLLATION __english_collation (provider = libc, locale = 'en_US.UTF-8'); - --- --- Name: __test; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE __test ( - id integer, - name character varying(255) -); - --- --- Name: __name_changes(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION __name_changes() RETURNS trigger - LANGUAGE plpgsql - AS $$ -BEGIN - IF NEW.name <> OLD.name THEN - INSERT INTO __test(id,name) - VALUES(OLD.id,OLD.name); - END IF; - - RETURN NEW; -END; -$$; - --- --- Name: __test __name_changes_trigger; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER __name_changes_trigger BEFORE UPDATE ON __test FOR EACH ROW EXECUTE PROCEDURE __name_changes(); - --- --- Name: __test __test_policy; Type: POLICY; Schema: public; Owner: db_user --- - -CREATE POLICY __test_policy ON __test USING (((name)::text = 'test'::text)); - -CREATE VIEW __test_view AS SELECT id, name FROM __test WHERE id > 0; - -CREATE MATERIALIZED VIEW __test_materialized_view AS SELECT id, name FROM __test WHERE id < 0; - -CREATE SCHEMA santa; - -CREATE TYPE santa.my_enum AS ENUM ('RED', 'BLUE'); - -CREATE TABLE santa."Ho-Ho-Ho"(i SERIAL PRIMARY KEY, t TEXT, f FLOAT, j santa.my_enum); - -INSERT INTO santa."Ho-Ho-Ho"(t, f, j) VALUES ('merry', 1.0, 'BLUE'), ('Christmas', 2.0, 'RED'); - -CREATE SEQUENCE santa."Rudolf" START 2; - - -CREATE TABLE wide_boys(column_1 int, column_2 int) PARTITION BY RANGE (column_2); - --- first type of part attachment -CREATE TABLE wide_boys_part_1 PARTITION OF wide_boys FOR VALUES FROM (0) TO (10); - --- second type of part attachment -CREATE TABLE wide_boys_part_2 (column_1 int primary key, column_2 int); -ALTER TABLE wide_boys ATTACH PARTITION wide_boys_part_2 FOR VALUES FROM (10) TO (25); - diff --git a/tests/e2e/pg2pg/pkey_update/check_db_test.go b/tests/e2e/pg2pg/pkey_update/check_db_test.go deleted file mode 100644 index 7ebd60e8..00000000 --- a/tests/e2e/pg2pg/pkey_update/check_db_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package pkeyupdate - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeIncrementOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source"), pgrecipe.WithDBTables("public.__test")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestPkeyUpdate(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - replicationWorker := local.NewLocalWorker( - coordinator.NewFakeClient(), - helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType), - helpers.EmptyRegistry(), - logger.Log, - ) - replicationWorker.Start() - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - err := replicationWorker.Stop() - require.NoError(t, err) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/pkey_update/init_source/dump.sql b/tests/e2e/pg2pg/pkey_update/init_source/dump.sql deleted file mode 100644 index 1f594503..00000000 --- a/tests/e2e/pg2pg/pkey_update/init_source/dump.sql +++ /dev/null @@ -1,12 +0,0 @@ -BEGIN; -CREATE TABLE __test ( - id integer PRIMARY KEY, - value text -); -INSERT INTO __test VALUES (1, 'a'); -COMMIT; - -SELECT pg_create_logical_replication_slot('testslot', 'wal2json'); - -UPDATE __test SET id = 2 WHERE id = 1; -INSERT INTO __test VALUES (3, 'c'); diff --git a/tests/e2e/pg2pg/pkey_update/init_target/dump.sql b/tests/e2e/pg2pg/pkey_update/init_target/dump.sql deleted file mode 100644 index 02ff6b4c..00000000 --- a/tests/e2e/pg2pg/pkey_update/init_target/dump.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE __test ( - id integer PRIMARY KEY, - value text -); - -INSERT INTO __test VALUES (1, 'a'); diff --git a/tests/e2e/pg2pg/replication/check_db_test.go b/tests/e2e/pg2pg/replication/check_db_test.go deleted file mode 100644 index 503b3af2..00000000 --- a/tests/e2e/pg2pg/replication/check_db_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package replication - -import ( - "context" - "os" - "strconv" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgsink "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - ytschema "github.com/doublecloud/tross/yt/go/schema" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("")) - dstPort, _ = strconv.Atoi(os.Getenv("DB0_PG_LOCAL_PORT")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, transferID -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Verify", Verify) - t.Run("Load", Load) - }) -} - -func Existence(t *testing.T) { - _, err := pgsink.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = pgsink.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Verify(t *testing.T) { - var transfer server.Transfer - transfer.Src = &Source - transfer.Dst = &Target - transfer.Type = "INCREMENT_ONLY" - - err := tasks.VerifyDelivery(transfer, logger.Log, helpers.EmptyRegistry()) - require.NoError(t, err) - - dstStorage, err := pgsink.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - var result bool - err = dstStorage.Conn.QueryRow(context.Background(), ` - SELECT EXISTS - ( - SELECT 1 - FROM pg_tables - WHERE schemaname = 'public' - AND tablename = '_ping' - ); - `).Scan(&result) - require.NoError(t, err) - require.Equal(t, false, result) -} - -func Load(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - //----------------------------------------------------------------------------------------------------------------- - - sink, err := pgsink.NewSink(logger.Log, helpers.TransferID, Source.ToSinkParams(), helpers.EmptyRegistry()) - require.NoError(t, err) - - arrColSchema := abstract.NewTableSchema([]abstract.ColSchema{ - {ColumnName: "aid", DataType: ytschema.TypeUint8.String(), PrimaryKey: true}, - {ColumnName: "str", DataType: ytschema.TypeString.String(), PrimaryKey: true}, - {ColumnName: "id", DataType: ytschema.TypeUint8.String(), PrimaryKey: true}, - {ColumnName: "jb", DataType: ytschema.TypeAny.String(), PrimaryKey: false}, - }) - changeItemBuilder := helpers.NewChangeItemsBuilder("public", "__test", arrColSchema) - - require.NoError(t, sink.Push(changeItemBuilder.Inserts(t, []map[string]interface{}{{"aid": 11, "str": "a", "id": 11, "jb": "{}"}, {"aid": 22, "str": "b", "id": 22, "jb": `{"x": 1, "y": -2}`}, {"aid": 33, "str": "c", "id": 33}}))) - require.NoError(t, sink.Push(changeItemBuilder.Updates(t, []map[string]interface{}{{"id": 34, "jb": `{"test": "test"}`}}, []map[string]interface{}{{"aid": 33, "str": "c", "id": 33}}))) - require.NoError(t, sink.Push(changeItemBuilder.Deletes(t, []map[string]interface{}{{"aid": 22, "str": "b", "id": 22}}))) - require.NoError(t, sink.Push(changeItemBuilder.Deletes(t, []map[string]interface{}{{"aid": 33, "str": "c", "id": 34}}))) - - //----------------------------------------------------------------------------------------------------------------- - - helpers.CheckRowsCount(t, Source, "public", "__test", 14) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/replication/dump/type_check.sql b/tests/e2e/pg2pg/replication/dump/type_check.sql deleted file mode 100644 index 8c54e02f..00000000 --- a/tests/e2e/pg2pg/replication/dump/type_check.sql +++ /dev/null @@ -1,421 +0,0 @@ --- needs to be sure there is db1 -create table __test -( - id bigint not null, - aid serial, - bid bigserial, - si smallint, - ss smallserial, - - uid uuid, - - bl boolean, - - -- numeric - f float, - d double precision, - de decimal(10, 2), --- ti tinyint, --- mi mediumint, - i int, - bi bigint, - biu bigint, - b bit(8), - vb varbit(8), - - -- date time - da date, - ts timestamp, - dt timestamp, - tst timestamp with time zone, - iv interval, - tm time without time zone, --- tt time with time zone, --- y year, - - -- strings - c char, - str varchar(256), - t text, --- bb blob, - - -- binary - ba bytea, --- bin binary(10), --- vbin varbinary(100), - - -- addresses - cr cidr, - it inet, - ma macaddr, - - -- geometric types - bx box, - cl circle, - ln line, - ls lseg, - ph path, - pt point, - pg polygon, - - -- text search --- tq tsquery, --- tv tsvector, - --- tx txid_snapshot, - - -- other --- e enum ("e1", "e2"), --- se set('a', 'b', 'c'), - j json, - jb jsonb, - x xml, - arr int[], --- gi int generated always as identity, --- pl pg_lsn - primary key (aid, str, id) -- test multi pk and reverse order keys -); - -insert into __test -values (1, - 0, - 9223372036854775807, - -32768, - 1, - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - false, - 1.45e-10, - 3.14e-100, - 2.5, --- -124, -- ti --- 32765, -- mi - -8388605, - 2147483642, - 9223372036854775804, - b'10101111', - b'10101111', - '2005-03-04', - now(), - now(), - '2004-10-19 10:23:54+02', - interval '1 day 01:00:00', - '04:05:06.789', --- '04:05:06 PST', --- '04:05:06.789', --- '2099', -- year - - '1', - 'hello, friend of mine', - 'okay, now bye-bye', --- 'this it actually text but blob', -- blob - - decode('CAFEBABE', 'hex'), --- 'a\0deadbeef', -- bin --- 'cafebabe', -- vbin - - '192.168.100.128/25', - '192.168.100.128/25', - '08:00:2b:01:02:03', - box(circle '((0,0),2.0)'), - circle(box '((0,0),(1,1))'), - line(point '(-1,0)', point '(1,0)'), - lseg(box '((-1,0),(1,0))'), - path(polygon '((0,0),(1,1),(2,0))'), - point(23.4, -44.5), - polygon(box '((0,0),(1,1))'), - --- to_tsquery('cat' & 'rat'), --- to_tsvector('fat cats ate rats'), - --- txid_current_snapshot(), - --- "e1", -- e --- 'a', -- se - '{ - "yandex is the best place to work at": [ - "wish i", - "would stay", - 4.15, - { - "here after": "the " - }, - [ - "i", - [ - "n", - [ - "t", - "e r n s h i" - ], - "p" - ] - ] - ] - }', - '{ - "yandex is the best place to work at": [ - "wish i", - "would stay", - 4.15, - { - "here after": "the " - }, - [ - "i", - [ - "n", - [ - "t", - "e r n s h i" - ], - "p" - ] - ] - ] - }', - ' - bar', - '{1, 2, 3}' --- '68/1225BB70' - ) - , - (2, - 1, - 9223372036854775806, - 32767, - 32767, - 'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11', - true, - 1.34e-10, - null, - null, --- -12, -- ti --- 1123, -- mi - -1294129412, - 112412412421941041, - 129491244912401240, - b'10000001', - b'10000001', - '1999-03-04', - now(), - null, - 'Wed Dec 17 07:37:16 1997 PST', - interval '-23:00:00', - '040506', --- '2003-04-12 04:05:06 America/New_York', --- '04:05 PM', --- '1971', -- year - - '2', - 'another hello', - 'okay, another bye', --- 'another blob', -- blob - - 'well, I got stuck with time and it took a huge amount of time XD', --- 'cafebabeda', -- bin --- '\0\0\0\0\1', -- vbin - - '192.168/24', - '192.168.0.0/24', - '08-00-2b-01-02-03', - box(point '(0,0)'), - circle(point '(0,0)', 2.0), - line(point '(-2,0)', point '(2,0)'), - lseg(point '(-1,0)', point '(1,0)'), - path(polygon '((0,0),(1,0),(1,1),(0,1))'), - point(box '((-1,0),(1,0))'), - polygon(circle '((0,0),2.0)'), - --- to_tsquery(('(fat | rat) & cat'), --- to_tsvector('a:1 b:2 c:1 d:2 b:3'), - --- txid_current_snapshot(), - --- "e2", -- e --- 'b', -- se - '{ - "simpler": [ - "than", - 13e-10, - { - "it": { - "could": "be" - } - } - ] - }', - '{ - "simpler": [ - "than", - 13e-10, - { - "it": { - "could": "be" - } - } - ] - }', - ' - - - I am new - intern at TM team. - TM team is - the - best - team. - - hazzus - you - were - absolutely - right - ', - NULL --- '0/0' - ) - , - (3, - 4, - 9223372036854775805, - 13452, - -12345, - 'a0eebc999c0b4ef8bb6d6bb9bd380a11', - false, - 5.34e-10, - null, - 123, --- -122, -- ti --- -1123, -- mi - 294129412, - -784124124219410491, - 129491098649360240, - b'10000010', - b'10000010', - '1999-03-05', - null, - now(), - '12/17/1997 07:37:16.00 PST', - interval '21 days', - '04:05 PM', --- '21:32:12 PST', --- '04:05-08:00', --- '1972', -- year - - 'c', - 'another another hello', - 'okay, another another bye', --- 'another another blob but looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg', -- blob - - 'john is gonna dance jaga-jaga', --- 'caafebabee', -- bin --- '\0\0\0\0\1abcd124edb', -- vbin - - '2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128', - '2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128', - '08002b010203', - box(point '(0,0)', point '(1,1)'), - circle(polygon '((0,0),(1,1),(2,0))'), - line(point '(-3,0)', point '(3,0)'), - lseg(box '((-2,0),(2,0))'), - path(polygon '((0,0),(1,1),(2,3),(3,1),(4,0))'), - point(circle '((0,0),2.0)'), - polygon(12, circle '((0,0),2.0)'), - --- to_tsquery('fat' <-> 'rat'), --- array_to_tsvector('{fat,cat,rat}'::text[]), - --- txid_current_snapshot(), - --- "e1", -- e --- 'c', -- se - '{ - "simpler": [ - "than", - 13e-10, - { - "it": { - "could": [ - "be", - "no", - "ideas ", - " again" - ], - "sorry": null - } - } - ] - }', - '{ - "simpler": [ - "than", - 13e-10, - { - "it": { - "could": [ - "be", - "no", - "ideas ", - " again" - ], - "sorry": null - } - } - ] - }', - ' - - 1465580861.7786624 - lady - - -695149882.8150392 - voice - - throat - saw - silk - accident - -1524256040.2926793 - 1095844440 - - -2013145083.260986 - element - -1281358606.1880667 - - 2085211696 - -748870413 - 986627174 - ', - NULL --- '0/0' - ) -; - -insert into __test (str, id) -values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) -values ('nvaapsijfapfn', 201, now()), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But Ill be back, this is public promise', 422, now()), - ('Remember me, my name is hazzus', 333, now()); - -alter table __test replica identity full; - --- insert into __test (id, str, mi) values (2020, 'thanks for everything, my team', 5), --- (2019, 'and other guys I worked with', 5); diff --git a/tests/e2e/pg2pg/replication_replica_identity/check_db_test.go b/tests/e2e/pg2pg/replication_replica_identity/check_db_test.go deleted file mode 100644 index 72bf9078..00000000 --- a/tests/e2e/pg2pg/replication_replica_identity/check_db_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package test - -import ( - "os" - "strings" - "testing" - - "cuelang.org/go/pkg/time" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga -} - -func TestReplicaIdentityFullInsert(t *testing.T) { - perTransactionPush := false - testReplicationWorks(t, "testslot1", "__replica_id_full_1", perTransactionPush, untilStoragesEqual) -} - -func TestReplicaIdentityFullDelete(t *testing.T) { - perTransactionPush := false - testReplicationWorks(t, "testslot2", "__replica_id_full_2", perTransactionPush, untilStoragesEqual) -} - -func TestReplicaIdentityFullUpdate(t *testing.T) { - perTransactionPush := false - testReplicationWorks(t, "testslot3", "__replica_id_full_3", perTransactionPush, untilStoragesEqual) -} - -func TestReplicaIdentityFullInsertRetry(t *testing.T) { - perTransactionPush := false - // We use two replication slots here to emulate replication retry. - // First we replicate one insert from testslot4_1. Both source and target - // will have 4 rows after that. - testReplicationWorks(t, "testslot4_1", "__replica_id_full_4", perTransactionPush, untilStoragesEqual) - // Then we replicate the same insert with the same LSN from the other slot. - // If the table in the destination had primary key constraint, nothing - // would be replicated. But instead we will have a duplicate row in the - // destination. - testReplicationWorks(t, "testslot4_2", "__replica_id_full_4", perTransactionPush, untilDestinationRowCountEquals(5)) -} - -func TestReplicaIdentityFullInsertRetryWithPerTransactionPush(t *testing.T) { - perTransactionPush := true - // Same as above, use two slots to emulate replication retry - testReplicationWorks(t, "testslot5_1", "__replica_id_full_5", perTransactionPush, untilStoragesEqual) - // Replicate for a while and compare the storages. We wait 30 seconds to - // ensure that with perTransactionPush = true no duplicate rows are - // inserted into the destination during that time. - testReplicationWorks(t, "testslot5_2", "__replica_id_full_5", perTransactionPush, untilTimeElapsesAndStoragesEqual(30*time.Second, 4)) -} - -func TestReplicaIdentityNotFullFails(t *testing.T) { - source := *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source"), pgrecipe.WithDBTables("public.__replica_id_not_full")) - source.SlotID = "testslot6" - target := *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) - - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "PG target", Port: target.Port}, - )) - }() - - TransferType := abstract.TransferTypeIncrementOnly - helpers.InitSrcDst(helpers.TransferID, &source, &target, TransferType) - - replicationWorker := local.NewLocalWorker( - coordinator.NewFakeClient(), - helpers.MakeTransfer(helpers.TransferID, &source, &target, TransferType), - helpers.EmptyRegistry(), - logger.Log, - ) - err := replicationWorker.Run() - require.Error(t, err) - require.Contains(t, strings.ToLower(err.Error()), "no key columns found") - err = replicationWorker.Stop() - require.NoError(t, err) -} diff --git a/tests/e2e/pg2pg/replication_replica_identity/helpers.go b/tests/e2e/pg2pg/replication_replica_identity/helpers.go deleted file mode 100644 index fe5f4070..00000000 --- a/tests/e2e/pg2pg/replication_replica_identity/helpers.go +++ /dev/null @@ -1,83 +0,0 @@ -package test - -import ( - "fmt" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -type stopCondition func(t *testing.T, tableName string, src postgres.PgSource, dst postgres.PgDestination) error - -func untilStoragesEqual(t *testing.T, tableName string, src postgres.PgSource, dst postgres.PgDestination) error { - params := helpers.NewCompareStorageParams().WithTableFilter(makeTableFilter(tableName)) - return helpers.WaitStoragesSynced(t, src, dst, 15, params) -} - -func untilDestinationRowCountEquals(rowCount uint64) stopCondition { - return func(t *testing.T, tableName string, src postgres.PgSource, dst postgres.PgDestination) error { - return helpers.WaitDestinationEqualRowsCount("public", tableName, helpers.GetSampleableStorageByModel(t, dst), time.Minute, rowCount) - } -} - -func untilTimeElapsesAndStoragesEqual(delay time.Duration, expectedDstRowCount uint64) stopCondition { - return func(t *testing.T, tableName string, src postgres.PgSource, dst postgres.PgDestination) error { - time.Sleep(delay) - params := helpers.NewCompareStorageParams().WithTableFilter(makeTableFilter(tableName)) - if err := helpers.WaitStoragesSynced(t, src, dst, 15, params); err != nil { - return err - } - return helpers.WaitDestinationEqualRowsCount("public", tableName, helpers.GetSampleableStorageByModel(t, dst), 5*time.Second, expectedDstRowCount) - } -} - -func testReplicationWorks(t *testing.T, slotID, tableName string, perTransactionPush bool, waitStopCondition stopCondition) { - source := *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source"), pgrecipe.WithDBTables(fmt.Sprintf("public.%s", tableName))) - source.SlotID = slotID - target := *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) - target.PerTransactionPush = perTransactionPush - - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: source.Port}, - helpers.LabeledPort{Label: "PG target", Port: target.Port}, - )) - }() - - TransferType := abstract.TransferTypeIncrementOnly - helpers.InitSrcDst(helpers.TransferID, &source, &target, TransferType) - - replicationWorker := local.NewLocalWorker( - coordinator.NewFakeClient(), - helpers.MakeTransfer(helpers.TransferID, &source, &target, TransferType), - helpers.EmptyRegistry(), - logger.Log, - ) - replicationWorker.Start() - - require.NoError(t, waitStopCondition(t, tableName, source, target)) - - err := replicationWorker.Stop() - require.NoError(t, err) -} - -func makeTableFilter(tableName string) func(tables abstract.TableMap) []abstract.TableDescription { - return func(tables abstract.TableMap) []abstract.TableDescription { - var filteredTables []abstract.TableDescription - for _, table := range helpers.FilterTechnicalTables(tables) { - if table.Name != tableName { - continue - } - filteredTables = append(filteredTables, table) - } - return filteredTables - } -} diff --git a/tests/e2e/pg2pg/replication_replica_identity/init_source/dump.sql b/tests/e2e/pg2pg/replication_replica_identity/init_source/dump.sql deleted file mode 100644 index 4fa93bdc..00000000 --- a/tests/e2e/pg2pg/replication_replica_identity/init_source/dump.sql +++ /dev/null @@ -1,63 +0,0 @@ -BEGIN; - -CREATE TABLE public.__replica_id_full_1(i INT, t TEXT); -ALTER TABLE public.__replica_id_full_1 REPLICA IDENTITY FULL; - -CREATE TABLE public.__replica_id_full_2(i INT, t TEXT); -ALTER TABLE public.__replica_id_full_2 REPLICA IDENTITY FULL; -INSERT INTO public.__replica_id_full_2 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); - -CREATE TABLE public.__replica_id_full_3(i INT, t TEXT); -ALTER TABLE public.__replica_id_full_3 REPLICA IDENTITY FULL; -INSERT INTO public.__replica_id_full_3 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); - -CREATE TABLE public.__replica_id_full_4(i INT, t TEXT); -ALTER TABLE public.__replica_id_full_4 REPLICA IDENTITY FULL; -INSERT INTO public.__replica_id_full_4 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); - -CREATE TABLE public.__replica_id_full_5(i INT, t TEXT); -ALTER TABLE public.__replica_id_full_5 REPLICA IDENTITY FULL; -INSERT INTO public.__replica_id_full_5 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); - -CREATE TABLE public.__replica_id_not_full(i INT, t TEXT); - -COMMIT; - -BEGIN; -SELECT pg_create_logical_replication_slot('testslot1', 'wal2json'); -SELECT pg_create_logical_replication_slot('testslot2', 'wal2json'); -SELECT pg_create_logical_replication_slot('testslot3', 'wal2json'); - -SELECT pg_create_logical_replication_slot('testslot4_1', 'wal2json'); -SELECT pg_create_logical_replication_slot('testslot4_2', 'wal2json'); - -SELECT pg_create_logical_replication_slot('testslot5_1', 'wal2json'); -SELECT pg_create_logical_replication_slot('testslot5_2', 'wal2json'); - -SELECT pg_create_logical_replication_slot('testslot6', 'wal2json'); -COMMIT; - -BEGIN; -INSERT INTO public.__replica_id_full_1 VALUES (1, '111'), (2, '222'); -COMMIT; - -BEGIN; -DELETE FROM public.__replica_id_full_2 where i = 1; -COMMIT; - -BEGIN; -UPDATE public.__replica_id_full_3 SET t = '11' where i = 1; -UPDATE public.__replica_id_full_3 SET t = '22', i = 22 where i = 2; -COMMIT; - -BEGIN; -INSERT INTO public.__replica_id_not_full VALUES (3, '333'), (4, '444'); -COMMIT; - -BEGIN; -INSERT INTO public.__replica_id_full_4 VALUES (4, '4'); -COMMIT; - -BEGIN; -INSERT INTO public.__replica_id_full_5 VALUES (4, '4'); -COMMIT; diff --git a/tests/e2e/pg2pg/replication_replica_identity/init_target/dump.sql b/tests/e2e/pg2pg/replication_replica_identity/init_target/dump.sql deleted file mode 100644 index 3361454f..00000000 --- a/tests/e2e/pg2pg/replication_replica_identity/init_target/dump.sql +++ /dev/null @@ -1,20 +0,0 @@ -CREATE TABLE public.__replica_id_full_1(i INT, t TEXT); -CREATE TABLE public.__replica_id_full_2(i INT, t TEXT); -CREATE TABLE public.__replica_id_full_3(i INT, t TEXT); -CREATE TABLE public.__replica_id_full_4(i INT, t TEXT); -CREATE TABLE public.__replica_id_full_5(i INT, t TEXT); - -INSERT INTO public.__replica_id_full_2 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); -INSERT INTO public.__replica_id_full_3 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); -INSERT INTO public.__replica_id_full_4 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); -INSERT INTO public.__replica_id_full_5 (i, t) VALUES (1, '1'), (2, '2'), (3, '3'); - --- Set full replica identity, otherwise checksum will return error on schema comparison --- i.e. primary keys on source and target do not match -ALTER TABLE public.__replica_id_full_1 REPLICA IDENTITY FULL; -ALTER TABLE public.__replica_id_full_2 REPLICA IDENTITY FULL; -ALTER TABLE public.__replica_id_full_3 REPLICA IDENTITY FULL; -ALTER TABLE public.__replica_id_full_4 REPLICA IDENTITY FULL; -ALTER TABLE public.__replica_id_full_5 REPLICA IDENTITY FULL; - -CREATE TABLE public.__replica_id_not_full(i INT, t TEXT); diff --git a/tests/e2e/pg2pg/replication_special_values/check_db_test.go b/tests/e2e/pg2pg/replication_special_values/check_db_test.go deleted file mode 100644 index 10abf729..00000000 --- a/tests/e2e/pg2pg/replication_special_values/check_db_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package replicationview - -import ( - "context" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func TestReplicationNullInJSON(t *testing.T) { - Source := pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target := pgrecipe.RecipeTarget() - transferType := abstract.TransferTypeSnapshotAndIncrement - - helpers.InitSrcDst(helpers.TransferID, Source, Target, transferType) - - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, transferType) - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - srcConn, err := postgres.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - - _, err = srcConn.Exec(context.Background(), `INSERT INTO rsv_null_in_json(i, j, jb) VALUES (101, 'null', 'null'), (102, '"null"', '"null"')`) - require.NoError(t, err) - - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "rsv_null_in_json", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/replication_special_values/init_source/dump.sql b/tests/e2e/pg2pg/replication_special_values/init_source/dump.sql deleted file mode 100644 index 99197702..00000000 --- a/tests/e2e/pg2pg/replication_special_values/init_source/dump.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE rsv_null_in_json( - i INT PRIMARY KEY, - j JSON NOT NULL, - jb jsonb NOT NULL -); - -INSERT INTO rsv_null_in_json(i, j, jb) VALUES -(1, 'null', 'null'), -(2, '"null"', '"null"'); diff --git a/tests/e2e/pg2pg/replication_toast/check_db_test.go b/tests/e2e/pg2pg/replication_toast/check_db_test.go deleted file mode 100644 index 85b9dce2..00000000 --- a/tests/e2e/pg2pg/replication_toast/check_db_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package toast - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/jackc/pgx/v4" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) - ErrRetry = xerrors.NewSentinel("Retry") -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeIncrementOnly) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func largeString(n int, s string) string { - var result string - for i := 0; i < n; i++ { - result += s - } - return result -} - -func makeTestFunction(usePolling bool) func(t *testing.T) { - var schema, slotID string - if usePolling { - schema = "s1" - slotID = "slot1" - } else { - schema = "s2" - slotID = "slot2" - } - - return func(t *testing.T) { - sourceCopy := Source - sourceCopy.UsePolling = usePolling - sourceCopy.SlotID = slotID - sourceCopy.KeeperSchema = schema - sourceCopy.DBTables = []string{fmt.Sprintf("%s.__test", schema)} - transfer := server.Transfer{ - ID: "test_id", - Src: &sourceCopy, - Dst: &Target, - } - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - dstConn, err := pgcommon.MakeConnPoolFromDst(&Target, logger.Log) - require.NoError(t, err) - defer dstConn.Close() - - defer func() { - r, err := srcConn.Exec(context.Background(), fmt.Sprintf(`DELETE FROM %s.__test`, schema)) - require.NoError(t, err) - require.EqualValues(t, 2, r.RowsAffected()) - r, err = dstConn.Exec(context.Background(), fmt.Sprintf(`DELETE FROM %s.__test`, schema)) - require.NoError(t, err) - require.EqualValues(t, 2, r.RowsAffected()) - }() - - worker := local.NewLocalWorker(coordinator.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - worker.Start() - defer worker.Stop() //nolint - - // 1. Insert two rows, a small and a big one - _, err = srcConn.Exec(context.Background(), fmt.Sprintf(`INSERT INTO %s.__test VALUES (1, 10, 'a')`, schema)) - require.NoError(t, err) - _, err = srcConn.Exec(context.Background(), fmt.Sprintf(`INSERT INTO %s.__test VALUES (2, 20, $1)`, schema), largeString(16384, "a")) - require.NoError(t, err) - - var small int - var large string - err = backoff.Retry(func() error { - err := dstConn.QueryRow(context.Background(), fmt.Sprintf(`SELECT small, large FROM %s.__test WHERE id = 1`, schema)).Scan(&small, &large) - if err != nil { - if !xerrors.Is(err, pgx.ErrNoRows) { - return backoff.Permanent(err) - } - logger.Log.Warnf("select err: %v", err) - } - return err - }, backoff.NewConstantBackOff(time.Second)) - require.NoError(t, err) - require.Equal(t, 10, small) - require.Equal(t, "a", large) - - err = backoff.Retry(func() error { - err = dstConn.QueryRow(context.Background(), fmt.Sprintf(`SELECT small, large FROM %s.__test WHERE id = 2`, schema)).Scan(&small, &large) - if err != nil { - if !xerrors.Is(err, pgx.ErrNoRows) { - return backoff.Permanent(err) - } - logger.Log.Warnf("select err: %v", err) - } - return err - }, backoff.NewConstantBackOff(time.Second)) - require.NoError(t, err) - require.Equal(t, 20, small) - require.Equal(t, largeString(16384, "a"), large) - - // 2. Modify both rows - r, err := srcConn.Exec(context.Background(), fmt.Sprintf(`UPDATE %s.__test SET small = 30`, schema)) - require.NoError(t, err) - require.EqualValues(t, 2, r.RowsAffected()) - r, err = srcConn.Exec(context.Background(), fmt.Sprintf(`UPDATE %s.__test SET large = 'b' WHERE id = 1`, schema)) - require.NoError(t, err) - require.EqualValues(t, 1, r.RowsAffected()) - - err = backoff.Retry(func() error { - err = dstConn.QueryRow(context.Background(), fmt.Sprintf(`SELECT small, large FROM %s.__test WHERE id = 1`, schema)).Scan(&small, &large) - require.NoError(t, err) - if small != 30 { - logger.Log.Warnf(`Unexpected "small" value: %d`, small) - return ErrRetry - } - if large != "b" { - logger.Log.Warnf(`Unexpected "large" value: %s`, large) - return ErrRetry - } - - err = dstConn.QueryRow(context.Background(), fmt.Sprintf(`SELECT small, large FROM %s.__test WHERE id = 2`, schema)).Scan(&small, &large) - require.NoError(t, err) - if small != 30 { - logger.Log.Warnf(`Unexpected "small" value: %d`, small) - return ErrRetry - } - if large != largeString(16384, "a") { - logger.Log.Warnf(`Unexpected "large" value: %s`, large) - return ErrRetry - } - return nil - }, backoff.NewConstantBackOff(time.Second)) - } -} - -func TestToast(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("TestToast/UsePollingFalse", makeTestFunction(false)) - t.Run("TestToast/UsePollingTrue", makeTestFunction(true)) -} diff --git a/tests/e2e/pg2pg/replication_toast/init_source/dump.sql b/tests/e2e/pg2pg/replication_toast/init_source/dump.sql deleted file mode 100644 index 76716660..00000000 --- a/tests/e2e/pg2pg/replication_toast/init_source/dump.sql +++ /dev/null @@ -1,22 +0,0 @@ -BEGIN; -CREATE SCHEMA s1; -CREATE TABLE s1.__test ( - id integer PRIMARY KEY, - small integer, - large text -); -ALTER TABLE s1.__test ALTER COLUMN large SET STORAGE EXTERNAL; -COMMIT; - -BEGIN; -CREATE SCHEMA s2; -CREATE TABLE s2.__test ( - id integer PRIMARY KEY, - small integer, - large text -); -ALTER TABLE s2.__test ALTER COLUMN large SET STORAGE EXTERNAL; -COMMIT; - -SELECT pg_create_logical_replication_slot('slot1', 'wal2json'); -SELECT pg_create_logical_replication_slot('slot2', 'wal2json'); diff --git a/tests/e2e/pg2pg/replication_toast/init_target/dump.sql b/tests/e2e/pg2pg/replication_toast/init_target/dump.sql deleted file mode 100644 index 16456c90..00000000 --- a/tests/e2e/pg2pg/replication_toast/init_target/dump.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE SCHEMA s1; -CREATE TABLE s1.__test ( - id integer primary key, - small integer, - large text -); - -CREATE SCHEMA s2; -CREATE TABLE s2.__test ( - id integer primary key, - small integer, - large text -); diff --git a/tests/e2e/pg2pg/replication_view/check_db_test.go b/tests/e2e/pg2pg/replication_view/check_db_test.go deleted file mode 100644 index bfeae396..00000000 --- a/tests/e2e/pg2pg/replication_view/check_db_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package replicationview - -import ( - "os" - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga -} - -func TestViewReplication(t *testing.T) { - Source := *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target := *pgrecipe.RecipeTarget(pgrecipe.WithInitDir("init_target")) - Target.Cleanup = server.Truncate - - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeIncrementOnly) - worker := local.NewLocalWorker(coordinator.NewFakeClient(), helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeIncrementOnly), helpers.EmptyRegistry(), logger.Log) - worker.Start() - - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "tv_table", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 20*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "odd_channels", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 20*time.Second)) - - require.NoError(t, worker.Stop()) -} diff --git a/tests/e2e/pg2pg/replication_view/init_source/dump.sql b/tests/e2e/pg2pg/replication_view/init_source/dump.sql deleted file mode 100644 index af0a882f..00000000 --- a/tests/e2e/pg2pg/replication_view/init_source/dump.sql +++ /dev/null @@ -1,19 +0,0 @@ -BEGIN; -SELECT pg_create_logical_replication_slot('testslot', 'wal2json'); -COMMIT; - -BEGIN; -CREATE TABLE tv_table(i INT PRIMARY KEY, cname TEXT); -COMMIT; - -BEGIN; -CREATE VIEW odd_channels AS SELECT i, cname FROM tv_table WHERE i > 2; -COMMIT; - -BEGIN; -INSERT INTO tv_table(i, cname) VALUES (1, 'ZDF'); -INSERT INTO tv_table(i, cname) VALUES (2, 'Das Erste'); -INSERT INTO tv_table(i, cname) VALUES (3, 'RTL'); -INSERT INTO tv_table(i, cname) VALUES (4, 'SAT.1'); -INSERT INTO tv_table(i, cname) VALUES (5, 'VOX'); -COMMIT; diff --git a/tests/e2e/pg2pg/replication_view/init_target/dump.sql b/tests/e2e/pg2pg/replication_view/init_target/dump.sql deleted file mode 100644 index 11b46388..00000000 --- a/tests/e2e/pg2pg/replication_view/init_target/dump.sql +++ /dev/null @@ -1,7 +0,0 @@ -BEGIN; -CREATE TABLE tv_table(i INT, cname TEXT); -COMMIT; - -BEGIN; -CREATE VIEW odd_channels AS SELECT i, cname FROM tv_table WHERE i > 2; -COMMIT; diff --git a/tests/e2e/pg2pg/snapshot/check_db_test.go b/tests/e2e/pg2pg/snapshot/check_db_test.go deleted file mode 100644 index 71815abb..00000000 --- a/tests/e2e/pg2pg/snapshot/check_db_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Verify", Verify) - t.Run("Snapshot", Snapshot) - }) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Verify(t *testing.T) { - var transfer server.Transfer - transfer.Src = &Source - transfer.Dst = &Target - transfer.Type = "SNAPSHOT_ONLY" - - err := tasks.VerifyDelivery(transfer, logger.Log, helpers.EmptyRegistry()) - require.NoError(t, err) - - dstStorage, err := postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - var result bool - err = dstStorage.Conn.QueryRow(context.Background(), ` - SELECT EXISTS - ( - SELECT 1 - FROM pg_tables - WHERE schemaname = 'public' - AND tablename = '_ping' - ); - `).Scan(&result) - require.NoError(t, err) - require.Equal(t, false, result) -} - -func Snapshot(t *testing.T) { - Source.PreSteps.Constraint = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - _ = helpers.Activate(t, transfer) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/snapshot/dump/type_check.sql b/tests/e2e/pg2pg/snapshot/dump/type_check.sql deleted file mode 100644 index e054b4a0..00000000 --- a/tests/e2e/pg2pg/snapshot/dump/type_check.sql +++ /dev/null @@ -1,172 +0,0 @@ --- needs to be sure there is db1 -create table __test ( - id bigint not null, - aid serial, - - -- numeric - f float, - d double precision, - de decimal(10,2), --- ti tinyint, --- mi mediumint, - i int, - bi bigint, - biu bigint, - b bit(8), - - -- date time - da date, - ts timestamp, - dt timestamp, --- tm time, --- y year, - - -- strings - c char, - str varchar(256), - t text, --- bb blob, - - -- binary --- bin binary(10), --- vbin varbinary(100), - - -- other - arr int[], - gi int generated always as identity, --- e enum ("e1", "e2"), --- se set('a', 'b', 'c'), --- j json, - primary key (aid, str, id) -- test multi pk and reverse order keys -); - -insert into __test values ( - 1, - 0, - 1.45e-10, - 3.14e-100, - 2.5, --- -124, -- ti --- 32765, -- mi - -8388605, - 2147483642, - 9223372036854775804, - - b'10101111', - - '2005-03-04', - now(), - now(), --- now(), --- '2099', -- year - - '1', - 'hello, friend of mine', - 'okay, now bye-bye', - '{1, 2, 3}' --- 'this it actually text but blob', -- blob --- 'a\0deadbeef', -- bin --- 'cafebabe', -- vbin --- "e1", -- e --- 'a', -- se --- '{"yandex is the best place to work at": ["wish i", "would stay", 4.15, {"here after":"the "}, ["i", ["n", ["t", "e r n s h i"], "p"]]]}' -) -, -( - 2, - 1, - 1.34e-10, - null, - null, --- -12, -- ti --- 1123, -- mi - -1294129412, - 112412412421941041, - 129491244912401240, - - b'10000001', - - '1999-03-04', - now(), - null, --- now(), --- '1971', -- year - - '2', - 'another hello', - 'okay, another bye', - NULL --- 'another blob', -- blob --- 'cafebabeda', -- bin --- '\0\0\0\0\1', -- vbin --- "e2", -- e --- 'b', -- se --- '{"simpler": ["than", 13e-10, {"it": {"could": "be"}}]}' -) -, -( - 3, - 4, - 5.34e-10, - null, - 123, --- -122, -- ti --- -1123, -- mi - 294129412, - -784124124219410491, - 129491098649360240, - - b'10000010', - - '1999-03-05', - null, - now(), --- now(), --- '1972', -- year - - 'c', - 'another another hello', - 'okay, another another bye', - NULL --- 'another another blob but looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg' --- 'nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn' --- 'ggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg', -- blob --- 'caafebabee', -- bin --- '\0\0\0\0\1abcd124edb', -- vbin --- "e1", -- e --- 'c', -- se --- '{"simpler": ["than", 13e-10, {"it": {"could": ["be", "no", "ideas ", " again"], "sorry": null}}]}' -) -; - -insert into __test (str, id) values ('hello', 0), - ('aaa', 214), - ('vvvv', 124124), - ('agpnaogapoajfqt-oqoo ginsdvnaojfspbnoaj apngpowo qeonwpbwpen', 1234), - ('aagiangsfnaofasoasvboas', 12345); - -insert into __test (str, id, da) values ('nvaapsijfapfn', 201, now()), - ('Day the creator of this code was born', 202, '1999-09-16'), - ('Coronavirus made me leave', 322, '2020-06-03'), - ('But Ill be back, this is public promise', 422, now()), - ('Remember me, my name is hazzus', 333, now()); - - - --- insert into __test (id, str, mi) values (2020, 'thanks for everything, my team', 5), --- (2019, 'and other guys I worked with', 5); --- TM-1238 -create schema test_schema; -create table test_schema.test_table ( - id int primary key, - body text -); -insert into test_schema.test_table (id, body) values (1, 'test value 1'), (2, 'test value 2'), (3, 'test value 3'); diff --git a/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go b/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go deleted file mode 100644 index 80e9d245..00000000 --- a/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithDBTables("public.t2")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - w := helpers.Activate(t, transfer) - w.Close(t) - - dstStorage, err := postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - exists, err := CheckTableExistence(context.Background(), dstStorage.Conn, "public", "t2") - require.NoError(t, err) - require.True(t, exists) - - exists, err = CheckTableExistence(context.Background(), dstStorage.Conn, "mysch", "t") - require.NoError(t, err) - require.False(t, exists) -} - -// CheckTableExistence is a helper function for PostgreSQL to check existence of the given table -func CheckTableExistence(ctx context.Context, conn *pgxpool.Pool, tableSchema string, tableName string) (bool, error) { - var result bool - err := conn.QueryRow(context.Background(), ` - SELECT EXISTS - ( - SELECT FROM information_schema.tables - WHERE table_schema = $1 AND table_name = $2 - ); - `, tableSchema, tableName).Scan(&result) - if err != nil { - return false, xerrors.Errorf("check-table-existence query failed: %w", err) - } - return result, nil -} diff --git a/tests/e2e/pg2pg/snapshot_missing_public/dump/dump.sql b/tests/e2e/pg2pg/snapshot_missing_public/dump/dump.sql deleted file mode 100644 index b8ab455a..00000000 --- a/tests/e2e/pg2pg/snapshot_missing_public/dump/dump.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE SCHEMA mysch; - -ALTER DATABASE postgres SET search_path = 'mysch'; - -ALTER ROLE postgres SET search_path = 'mysch'; - -CREATE TABLE mysch.t(i INT PRIMARY KEY, t TEXT); -INSERT INTO mysch.t(i, t) VALUES (1, 'a'), (2, 'b'); - -CREATE TABLE public.t2(i INT PRIMARY KEY, f REAL); -INSERT INTO public.t2(i, f) VALUES (1, 1.0), (2, 4.0); diff --git a/tests/e2e/pg2pg/table_capital_letter/check_db_test.go b/tests/e2e/pg2pg/table_capital_letter/check_db_test.go deleted file mode 100644 index b33a7304..00000000 --- a/tests/e2e/pg2pg/table_capital_letter/check_db_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package snapshot - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - cpclient "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotOnly - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithDBTables("public.FooContents")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestSnapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - err := tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - //------------------------------------------------------------------------------ - // check case when tableName starts with capital letter - helpers.CheckRowsCount(t, Target, "public", "FooContents", 1) -} diff --git a/tests/e2e/pg2pg/table_capital_letter/dump/type_check.sql b/tests/e2e/pg2pg/table_capital_letter/dump/type_check.sql deleted file mode 100644 index a4183eac..00000000 --- a/tests/e2e/pg2pg/table_capital_letter/dump/type_check.sql +++ /dev/null @@ -1,5 +0,0 @@ -create table "FooContents" ( - id int primary key, - body text -); -insert into "FooContents" (id, body) values (1, 'test value 1'); diff --git a/tests/e2e/pg2pg/time_with_fallback/check_db_test.go b/tests/e2e/pg2pg/time_with_fallback/check_db_test.go deleted file mode 100644 index 10781675..00000000 --- a/tests/e2e/pg2pg/time_with_fallback/check_db_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package timewithfallback - -import ( - "os" - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - Target.Cleanup = server.DisabledCleanup - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func loadSnapshot(t *testing.T) { - Source.PreSteps.Constraint = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - _ = helpers.Activate(t, transfer) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -// This test is kind of tricky -// -// We haven't options to turn-off CopyUpload behaviour, but we need to test behaviour on homo-inserts (who runs after COPY insert failed) -// -// So, this test initializes 'dst' table by the same table_schema, that in the 'src'. -// And except this, initialization put in 'dst' one row (which is the same as one in 'src'). -// This leads to next behaviour: when COPY upload starts, COPY failed bcs of rows collision, and fallback into inserts - which successfully finished bcs of my fix. -// -// If run this test on trunk (before my fix) - it's failed. - -func TestUserTypes(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - loadSnapshot(t) -} diff --git a/tests/e2e/pg2pg/time_with_fallback/init_source/init.sql b/tests/e2e/pg2pg/time_with_fallback/init_source/init.sql deleted file mode 100644 index b38fdec5..00000000 --- a/tests/e2e/pg2pg/time_with_fallback/init_source/init.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE times(i INT PRIMARY KEY, t TIME); - -INSERT INTO times(i, t) VALUES -(1, '04:05:06'), -(2, '04:05:06'); diff --git a/tests/e2e/pg2pg/time_with_fallback/init_target/init.sql b/tests/e2e/pg2pg/time_with_fallback/init_target/init.sql deleted file mode 100644 index 625f65d4..00000000 --- a/tests/e2e/pg2pg/time_with_fallback/init_target/init.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE times(i INT PRIMARY KEY, t TIME); - -INSERT INTO times(i, t) VALUES -(1, '04:05:06'); diff --git a/tests/e2e/pg2pg/tx_boundaries/check_db_test.go b/tests/e2e/pg2pg/tx_boundaries/check_db_test.go deleted file mode 100644 index 9c2a6650..00000000 --- a/tests/e2e/pg2pg/tx_boundaries/check_db_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package replication - -import ( - "context" - "os" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - "github.com/doublecloud/tross/pkg/config" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/worker/tasks" - "github.com/doublecloud/tross/tests/helpers" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/stretchr/testify/require" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable - dpconfig.Cloud = &dpconfig.CloudConfig{ - CloudAPIEndpoint: "gw.db.yandex-team.ru:443", - CloudCreds: &config.PassportOauthToken{Token: config.Secret(os.Getenv("DBAAS_TOKEN"))}, - } -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - Target.PerTransactionPush = true - t.Run("Main group", func(t *testing.T) { - t.Run("Existence", Existence) - t.Run("Snapshot", Snapshot) - t.Run("Replication", Load) - }) -} - -func Existence(t *testing.T) { - _, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = postgres.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - require.NoError(t, tasks.ActivateDelivery(context.Background(), nil, coordinator.NewFakeClient(), *transfer, helpers.EmptyRegistry())) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Load(t *testing.T) { - Target.CopyUpload = false - Target.PerTransactionPush = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - Source.BatchSize = 10 * 1024 // to speedup repl - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - st, err := postgres.NewStorage(Source.ToStorageParams(nil)) - require.NoError(t, err) - _, err = st.Conn.Exec(context.Background(), "delete from __test where id > 10") - require.NoError(t, err) - defer st.Close() - - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 180*time.Second)) - - //----------------------------------------------------------------------------------------------------------------- - - addData(t, st.Conn) - - helpers.CheckRowsCount(t, Source, "public", "trash", 1) - require.NoError(t, helpers.WaitEqualRowsCount(t, "public", "__test", helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 180*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func addData(t *testing.T, conn *pgxpool.Pool) { - _, err := conn.Exec(context.Background(), "INSERT INTO trash (title) VALUES ('xyz');") - require.NoError(t, err) - _, err = conn.Exec(context.Background(), "INSERT INTO __test (id, title) VALUES (11, 'abc'), (12, 'def');") - require.NoError(t, err) -} diff --git a/tests/e2e/pg2pg/tx_boundaries/dump/type_check.sql b/tests/e2e/pg2pg/tx_boundaries/dump/type_check.sql deleted file mode 100644 index 209a1c63..00000000 --- a/tests/e2e/pg2pg/tx_boundaries/dump/type_check.sql +++ /dev/null @@ -1,5 +0,0 @@ -create table trash (trash_id serial primary key, title text); - -create table __test (id serial primary key, title text); - -insert into __test select s, md5(random()::text) from generate_Series(1, 50000) as s; diff --git a/tests/e2e/pg2pg/unusual_dates/check_db_test.go b/tests/e2e/pg2pg/unusual_dates/check_db_test.go deleted file mode 100644 index 853b064a..00000000 --- a/tests/e2e/pg2pg/unusual_dates/check_db_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package usertypes - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump")) - Target = pgrecipe.RecipeTarget() -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - t.Run("UnusualDates", func(t *testing.T) { - t.Run("Snapshot", Snapshot) - t.Run("Replication", Replication) - }) -} - -func Snapshot(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotOnly) - - _ = helpers.Activate(t, transfer) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func Replication(t *testing.T) { - transfer := helpers.MakeTransfer(helpers.TransferID, Source, Target, abstract.TransferTypeSnapshotAndIncrement) - - srcConn, err := pgcommon.MakeConnPoolFromSrc(Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - worker := helpers.Activate(t, transfer) - defer worker.Close(t) - - _, err = srcConn.Exec(context.Background(), - `insert into testtable values (11, '2000-10-19 10:23:54.123', '2000-10-19 10:23:54.123+02', '2000-10-19')`) - require.NoError(t, err) - - // BC dates will be supported in https://st.yandex-team.ru/TM-5127 - // _, err = srcConn.Exec(context.Background(), - // `insert into testtable values (12, '2000-10-19 10:23:54.123 BC', '2000-10-19 10:23:54.123+02 BC', '2000-10-19 BC')`) - // require.NoError(t, err) - - _, err = srcConn.Exec(context.Background(), - `insert into testtable values (13, '40000-10-19 10:23:54.123456', '40000-10-19 10:23:54.123456+02', '40000-10-19')`) - require.NoError(t, err) - - // _, err = srcConn.Exec(context.Background(), - // `insert into testtable values (14, '4000-10-19 10:23:54.123456 BC', '4000-10-19 10:23:54.123456+02 BC', '4000-10-19 BC')`) - // require.NoError(t, err) - - require.NoError(t, helpers.WaitStoragesSynced(t, Source, Target, 50, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/pg2pg/unusual_dates/dump/dump.sql b/tests/e2e/pg2pg/unusual_dates/dump/dump.sql deleted file mode 100644 index 70f5c4a7..00000000 --- a/tests/e2e/pg2pg/unusual_dates/dump/dump.sql +++ /dev/null @@ -1,18 +0,0 @@ -BEGIN; -create table testtable ( - id integer primary key, - val1 timestamp (6) without time zone, - val2 timestamp (6) with time zone, - val3 date -); -insert into testtable values (1, '2000-10-19 10:23:54.123', '2000-10-19 10:23:54.123+02', '2000-10-19'); --- insert into testtable values (2, '2000-10-19 10:23:54.123 BC', '2000-10-19 10:23:54.123+02 BC', '2000-10-19 BC'); -insert into testtable values (3, '40000-10-19 10:23:54.123456', '40000-10-19 10:23:54.123456+02', '40000-10-19'); --- insert into testtable values (4, '4000-10-19 10:23:54.123456 BC', '4000-10-19 10:23:54.123456+02 BC', '4000-10-19 BC'); -COMMIT; - --- BC dates will be supported in https://st.yandex-team.ru/TM-5127 - -BEGIN; -SELECT pg_create_logical_replication_slot('testslot', 'wal2json'); -COMMIT; diff --git a/tests/e2e/pg2pg/user_types/check_db_test.go b/tests/e2e/pg2pg/user_types/check_db_test.go deleted file mode 100644 index 6cd66c95..00000000 --- a/tests/e2e/pg2pg/user_types/check_db_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package usertypes - -import ( - "context" - "os" - "testing" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - pgcommon "github.com/doublecloud/tross/pkg/providers/postgres" - "github.com/doublecloud/tross/pkg/providers/postgres/pgrecipe" - "github.com/doublecloud/tross/pkg/runtime/local" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -var ( - Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("init_source")) - Target = *pgrecipe.RecipeTarget() - ErrRetry = xerrors.NewSentinel("Retry") -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotAndIncrement) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable -} - -func loadSnapshot(t *testing.T) { - Source.PreSteps.Constraint = true - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, abstract.TransferTypeSnapshotOnly) - - _ = helpers.Activate(t, transfer) - - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} - -func checkReplicationWorks(t *testing.T) { - transfer := server.Transfer{ - ID: "test_id", - Src: &Source, - Dst: &Target, - Type: abstract.TransferTypeSnapshotAndIncrement, - } - - srcConn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) - require.NoError(t, err) - defer srcConn.Close() - - worker := local.NewLocalWorker(coordinator.NewFakeClient(), &transfer, helpers.EmptyRegistry(), logger.Log) - worker.Start() - defer worker.Stop() - - _, err = srcConn.Exec(context.Background(), `INSERT INTO testtable VALUES (2, 'choovuck', 'zhepa', 'EinScheissdreckWerdeIchTun', (2, '456')::udt, ARRAY [(3, 'foo1')::udt, (4, 'bar1')::udt])`) - require.NoError(t, err) - require.NoError(t, helpers.WaitStoragesSynced(t, Source, Target, 50, helpers.NewCompareStorageParams())) - - tag, err := srcConn.Exec(context.Background(), `UPDATE testtable SET fancy = 'zhopa', deuch = 'DuGehstMirAufDieEier', udt = (3, '789')::udt, udt_arr = ARRAY [(5, 'foo2')::udt, (6, 'bar2')::udt] where id = 2`) - require.NoError(t, err) - require.EqualValues(t, tag.RowsAffected(), 1) - require.NoError(t, helpers.WaitStoragesSynced(t, Source, Target, 50, helpers.NewCompareStorageParams())) -} - -func TestUserTypes(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "PG source", Port: Source.Port}, - helpers.LabeledPort{Label: "PG target", Port: Target.Port}, - )) - }() - - loadSnapshot(t) - // loadSnapshot always assigns true to CopyUpload flag which is used by sinker. - // In order for replication to work we must set CopyUpload value back to false. - Target.CopyUpload = false - checkReplicationWorks(t) -} diff --git a/tests/e2e/pg2pg/user_types/init_source/init.sql b/tests/e2e/pg2pg/user_types/init_source/init.sql deleted file mode 100644 index d820335c..00000000 --- a/tests/e2e/pg2pg/user_types/init_source/init.sql +++ /dev/null @@ -1,34 +0,0 @@ -BEGIN; -create schema "woshiPushiMushi"; -create type "woshiPushiMushi"."Wut" as enum ('DuGehstMirAufDieEier', 'EinScheissdreckWerdeIchTun'); -create type "fancyCamelCaseType" as enum ('zhopa', 'zhepa'); - -CREATE TYPE udt AS -( - int_field int, - text_field text -); - -CREATE TYPE with_nested_udt_array AS -( - int_field int, - array_field udt array -); - -create table testtable ( - id integer primary key, - charvar character varying(256), - fancy "fancyCamelCaseType", - deuch "woshiPushiMushi"."Wut", - udt udt, - udt_arr udt array, - nested_udt_arr with_nested_udt_array -); - -INSERT INTO testtable (id, charvar, fancy, deuch, udt, udt_arr, nested_udt_arr) -VALUES (1, 'chuvak', 'zhopa', 'DuGehstMirAufDieEier', (1, '123')::udt, ARRAY [(1, 'foo')::udt, (2, 'bar')::udt], (1, ARRAY[(2, 'sometext')::udt])); -COMMIT; - -BEGIN; -SELECT pg_create_logical_replication_slot('testslot', 'wal2json'); -COMMIT; diff --git a/tests/e2e/sample2ch/replication/check_db_test.go b/tests/e2e/sample2ch/replication/check_db_test.go deleted file mode 100644 index b5e65548..00000000 --- a/tests/e2e/sample2ch/replication/check_db_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package replication - -import ( - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - server "github.com/doublecloud/tross/pkg/abstract/model" - chrecipe "github.com/doublecloud/tross/pkg/providers/clickhouse/recipe" - "github.com/doublecloud/tross/pkg/providers/sample" - "github.com/doublecloud/tross/tests/helpers" - "github.com/stretchr/testify/require" -) - -const minNumberOfRows = 400 - -var ( - schemaName = "mtmobproxy" - TransferType = abstract.TransferTypeIncrementOnly - Source = *sample.RecipeSource() - Target = *chrecipe.MustTarget(chrecipe.WithInitFile("dump/dst.sql"), chrecipe.WithDatabase(schemaName), chrecipe.WithPrefix("DB0_")) -) - -func TestReplication(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "CH target", Port: Target.NativePort}, - )) - }() - Target.WithDefaults() - Target.Cleanup = server.DisabledCleanup - - Source.WithDefaults() - helpers.InitSrcDst(helpers.TransferID, &Source, &Target, TransferType) - - transfer := helpers.MakeTransfer(helpers.TransferID, &Source, &Target, TransferType) - - helpers.Activate(t, transfer) - require.NoError(t, helpers.WaitCond(60*time.Second, func() bool { - storage := helpers.GetSampleableStorageByModel(t, &Target) - tableDescription := abstract.TableDescription{Name: Source.SampleType, Schema: schemaName} - rowsInSrc, err := storage.ExactTableRowsCount(tableDescription.ID()) - if err != nil { - logger.Log.Errorf("reading number of rows from schema: %v, table: %v and occured error: %v", schemaName, Source.SampleType, err) - return false - } - logger.Log.Infof("number of rows in clickhouse %v", rowsInSrc) - // minimum number of rows counted according to sampleSource defaults - // maximumSleepTime = 2*minimumSleepTime = 200ms - // overall in every asyncPush 128 rows - return rowsInSrc > minNumberOfRows - })) - -} diff --git a/tests/e2e/sample2ch/replication/dump/dst.sql b/tests/e2e/sample2ch/replication/dump/dst.sql deleted file mode 100644 index d6547eec..00000000 --- a/tests/e2e/sample2ch/replication/dump/dst.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE DATABASE IF NOT EXISTS mtmobproxy; diff --git a/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_key.txt b/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_key.txt deleted file mode 100644 index dc160701..00000000 --- a/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_key.txt +++ /dev/null @@ -1,17 +0,0 @@ -{ - "payload": { - "id": 1 - }, - "schema": { - "fields": [ - { - "field": "id", - "optional": false, - "type": "int64" - } - ], - "name": "fullfillment..dectest/timmyb32r-test.Key", - "optional": false, - "type": "struct" - } -} \ No newline at end of file diff --git a/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_val.txt b/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_val.txt deleted file mode 100644 index f61df170..00000000 --- a/tests/e2e/ydb2mock/debezium/debezium_snapshot/testdata/change_item_val.txt +++ /dev/null @@ -1,456 +0,0 @@ -{ - "payload": { - "after": { - "Bool_": true, - "Date_": 18294, - "Datetime_": 1580637742000, - "Decimal_": "Nnt8pAA=", - "Double_": 2.2, - "DyNumber_": { - "scale": 0, - "value": "ew==" - }, - "Float_": 1.1, - "Int8_": 1, - "Int16_": 2, - "Int32_": 3, - "Int64_": 4, - "Interval_": 123000, - "JsonDocument_": "{}", - "Json_": "{}", - "String_": "AQ==", - "Timestamp_": 1580637742000000, - "Uint16_": 6, - "Uint32_": 7, - "Uint64_": 8, - "Uint8_": 5, - "Utf8_": "my_utf8_string", - "id": 1 - }, - "before": null, - "op": "r", - "source": { - "connector": "postgresql", - "db": "pguser", - "lsn": 0, - "name": "fullfillment", - "schema": "", - "snapshot": "true", - "table": "dectest/timmyb32r-test", - "ts_ms": 1655749575794, - "txId": 0, - "version": "1.1.2.Final", - "xmin": null - }, - "transaction": null, - "ts_ms": 1655749575794 - }, - "schema": { - "fields": [ - { - "field": "before", - "fields": [ - { - "field": "id", - "optional": false, - "type": "int64" - }, - { - "field": "Bool_", - "optional": true, - "type": "boolean" - }, - { - "field": "Int8_", - "optional": true, - "type": "int8" - }, - { - "field": "Int16_", - "optional": true, - "type": "int16" - }, - { - "field": "Int32_", - "optional": true, - "type": "int32" - }, - { - "field": "Int64_", - "optional": true, - "type": "int64" - }, - { - "field": "Uint8_", - "optional": true, - "type": "int8" - }, - { - "field": "Uint16_", - "optional": true, - "type": "int16" - }, - { - "field": "Uint32_", - "optional": true, - "type": "int32" - }, - { - "field": "Uint64_", - "optional": true, - "type": "int64" - }, - { - "field": "Float_", - "optional": true, - "type": "float" - }, - { - "field": "Double_", - "optional": true, - "type": "double" - }, - { - "field": "Decimal_", - "name": "org.apache.kafka.connect.data.Decimal", - "optional": true, - "parameters": { - "scale": "9" - }, - "type": "bytes", - "version": 1 - }, - { - "doc": "Variable scaled decimal", - "field": "DyNumber_", - "fields": [ - { - "field": "scale", - "optional": false, - "type": "int32" - }, - { - "field": "value", - "optional": false, - "type": "bytes" - } - ], - "name": "io.debezium.data.VariableScaleDecimal", - "optional": true, - "type": "struct", - "version": 1 - }, - { - "field": "String_", - "optional": true, - "type": "bytes" - }, - { - "field": "Utf8_", - "optional": true, - "type": "string" - }, - { - "field": "Json_", - "name": "io.debezium.data.Json", - "optional": true, - "type": "string", - "version": 1 - }, - { - "field": "JsonDocument_", - "name": "io.debezium.data.Json", - "optional": true, - "type": "string", - "version": 1 - }, - { - "field": "Date_", - "name": "io.debezium.time.Date", - "optional": true, - "type": "int32", - "version": 1 - }, - { - "field": "Datetime_", - "name": "io.debezium.time.Timestamp", - "optional": true, - "type": "int64", - "version": 1 - }, - { - "field": "Timestamp_", - "name": "io.debezium.time.MicroTimestamp", - "optional": true, - "type": "int64", - "version": 1 - }, - { - "field": "Interval_", - "name": "io.debezium.time.MicroDuration", - "optional": true, - "type": "int64", - "version": 1 - } - ], - "name": "fullfillment..dectest/timmyb32r-test.Value", - "optional": true, - "type": "struct" - }, - { - "field": "after", - "fields": [ - { - "field": "id", - "optional": false, - "type": "int64" - }, - { - "field": "Bool_", - "optional": true, - "type": "boolean" - }, - { - "field": "Int8_", - "optional": true, - "type": "int8" - }, - { - "field": "Int16_", - "optional": true, - "type": "int16" - }, - { - "field": "Int32_", - "optional": true, - "type": "int32" - }, - { - "field": "Int64_", - "optional": true, - "type": "int64" - }, - { - "field": "Uint8_", - "optional": true, - "type": "int8" - }, - { - "field": "Uint16_", - "optional": true, - "type": "int16" - }, - { - "field": "Uint32_", - "optional": true, - "type": "int32" - }, - { - "field": "Uint64_", - "optional": true, - "type": "int64" - }, - { - "field": "Float_", - "optional": true, - "type": "float" - }, - { - "field": "Double_", - "optional": true, - "type": "double" - }, - { - "field": "Decimal_", - "name": "org.apache.kafka.connect.data.Decimal", - "optional": true, - "parameters": { - "scale": "9" - }, - "type": "bytes", - "version": 1 - }, - { - "doc": "Variable scaled decimal", - "field": "DyNumber_", - "fields": [ - { - "field": "scale", - "optional": false, - "type": "int32" - }, - { - "field": "value", - "optional": false, - "type": "bytes" - } - ], - "name": "io.debezium.data.VariableScaleDecimal", - "optional": true, - "type": "struct", - "version": 1 - }, - { - "field": "String_", - "optional": true, - "type": "bytes" - }, - { - "field": "Utf8_", - "optional": true, - "type": "string" - }, - { - "field": "Json_", - "name": "io.debezium.data.Json", - "optional": true, - "type": "string", - "version": 1 - }, - { - "field": "JsonDocument_", - "name": "io.debezium.data.Json", - "optional": true, - "type": "string", - "version": 1 - }, - { - "field": "Date_", - "name": "io.debezium.time.Date", - "optional": true, - "type": "int32", - "version": 1 - }, - { - "field": "Datetime_", - "name": "io.debezium.time.Timestamp", - "optional": true, - "type": "int64", - "version": 1 - }, - { - "field": "Timestamp_", - "name": "io.debezium.time.MicroTimestamp", - "optional": true, - "type": "int64", - "version": 1 - }, - { - "field": "Interval_", - "name": "io.debezium.time.MicroDuration", - "optional": true, - "type": "int64", - "version": 1 - } - ], - "name": "fullfillment..dectest/timmyb32r-test.Value", - "optional": true, - "type": "struct" - }, - { - "field": "source", - "fields": [ - { - "field": "version", - "optional": false, - "type": "string" - }, - { - "field": "connector", - "optional": false, - "type": "string" - }, - { - "field": "name", - "optional": false, - "type": "string" - }, - { - "field": "ts_ms", - "optional": false, - "type": "int64" - }, - { - "default": "false", - "field": "snapshot", - "name": "io.debezium.data.Enum", - "optional": true, - "parameters": { - "allowed": "true,last,false" - }, - "type": "string", - "version": 1 - }, - { - "field": "db", - "optional": false, - "type": "string" - }, - { - "field": "schema", - "optional": false, - "type": "string" - }, - { - "field": "table", - "optional": false, - "type": "string" - }, - { - "field": "txId", - "optional": true, - "type": "int64" - }, - { - "field": "lsn", - "optional": true, - "type": "int64" - }, - { - "field": "xmin", - "optional": true, - "type": "int64" - } - ], - "name": "io.debezium.connector.postgresql.Source", - "optional": false, - "type": "struct" - }, - { - "field": "op", - "optional": false, - "type": "string" - }, - { - "field": "ts_ms", - "optional": true, - "type": "int64" - }, - { - "field": "transaction", - "fields": [ - { - "field": "id", - "optional": false, - "type": "string" - }, - { - "field": "total_order", - "optional": false, - "type": "int64" - }, - { - "field": "data_collection_order", - "optional": false, - "type": "int64" - } - ], - "optional": true, - "type": "struct" - } - ], - "name": "fullfillment..dectest/timmyb32r-test.Envelope", - "optional": false, - "type": "struct" - } -} \ No newline at end of file