From 6e6775c9d45c53d95fbfb0c6b47c8609510e6046 Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 18:33:22 -0300 Subject: [PATCH 1/7] linting checkpoint Signed-off-by: Henrique Goncalves --- cmd/trcli/config/config_test.go | 14 +- cmd/trcli/config/model.go | 34 ++--- internal/config/config.go | 2 +- internal/config/nirvana.go | 4 +- internal/logger/common.go | 23 --- internal/logger/yt_log_bundle.go | 6 +- internal/metrics/metrics.go | 2 +- internal/metrics/pidstat.go | 10 +- library/go/core/buildinfo/buildinfo.go | 2 +- .../collect/policy/inflight/inflight.go | 4 +- .../internal/pkg/registryutil/registryutil.go | 8 +- library/go/core/metrics/metrics.go | 2 +- library/go/core/metrics/mock/registry_opts.go | 12 +- library/go/core/metrics/mock/vec.go | 14 +- .../core/metrics/prometheus/registry_opts.go | 4 +- library/go/core/metrics/solomon/counter.go | 4 - .../go/core/metrics/solomon/func_counter.go | 4 - library/go/core/metrics/solomon/func_gauge.go | 4 - .../go/core/metrics/solomon/func_int_gauge.go | 4 - library/go/core/metrics/solomon/gauge.go | 4 - library/go/core/metrics/solomon/histogram.go | 4 - library/go/core/metrics/solomon/int_gauge.go | 4 - library/go/core/metrics/solomon/metrics.go | 2 +- .../go/core/metrics/solomon/registry_opts.go | 18 +-- library/go/core/metrics/solomon/spack.go | 13 +- .../core/metrics/solomon/spack_compression.go | 2 +- library/go/core/metrics/solomon/timer.go | 4 - library/go/core/resource/resource.go | 2 +- .../internal/modes/stack_trace_mode.go | 4 +- library/go/core/xerrors/mode.go | 2 +- library/go/core/xerrors/sentinel.go | 12 +- library/go/poolba/pool.go | 20 +-- library/go/ptr/ptr.go | 48 +++---- library/go/slices/filter.go | 2 +- library/go/slices/group_by.go | 2 +- library/go/slices/join.go | 2 +- library/go/slices/map.go | 6 +- library/go/slices/sort.go | 10 +- library/go/test/canon/canon.go | 6 +- library/go/test/canon/dctest.go | 2 +- library/go/test/canon/gotest.go | 2 +- library/go/test/yatest/dctest.go | 1 - library/go/test/yatest/env.go | 68 ++------- library/go/x/xruntime/stacktrace.go | 2 +- library/go/yandex/cloud/filter/filters.go | 8 +- .../go/yandex/cloud/filter/grammar/grammar.go | 8 +- library/go/yatool/root.go | 2 +- library/go/yatool/ya.go | 4 +- pkg/abstract/async_sink.go | 2 +- pkg/abstract/change_item_builders.go | 2 +- pkg/abstract/changeitem/change_item.go | 30 ++-- pkg/abstract/changeitem/kind.go | 4 +- pkg/abstract/changeitem/table_id.go | 2 +- pkg/abstract/changeitem/utils.go | 4 +- pkg/abstract/coordinator/editor.go | 2 +- pkg/abstract/coordinator/operation.go | 4 +- pkg/abstract/coordinator/transfer.go | 6 +- pkg/abstract/coordinator/transfer_state.go | 2 +- pkg/abstract/dterrors/error.go | 4 +- pkg/abstract/dterrors/error_test.go | 2 +- pkg/abstract/dterrors/errors_test_helper.go | 2 +- pkg/abstract/filter.go | 6 +- pkg/abstract/homo_valuer.go | 2 +- pkg/abstract/local_runtime.go | 10 +- pkg/abstract/middleware.go | 2 +- pkg/abstract/model/endpoint.go | 28 ++-- pkg/abstract/model/endpoint_registry.go | 4 +- pkg/abstract/model/endpoint_rotator_config.go | 22 +-- .../model/endpoint_rotator_config_test.go | 132 +++--------------- pkg/abstract/model/includeable.go | 2 +- pkg/abstract/model/tmp_policy_config.go | 8 +- pkg/abstract/model/transfer.go | 8 +- pkg/abstract/model/transfer_labels.go | 2 +- pkg/abstract/operations.go | 10 +- pkg/abstract/runtime.go | 6 +- pkg/abstract/sink.go | 4 +- pkg/abstract/storage.go | 21 ++- pkg/abstract/test_result.go | 8 +- pkg/abstract/typed_change_item.go | 34 ++--- pkg/abstract/typed_change_item_test.go | 4 +- pkg/abstract/typesystem/fallback.go | 4 +- .../typesystem/values/type_checkers.go | 4 +- pkg/base/events/cleanup.go | 2 +- pkg/base/events/table_events.go | 2 +- pkg/base/events/table_load.go | 2 +- pkg/base/events/transaction.go | 2 +- pkg/base/transfer.go | 12 +- pkg/cobraaux/cobraaux.go | 2 +- .../s3coordinator/coordinator_s3.go | 2 +- pkg/csv/error.go | 1 - pkg/csv/reader.go | 4 +- pkg/csv/reader_test.go | 2 +- pkg/csv/splitter.go | 6 +- pkg/dbaas/host_port.go | 2 +- pkg/dblog/tablequery/storage.go | 2 +- pkg/dblog/utils.go | 5 +- pkg/debezium/common/field_receiver.go | 10 +- pkg/debezium/emitter_value_converter.go | 26 ++-- .../packer/lightning_cache/lightning_cache.go | 2 +- pkg/debezium/packer/packer.go | 2 +- pkg/debezium/parameters/parameters.go | 4 +- pkg/debezium/pg/emitter.go | 2 +- pkg/debezium/pg/tests/emitter_chain_test.go | 4 +- pkg/debezium/pg/tests/receiver_test.go | 92 ++++++------ pkg/debezium/typeutil/helpers.go | 121 ++++++++-------- pkg/debezium/typeutil/helpers_test.go | 4 +- pkg/errors/categories/category.go | 2 +- pkg/errors/categorized.go | 4 +- pkg/errors/coded/error.go | 4 +- pkg/errors/coded/registry.go | 2 +- pkg/functions/cloud_functions.go | 3 +- pkg/instanceutil/metadata_service.go | 6 +- pkg/maplock/mutex.go | 18 +-- pkg/maplock/mutex_test.go | 12 +- pkg/metering/initializer_os.go | 1 - pkg/metering/rows_metric.go | 6 +- pkg/middlewares/async/bufferer/bufferer.go | 4 +- pkg/middlewares/async/measurer.go | 2 +- pkg/middlewares/error_tracker.go | 2 +- pkg/middlewares/interval_throttler.go | 2 +- pkg/middlewares/memthrottle/middleware.go | 6 +- pkg/middlewares/nonrow_separator.go | 2 +- pkg/middlewares/retrier.go | 2 +- pkg/middlewares/statistician.go | 2 +- pkg/middlewares/table_temporator.go | 2 +- pkg/middlewares/type_strictness_tracker.go | 2 +- pkg/parsequeue/waitable_parsequeue.go | 2 +- pkg/parsers/abstract.go | 4 +- pkg/parsers/generic/generic_parser.go | 2 +- .../registry/audittrailsv1/engine/parser.go | 2 +- .../registry/cloudevents/engine/protobuf.go | 2 +- .../cloudevents/engine/testutils/testutils.go | 2 +- .../confluentschemaregistry/engine/parser.go | 4 +- .../registry/debezium/engine/parser.go | 2 +- .../engine/fallback_timestamp_as_datetime.go | 4 +- .../protoparser/proto_parser_config.go | 2 +- .../protobuf/protoparser/proto_parser_test.go | 2 +- .../registry/raw2table/engine/parser_test.go | 2 +- pkg/parsers/scanner/protoseq_scanner.go | 4 +- pkg/parsers/tests/generic_parser_test.go | 26 +--- pkg/parsers/utils.go | 6 +- pkg/pgha/pg.go | 4 +- pkg/predicate/parser.go | 18 +-- pkg/predicate/token.go | 14 +- pkg/providers/airbyte/provider.go | 2 +- pkg/providers/airbyte/provider_model.go | 2 +- pkg/providers/airbyte/storage.go | 6 +- pkg/providers/airbyte/storage_incremental.go | 2 +- pkg/providers/bigquery/provider.go | 2 +- pkg/providers/clickhouse/a2_cluster_tables.go | 2 +- pkg/providers/clickhouse/a2_data_provider.go | 2 +- .../clickhouse/async/model/parts/part.go | 2 +- pkg/providers/clickhouse/async/sink.go | 2 +- pkg/providers/clickhouse/async/streamer.go | 2 +- pkg/providers/clickhouse/columntypes/types.go | 4 +- pkg/providers/clickhouse/conn/connection.go | 6 +- .../clickhouse/errors/check_distributed.go | 8 +- pkg/providers/clickhouse/format/csv_event.go | 2 +- .../clickhouse/format/json_compact_event.go | 2 +- .../clickhouse/httpclient/http_client_impl.go | 2 +- .../clickhouse/httpclient/http_client_mock.go | 2 +- .../clickhouse/httpuploader/marshal.go | 8 +- .../clickhouse/httpuploader/uploader.go | 2 +- .../clickhouse/model/model_ch_destination.go | 12 +- .../clickhouse/model/model_sink_params.go | 2 +- pkg/providers/clickhouse/provider.go | 2 +- pkg/providers/clickhouse/schema/ddl.go | 4 +- pkg/providers/clickhouse/schema/describe.go | 4 +- pkg/providers/clickhouse/schema/engine.go | 28 +--- pkg/providers/clickhouse/storage.go | 12 +- pkg/providers/clickhouse/toast.go | 4 +- pkg/providers/coralogix/api.go | 4 +- pkg/providers/coralogix/provider.go | 2 +- pkg/providers/datadog/provider.go | 2 +- pkg/providers/delta/action/action.go | 17 +-- pkg/providers/delta/golden_storage_test.go | 4 +- pkg/providers/delta/model_source.go | 2 +- pkg/providers/delta/protocol/checkpoint.go | 4 +- .../delta/protocol/protocol_golden_test.go | 2 +- pkg/providers/delta/protocol/replayer.go | 2 +- pkg/providers/delta/protocol/snapshot.go | 26 +--- .../delta/protocol/snapshot_reader.go | 2 +- pkg/providers/delta/protocol/table_config.go | 6 +- pkg/providers/delta/provider.go | 4 +- pkg/providers/delta/storage.go | 4 +- pkg/providers/delta/storage_sharding.go | 2 +- pkg/providers/delta/storage_snapshotable.go | 2 +- pkg/providers/elastic/client.go | 13 +- pkg/providers/elastic/dump_index.go | 2 +- pkg/providers/elastic/provider.go | 2 +- pkg/providers/elastic/schema.go | 4 +- pkg/providers/elastic/sharding_storage.go | 2 +- pkg/providers/elastic/sink.go | 14 +- pkg/providers/elastic/storage.go | 2 +- pkg/providers/error_codes.go | 8 +- pkg/providers/eventhub/provider.go | 2 +- pkg/providers/greenplum/connection.go | 8 +- pkg/providers/greenplum/ddl_operations.go | 6 +- pkg/providers/greenplum/flavour.go | 2 +- pkg/providers/greenplum/gptx.go | 6 +- pkg/providers/greenplum/model_gp_source.go | 9 +- pkg/providers/greenplum/mutexed_postgreses.go | 4 +- pkg/providers/greenplum/progress.go | 2 +- pkg/providers/greenplum/progress_test.go | 4 +- pkg/providers/greenplum/provider.go | 2 +- pkg/providers/greenplum/segpointerpool.go | 4 +- pkg/providers/greenplum/sink.go | 4 +- pkg/providers/greenplum/sink_test.go | 2 +- pkg/providers/greenplum/storage.go | 4 +- pkg/providers/kafka/model_connection.go | 2 +- pkg/providers/kafka/model_source.go | 22 +-- pkg/providers/kafka/provider.go | 2 +- pkg/providers/kafka/reader.go | 2 +- pkg/providers/kafka/source.go | 5 +- pkg/providers/kafka/writer/writer_mock.go | 2 +- pkg/providers/kinesis/consumer/group.go | 2 +- pkg/providers/kinesis/consumer/group_all.go | 4 +- pkg/providers/kinesis/consumer/options.go | 16 +-- pkg/providers/kinesis/consumer/store.go | 4 +- pkg/providers/kinesis/source.go | 19 +-- pkg/providers/mongo/batcher.go | 20 ++- pkg/providers/mongo/bson.go | 4 +- pkg/providers/mongo/change_stream_watcher.go | 2 +- pkg/providers/mongo/client.go | 17 +-- pkg/providers/mongo/convert.go | 2 +- .../mongo/database_document_key_watcher.go | 4 +- .../mongo/database_full_document_watcher.go | 2 +- .../mongo/fallback_dvalue_json_repack.go | 2 +- pkg/providers/mongo/local_oplog_rs_watcher.go | 4 +- .../mongo/model_mongo_connection_options.go | 2 +- pkg/providers/mongo/model_mongo_source.go | 6 +- pkg/providers/mongo/mongo_recipe.go | 2 +- pkg/providers/mongo/namespace_only_watcher.go | 6 +- pkg/providers/mongo/oplog_v2_parser.go | 4 +- pkg/providers/mongo/provider.go | 10 +- pkg/providers/mongo/sharding_storage.go | 26 ++-- pkg/providers/mongo/source.go | 4 +- pkg/providers/mongo/storage.go | 4 +- pkg/providers/mongo/time.go | 2 +- pkg/providers/mongo/version.go | 2 +- pkg/providers/mysql/canal.go | 10 +- pkg/providers/mysql/config.go | 2 +- .../mysql/mysql_connection_params.go | 2 +- pkg/providers/mysql/mysqlrecipe/container.go | 6 +- pkg/providers/mysql/provider.go | 2 +- pkg/providers/mysql/queries_builder.go | 4 +- pkg/providers/mysql/rows.go | 2 +- pkg/providers/mysql/sink.go | 2 +- pkg/providers/mysql/sink_test.go | 2 +- pkg/providers/mysql/storage.go | 1 - pkg/providers/mysql/sync.go | 5 +- pkg/providers/mysql/tasks.go | 2 +- .../mysql/unmarshaller/replication/hetero.go | 2 +- .../mysql/unmarshaller/types/temporal.go | 2 +- pkg/providers/mysql/utils.go | 2 +- pkg/providers/opensearch/provider.go | 2 +- pkg/providers/postgres/change_processor.go | 8 +- pkg/providers/postgres/changeitems_fetcher.go | 4 +- pkg/providers/postgres/client.go | 24 ++-- pkg/providers/postgres/client_test.go | 4 +- pkg/providers/postgres/composite.go | 6 +- pkg/providers/postgres/conn.go | 2 +- pkg/providers/postgres/date.go | 4 +- pkg/providers/postgres/incremental_storage.go | 2 +- .../postgres/model_pg_destination.go | 2 +- pkg/providers/postgres/model_pg_source.go | 2 +- .../postgres/model_pg_storage_params.go | 2 +- pkg/providers/postgres/pg_dump.go | 37 +++-- .../postgres/pgrecipe/postgres_recipe.go | 4 +- pkg/providers/postgres/provider.go | 2 +- pkg/providers/postgres/publisher.go | 6 +- pkg/providers/postgres/publisher_polling.go | 2 +- .../postgres/publisher_replication.go | 8 +- pkg/providers/postgres/schema.go | 10 +- pkg/providers/postgres/sequence.go | 4 +- .../postgres/sequencer/lsn_transaction.go | 8 +- .../postgres/sequencer/progress_info.go | 6 +- pkg/providers/postgres/sequencer/sequencer.go | 4 +- pkg/providers/postgres/sink.go | 14 +- pkg/providers/postgres/sqltimestamp/parse.go | 2 +- pkg/providers/postgres/storage.go | 10 +- pkg/providers/postgres/timestamp.go | 6 +- pkg/providers/postgres/timestamptz.go | 4 +- pkg/providers/postgres/txutils.go | 2 +- pkg/providers/postgres/unmarshaller.go | 4 +- .../postgres/wal2json_parser_test.go | 4 +- pkg/providers/provider.go | 14 +- pkg/providers/provider_tasks.go | 2 +- pkg/providers/s3/model_source.go | 2 +- pkg/providers/s3/provider/provider.go | 2 +- pkg/providers/s3/pusher/pusher_state.go | 4 +- pkg/providers/s3/reader/estimator_test.go | 2 +- pkg/providers/s3/reader/reader_csv.go | 2 +- pkg/providers/s3/reader/reader_json_line.go | 2 +- pkg/providers/s3/reader/s3_reader.go | 6 +- pkg/providers/s3/sink/file_cache.go | 4 +- pkg/providers/s3/sink/object_range.go | 8 -- pkg/providers/s3/sink/testutil/fake_client.go | 2 +- pkg/providers/s3/sink/uploader.go | 6 +- .../sharded_replication_test.go | 2 +- .../s3/storage/storage_incremental.go | 2 +- pkg/providers/s3/storage/storage_sharding.go | 2 +- pkg/providers/stdout/provider.go | 2 +- pkg/providers/ydb/decimal/parse.go | 3 - pkg/providers/ydb/logadapter/adapter.go | 2 +- pkg/providers/ydb/provider.go | 2 +- pkg/providers/ydb/sink.go | 9 +- pkg/providers/ydb/source_tasks.go | 2 +- pkg/providers/ydb/source_test.go | 2 +- pkg/providers/ydb/storage_sampleable.go | 8 +- pkg/providers/ydb/utils.go | 16 +-- pkg/providers/yt/copy/source/source.go | 2 +- pkg/providers/yt/cypress.go | 2 +- pkg/providers/yt/executable.go | 5 +- pkg/providers/yt/init/provider.go | 2 +- pkg/providers/yt/lfstaging/close_gaps.go | 16 --- pkg/providers/yt/lfstaging/lock.go | 40 ------ pkg/providers/yt/lfstaging/rows.go | 4 +- pkg/providers/yt/model_yt_destination.go | 4 +- pkg/providers/yt/provider/dataobjects/part.go | 2 +- pkg/providers/yt/provider/snapshot.go | 4 +- pkg/providers/yt/provider/source.go | 2 +- pkg/providers/yt/recipe/env.go | 2 +- pkg/providers/yt/sink/common.go | 4 +- pkg/providers/yt/sink/ordered_table.go | 8 +- pkg/providers/yt/sink/single_static_table.go | 6 +- .../yt/sink/single_static_table_test.go | 14 +- pkg/providers/yt/sink/sink.go | 2 +- pkg/providers/yt/sink/sorted_table.go | 2 +- pkg/providers/yt/sink/static_table.go | 11 +- .../yt/sink/v2/transactions/state_storage.go | 2 +- pkg/providers/yt/sink/versioned_table.go | 2 +- pkg/randutil/randutil.go | 6 +- pkg/runtime/local/logger_injestor.go | 2 +- pkg/runtime/local/replication.go | 10 +- pkg/runtime/local/task_sync_runtime.go | 2 +- pkg/runtime/shared/limits.go | 2 +- .../confluent/load_balancer_test.go | 7 +- pkg/schemaregistry/format/common.go | 2 +- pkg/serializer/parquet.go | 2 +- pkg/serializer/parquet_format.go | 2 +- pkg/serializer/queue/debezium_serializer.go | 2 +- pkg/serializer/queue/mirror_serializer.go | 4 +- pkg/serializer/queue/native_serializer.go | 4 +- pkg/serverutil/endpoint.go | 2 +- pkg/sink/sink.go | 6 +- pkg/source/eventsource/source.go | 2 +- pkg/stats/fallbacks.go | 2 +- pkg/stats/middleware_bufferer.go | 4 +- pkg/stats/middleware_transformer.go | 2 +- pkg/stats/pool.go | 2 +- pkg/stringutil/stringutil.go | 6 +- pkg/transformer/abstract.go | 2 +- .../registry/clickhouse/clickhouse_local.go | 2 +- pkg/transformer/registry/filter/filter.go | 10 +- pkg/transformer/registry/filter_rows/util.go | 24 +--- .../registry/lambda/lambda_test.go | 2 +- .../raw_doc_grouper/raw_cdc_doc_grouper.go | 15 +- .../raw_cdc_doc_grouper_test.go | 24 ++-- .../raw_doc_grouper/raw_data_utils.go | 6 +- .../raw_doc_grouper/raw_doc_grouper.go | 5 - .../raw_doc_grouper/raw_doc_grouper_test.go | 26 ++-- .../raw_doc_grouper/raw_doc_test_utils.go | 2 +- .../registry/to_string/to_string_test.go | 7 +- pkg/util/backoff.go | 2 +- pkg/util/castx/caste.go | 2 +- pkg/util/cli/spinner.go | 6 +- pkg/util/diff/diff.go | 24 +--- pkg/util/generics.go | 2 +- pkg/util/glob/glob.go | 15 +- pkg/util/hash.go | 2 +- pkg/util/jsonx/json_null.go | 2 +- pkg/util/jsonx/value_decoder.go | 6 +- pkg/util/make_chan_with_error.go | 2 +- pkg/util/map_keys_in_order.go | 2 +- pkg/util/multibuf/pooledmultibuf.go | 8 +- pkg/util/oneof/oneof_value.go | 4 +- pkg/util/pool/impl.go | 2 +- pkg/util/pool/pool.go | 4 +- pkg/util/ports.go | 2 +- pkg/util/queues/sequencer/sequencer.go | 2 +- pkg/util/queues/sequencer/util_kafka.go | 6 +- pkg/util/runtime.go | 2 +- pkg/util/smart_timer.go | 2 +- pkg/util/snaker.go | 15 +- pkg/util/throttler/throttler.go | 2 +- pkg/util/time.go | 4 +- pkg/util/validators/validators.go | 2 +- pkg/util/xd_array.go | 4 +- pkg/worker/tasks/activate_delivery.go | 2 +- pkg/worker/tasks/add_tables.go | 11 -- pkg/worker/tasks/checksum.go | 2 +- pkg/worker/tasks/load_sharded_snapshot.go | 2 +- pkg/worker/tasks/load_snapshot.go | 4 +- .../load_sharded_snapshot_test.go | 2 +- .../tasks/snapshot_table_metrics_tracker.go | 2 +- .../tasks/snapshot_table_progress_tracker.go | 6 +- pkg/worker/tasks/transitional_upload.go | 4 +- pkg/worker/tasks/update_transfer.go | 2 +- pkg/xtls/create.go | 4 +- 400 files changed, 1088 insertions(+), 1593 deletions(-) delete mode 100644 pkg/providers/yt/lfstaging/lock.go diff --git a/cmd/trcli/config/config_test.go b/cmd/trcli/config/config_test.go index a930850e..c27776c6 100644 --- a/cmd/trcli/config/config_test.go +++ b/cmd/trcli/config/config_test.go @@ -9,10 +9,8 @@ import ( ) func TestParseTransferYaml_WithEnvSubstitution(t *testing.T) { - require.NoError(t, os.Setenv("FOO", "secret1")) - require.NoError(t, os.Setenv("BAR", "secret2")) - defer os.Unsetenv("FOO") - defer os.Unsetenv("BAR") + t.Setenv("FOO", "secret1") + t.Setenv("BAR", "secret2") transfer, err := ParseTransferYaml([]byte(` src: @@ -31,8 +29,8 @@ dst: } func TestParserTransferYaml_WithRawYaml(t *testing.T) { - require.NoError(t, os.Setenv("FOO", "secret1")) - require.NoError(t, os.Setenv("BAR", "secret2")) + t.Setenv("FOO", "secret1") + t.Setenv("BAR", "secret2") defer os.Unsetenv("FOO") defer os.Unsetenv("BAR") @@ -53,8 +51,8 @@ dst: } func TestParserTransferYaml_WithYaml(t *testing.T) { - require.NoError(t, os.Setenv("FOO", "secret1")) - require.NoError(t, os.Setenv("BAR", "secret2")) + t.Setenv("FOO", "secret1") + t.Setenv("BAR", "secret2") defer os.Unsetenv("FOO") defer os.Unsetenv("BAR") diff --git a/cmd/trcli/config/model.go b/cmd/trcli/config/model.go index 5caa1b3c..cd4af2c0 100644 --- a/cmd/trcli/config/model.go +++ b/cmd/trcli/config/model.go @@ -25,27 +25,27 @@ func (e Endpoint) RawParams() string { } type Runtime struct { - Type string - Params interface{} + Type string `json:"type" yaml:"type"` + Params interface{} `json:"params" yaml:"params"` } type UploadTables struct { - Tables []abstract.TableDescription + Tables []abstract.TableDescription `json:"tables" yaml:"tables"` } type TransferYamlView struct { - ID string - TransferName string - Description string - Labels string - Status model.TransferStatus - Type abstract.TransferType - FolderID string - CloudID string - Src Endpoint - Dst Endpoint - RegularSnapshot *abstract.RegularSnapshot `yaml:"regular_snapshot"` - Transformation *transformer.Transformers `yaml:"transformation"` - DataObjects *model.DataObjects `yaml:"data_objects"` - TypeSystemVersion int `yaml:"type_system_version"` + ID string `json:"id" yaml:"id"` + TransferName string `json:"transfer_name" yaml:"transfer_name"` + Description string `json:"description" yaml:"description"` + Labels string `json:"labels" yaml:"labels"` + Status model.TransferStatus `json:"status" yaml:"status"` + Type abstract.TransferType `json:"type" yaml:"type"` + FolderID string `json:"folder_id" yaml:"folder_id"` + CloudID string `json:"cloud_id" yaml:"cloud_id"` + Src Endpoint `json:"src" yaml:"src"` + Dst Endpoint `json:"dst" yaml:"dst"` + RegularSnapshot *abstract.RegularSnapshot `json:"regular_snapshot" yaml:"regular_snapshot"` + Transformation *transformer.Transformers `json:"transformation" yaml:"transformation"` + DataObjects *model.DataObjects `json:"data_objects" yaml:"data_objects"` + TypeSystemVersion int `json:"type_system_version" yaml:"type_system_version"` } diff --git a/internal/config/config.go b/internal/config/config.go index 8c502145..4dab59e6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,7 +5,7 @@ import ( "os" ) -// legacy thing - used only for back-compatiblity command-line tools pg* +// legacy thing - used only for back-compatiblity command-line tools pg*. type DBConfig struct { DBHost string `json:"db_host" yaml:"db_host"` DBPort int `json:"db_port" yaml:"db_port"` diff --git a/internal/config/nirvana.go b/internal/config/nirvana.go index 8f0ccd07..2ddb129e 100644 --- a/internal/config/nirvana.go +++ b/internal/config/nirvana.go @@ -3,7 +3,7 @@ package config import ( "encoding/json" "fmt" - "io/ioutil" + "os" ) type JobContext struct { @@ -35,7 +35,7 @@ type JobContext struct { } func TryParseNirvanaConfig() (*Config, error) { - dat, err := ioutil.ReadFile("job_context.json") + dat, err := os.ReadFile("job_context.json") if err != nil { return nil, err } diff --git a/internal/logger/common.go b/internal/logger/common.go index e1d4a0ec..839c8cce 100644 --- a/internal/logger/common.go +++ b/internal/logger/common.go @@ -59,26 +59,3 @@ func copySlice(src []byte) []byte { copy(dst, src) return dst } - -func copyBytes(source []byte) []byte { - dup := make([]byte, len(source)) - copy(dup, source) - return dup -} - -func newConsoleLogger() log.Logger { - levels := getEnvLogLevels() - if os.Getenv("CONSOLE_LOG_LEVEL") != "" { - levels = parseLevel(os.Getenv("CONSOLE_LOG_LEVEL")) - } - - levelEnabler := levelEnablerFactory(levels.Zap) - - encoderConfig := zap.CLIConfig(levels.Log).EncoderConfig - encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - - encoder := zapcore.NewConsoleEncoder(encoderConfig) - writeSyncer := zapcore.AddSync(os.Stdout) - - return newLogger(zapcore.NewCore(encoder, writeSyncer, levelEnabler)) -} diff --git a/internal/logger/yt_log_bundle.go b/internal/logger/yt_log_bundle.go index 34e7d40e..7597aef6 100644 --- a/internal/logger/yt_log_bundle.go +++ b/internal/logger/yt_log_bundle.go @@ -20,7 +20,7 @@ type ytLogBundleImpl struct { } // NewYtLogBundle constructs LogBundle from two loggers: original one and separate logger for YT. Acts like -// standard zap.Must(config), but hooks system log calls "With" to apply to zap.Must(ytConfig) +// standard zap.Must(config), but hooks system log calls "With" to apply to zap.Must(ytConfig). func NewYtLogBundle(log, ytLogger log.Logger) YtLogBundle { return &ytLogBundleImpl{ Logger: log, @@ -38,13 +38,13 @@ func (l *ytLogBundleImpl) With(fields ...log.Field) log.Logger { return &lCopy } -// ExtractYTLogger extracts preconfigured YT logger with corresponding registered 'With' calls +// ExtractYTLogger extracts preconfigured YT logger with corresponding registered 'With' calls. func (l *ytLogBundleImpl) ExtractYTLogger() log.Logger { return l.ytLogger } // ExtractYTLogger is a helper function to extract YT log. If the `lgr` parameter is not of thge type -// logger.YtLogBundle, then the log returned itself, otherwise log for YT is returned +// logger.YtLogBundle, then the log returned itself, otherwise log for YT is returned. func ExtractYTLogger(lgr log.Logger) log.Logger { if ytLogBundle, ok := lgr.(YtLogBundle); ok { lgr.Infof("YT Logger extracted successfully") diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f489a9e6..e980a9de 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -155,7 +155,7 @@ func (r registryWrapper) DurationHistogramVec(name string, buckets metrics.Durat return r.registry.DurationHistogramVec(r.nameProcessor(name), buckets, labels) } -// DTSUPPORT-1013 +// DTSUPPORT-1013. func refineValue(original string) string { filteredCharacters := []rune(original) reqLen := len(filteredCharacters) diff --git a/internal/metrics/pidstat.go b/internal/metrics/pidstat.go index fabf592e..389ba0f8 100644 --- a/internal/metrics/pidstat.go +++ b/internal/metrics/pidstat.go @@ -4,8 +4,8 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" "math" + "os" "os/exec" "path" "runtime" @@ -78,7 +78,7 @@ func stat(pid int, statType string) (*SysInfo, error) { var clkTck float64 = 100 var pageSize float64 = 4096 - uptimeFileBytes, _ := ioutil.ReadFile(path.Join("/proc", "uptime")) + uptimeFileBytes, _ := os.ReadFile(path.Join("/proc", "uptime")) uptime := parseFloat(strings.Split(string(uptimeFileBytes), " ")[0]) clkTckStdout, err := exec.Command("getconf", "CLK_TCK").Output() @@ -91,7 +91,7 @@ func stat(pid int, statType string) (*SysInfo, error) { pageSize = parseFloat(formatStdOut(pageSizeStdout, 0)[0]) } - procStatFileBytes, _ := ioutil.ReadFile(path.Join("/proc", strconv.Itoa(pid), "stat")) + procStatFileBytes, _ := os.ReadFile(path.Join("/proc", strconv.Itoa(pid), "stat")) splitAfter := strings.SplitAfter(string(procStatFileBytes), ")") if len(splitAfter) == 0 || len(splitAfter) == 1 { @@ -118,7 +118,7 @@ func stat(pid int, statType string) (*SysInfo, error) { _utime = _history.utime } total := stat.stime - _stime + stat.utime - _utime - total = total / clkTck + total /= clkTck seconds := stat.start - uptime if _history.uptime != 0 { @@ -154,7 +154,7 @@ func getOpenFilesCount(pid int, platform string) (float64, error) { return float64(bytes.Count(stdout, []byte(eol)) - 1), nil } -// GetStat may return incomplete result (with some fields unfilled) +// GetStat may return incomplete result (with some fields unfilled). func GetStat(pid int) (*SysInfo, error) { platform = runtime.GOOS if eol = "\n"; strings.Index(platform, "win") == 0 { diff --git a/library/go/core/buildinfo/buildinfo.go b/library/go/core/buildinfo/buildinfo.go index 0f3a4a99..80ba92a1 100644 --- a/library/go/core/buildinfo/buildinfo.go +++ b/library/go/core/buildinfo/buildinfo.go @@ -50,7 +50,7 @@ var Info BuildInfo // InitBuildInfo is internal, but exported API. // -// This function is called from the main package by the code generated in build/scripts/vcs_info.py +// This function is called from the main package by the code generated in build/scripts/vcs_info.py. func InitBuildInfo(buildinfo map[string]string) { Info.ProgramVersion = strings.TrimRight(buildinfo["PROGRAM_VERSION"], " ") Info.CustomVersion = strings.TrimRight(buildinfo["CUSTOM_VERSION"], " ") diff --git a/library/go/core/metrics/collect/policy/inflight/inflight.go b/library/go/core/metrics/collect/policy/inflight/inflight.go index 1e12b02c..e1b0525c 100644 --- a/library/go/core/metrics/collect/policy/inflight/inflight.go +++ b/library/go/core/metrics/collect/policy/inflight/inflight.go @@ -2,7 +2,6 @@ package inflight import ( "context" - "sync" "sync/atomic" "time" @@ -13,8 +12,7 @@ import ( var _ metrics.CollectPolicy = (*inflightPolicy)(nil) type inflightPolicy struct { - addCollectLock sync.Mutex - collect atomic.Value // func(ctx context.Context) + collect atomic.Value // func(ctx context.Context) minUpdateInterval time.Duration lastUpdate time.Time diff --git a/library/go/core/metrics/internal/pkg/registryutil/registryutil.go b/library/go/core/metrics/internal/pkg/registryutil/registryutil.go index 44257cc8..f2b5f2b8 100644 --- a/library/go/core/metrics/internal/pkg/registryutil/registryutil.go +++ b/library/go/core/metrics/internal/pkg/registryutil/registryutil.go @@ -16,7 +16,7 @@ var keyBufferPool = sync.Pool{New: func() any { return bytes.NewBuffer(make([]byte, 0, 512)) }} -// BuildRegistryKey creates registry name based on given prefix and tags +// BuildRegistryKey creates registry name based on given prefix and tags. func BuildRegistryKey(prefix string, tags map[string]string) string { buffer := keyBufferPool.Get().(*bytes.Buffer) defer func() { @@ -32,7 +32,7 @@ func BuildRegistryKey(prefix string, tags map[string]string) string { } // BuildFQName returns name parts joined by given separator. -// Mainly used to append prefix to registry +// Mainly used to append prefix to registry. func BuildFQName(separator string, parts ...string) (name string) { var b strings.Builder for _, p := range parts { @@ -47,7 +47,7 @@ func BuildFQName(separator string, parts ...string) (name string) { return b.String() } -// MergeTags merges 2 sets of tags with the tags from tagsRight overriding values from tagsLeft +// MergeTags merges 2 sets of tags with the tags from tagsRight overriding values from tagsLeft. func MergeTags(leftTags map[string]string, rightTags map[string]string) map[string]string { if leftTags == nil && rightTags == nil { return nil @@ -90,7 +90,7 @@ func StringifyTags(tags map[string]string, buffer *bytes.Buffer) { } } -// VectorHash computes hash of metrics vector element +// VectorHash computes hash of metrics vector element. func VectorHash(tags map[string]string, labels []string) (uint64, error) { if len(tags) != len(labels) { return 0, errors.New("inconsistent tags and labels sets") diff --git a/library/go/core/metrics/metrics.go b/library/go/core/metrics/metrics.go index 43beaa47..32c01de9 100644 --- a/library/go/core/metrics/metrics.go +++ b/library/go/core/metrics/metrics.go @@ -156,7 +156,7 @@ type Registry interface { DurationHistogramVec(name string, buckets DurationBuckets, labels []string) TimerVec } -// MetricsStreamer represents a registry that can stream collected metrics data to some destination +// MetricsStreamer represents a registry that can stream collected metrics data to some destination. type MetricsStreamer interface { Stream(context.Context, io.Writer) (int, error) } diff --git a/library/go/core/metrics/mock/registry_opts.go b/library/go/core/metrics/mock/registry_opts.go index b2ed2cbc..6df1f642 100644 --- a/library/go/core/metrics/mock/registry_opts.go +++ b/library/go/core/metrics/mock/registry_opts.go @@ -11,7 +11,7 @@ type RegistryOpts struct { AllowLoadRegisteredMetrics bool } -// NewRegistryOpts returns new initialized instance of RegistryOpts +// NewRegistryOpts returns new initialized instance of RegistryOpts. func NewRegistryOpts() *RegistryOpts { return &RegistryOpts{ Separator: '.', @@ -19,13 +19,13 @@ func NewRegistryOpts() *RegistryOpts { } } -// SetTags overrides existing tags +// SetTags overrides existing tags. func (o *RegistryOpts) SetTags(tags map[string]string) *RegistryOpts { o.Tags = tags return o } -// AddTags merges given tags with existing +// AddTags merges given tags with existing. func (o *RegistryOpts) AddTags(tags map[string]string) *RegistryOpts { for k, v := range tags { o.Tags[k] = v @@ -33,19 +33,19 @@ func (o *RegistryOpts) AddTags(tags map[string]string) *RegistryOpts { return o } -// SetPrefix overrides existing prefix +// SetPrefix overrides existing prefix. func (o *RegistryOpts) SetPrefix(prefix string) *RegistryOpts { o.Prefix = prefix return o } -// AppendPrefix adds given prefix as postfix to existing using separator +// AppendPrefix adds given prefix as postfix to existing using separator. func (o *RegistryOpts) AppendPrefix(prefix string) *RegistryOpts { o.Prefix = registryutil.BuildFQName(string(o.Separator), o.Prefix, prefix) return o } -// SetSeparator overrides existing separator +// SetSeparator overrides existing separator. func (o *RegistryOpts) SetSeparator(separator rune) *RegistryOpts { o.Separator = separator return o diff --git a/library/go/core/metrics/mock/vec.go b/library/go/core/metrics/mock/vec.go index f69817cb..22dae239 100644 --- a/library/go/core/metrics/mock/vec.go +++ b/library/go/core/metrics/mock/vec.go @@ -14,7 +14,7 @@ type MetricsVector interface { Reset() } -// Vector is base implementation of vector of metrics of any supported type +// Vector is base implementation of vector of metrics of any supported type. type Vector struct { Labels []string Mtx sync.RWMutex // Protects metrics. @@ -60,7 +60,7 @@ func (v *Vector) Reset() { var _ metrics.CounterVec = (*CounterVec)(nil) // CounterVec stores counters and -// implements metrics.CounterVec interface +// implements metrics.CounterVec interface. type CounterVec struct { Vec MetricsVector } @@ -93,7 +93,7 @@ func (v *CounterVec) Reset() { var _ metrics.GaugeVec = new(GaugeVec) // GaugeVec stores gauges and -// implements metrics.GaugeVec interface +// implements metrics.GaugeVec interface. type GaugeVec struct { Vec MetricsVector } @@ -126,7 +126,7 @@ func (v *GaugeVec) Reset() { var _ metrics.IntGaugeVec = new(IntGaugeVec) // IntGaugeVec stores gauges and -// implements metrics.IntGaugeVec interface +// implements metrics.IntGaugeVec interface. type IntGaugeVec struct { Vec MetricsVector } @@ -159,7 +159,7 @@ func (v *IntGaugeVec) Reset() { var _ metrics.TimerVec = new(TimerVec) // TimerVec stores timers and -// implements metrics.TimerVec interface +// implements metrics.TimerVec interface. type TimerVec struct { Vec MetricsVector } @@ -192,7 +192,7 @@ func (v *TimerVec) Reset() { var _ metrics.HistogramVec = (*HistogramVec)(nil) // HistogramVec stores histograms and -// implements metrics.HistogramVec interface +// implements metrics.HistogramVec interface. type HistogramVec struct { Vec MetricsVector } @@ -225,7 +225,7 @@ func (v *HistogramVec) Reset() { var _ metrics.TimerVec = (*DurationHistogramVec)(nil) // DurationHistogramVec stores duration histograms and -// implements metrics.TimerVec interface +// implements metrics.TimerVec interface. type DurationHistogramVec struct { Vec MetricsVector } diff --git a/library/go/core/metrics/prometheus/registry_opts.go b/library/go/core/metrics/prometheus/registry_opts.go index 2c6b444f..5e07261c 100644 --- a/library/go/core/metrics/prometheus/registry_opts.go +++ b/library/go/core/metrics/prometheus/registry_opts.go @@ -85,13 +85,13 @@ func (o *RegistryOpts) AddCollectors( } // SetNameSanitizer sets a functions which will be called for each metric's name. -// It allows to alter names, for example to replace invalid characters +// It allows to alter names, for example to replace invalid characters. func (o *RegistryOpts) SetNameSanitizer(v func(string) string) *RegistryOpts { o.NameSanitizer = v return o } -// SetStreamFormat sets default metrics stream format +// SetStreamFormat sets default metrics stream format. func (o *RegistryOpts) SetStreamFormat(format expfmt.Format) *RegistryOpts { o.StreamFormat = format return o diff --git a/library/go/core/metrics/solomon/counter.go b/library/go/core/metrics/solomon/counter.go index 699f1661..acb770ed 100644 --- a/library/go/core/metrics/solomon/counter.go +++ b/library/go/core/metrics/solomon/counter.go @@ -67,10 +67,6 @@ func (c *Counter) isMemOnly() bool { return c.memOnly } -func (c *Counter) setMemOnly() { - c.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (c *Counter) MarshalJSON() ([]byte, error) { return json.Marshal(struct { diff --git a/library/go/core/metrics/solomon/func_counter.go b/library/go/core/metrics/solomon/func_counter.go index cc7681e8..bcb1212c 100644 --- a/library/go/core/metrics/solomon/func_counter.go +++ b/library/go/core/metrics/solomon/func_counter.go @@ -57,10 +57,6 @@ func (c *FuncCounter) isMemOnly() bool { return c.memOnly } -func (c *FuncCounter) setMemOnly() { - c.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (c *FuncCounter) MarshalJSON() ([]byte, error) { return json.Marshal(struct { diff --git a/library/go/core/metrics/solomon/func_gauge.go b/library/go/core/metrics/solomon/func_gauge.go index 7097974d..498d29e5 100644 --- a/library/go/core/metrics/solomon/func_gauge.go +++ b/library/go/core/metrics/solomon/func_gauge.go @@ -57,10 +57,6 @@ func (g *FuncGauge) isMemOnly() bool { return g.memOnly } -func (g *FuncGauge) setMemOnly() { - g.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (g *FuncGauge) MarshalJSON() ([]byte, error) { return json.Marshal(struct { diff --git a/library/go/core/metrics/solomon/func_int_gauge.go b/library/go/core/metrics/solomon/func_int_gauge.go index deef136f..78b7a201 100644 --- a/library/go/core/metrics/solomon/func_int_gauge.go +++ b/library/go/core/metrics/solomon/func_int_gauge.go @@ -57,10 +57,6 @@ func (g *FuncIntGauge) isMemOnly() bool { return g.memOnly } -func (g *FuncIntGauge) setMemOnly() { - g.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (g *FuncIntGauge) MarshalJSON() ([]byte, error) { return json.Marshal(struct { diff --git a/library/go/core/metrics/solomon/gauge.go b/library/go/core/metrics/solomon/gauge.go index 62702759..8c13b974 100644 --- a/library/go/core/metrics/solomon/gauge.go +++ b/library/go/core/metrics/solomon/gauge.go @@ -82,10 +82,6 @@ func (g *Gauge) isMemOnly() bool { return g.memOnly } -func (g *Gauge) setMemOnly() { - g.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (g *Gauge) MarshalJSON() ([]byte, error) { metricType := g.metricType.String() diff --git a/library/go/core/metrics/solomon/histogram.go b/library/go/core/metrics/solomon/histogram.go index 9500e3f1..1f22e6d2 100644 --- a/library/go/core/metrics/solomon/histogram.go +++ b/library/go/core/metrics/solomon/histogram.go @@ -134,10 +134,6 @@ func (h *Histogram) isMemOnly() bool { return h.memOnly } -func (h *Histogram) setMemOnly() { - h.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (h *Histogram) MarshalJSON() ([]byte, error) { valuesCopy := make([]int64, len(h.bucketValues)) diff --git a/library/go/core/metrics/solomon/int_gauge.go b/library/go/core/metrics/solomon/int_gauge.go index 2efe20ee..76fa3ff4 100644 --- a/library/go/core/metrics/solomon/int_gauge.go +++ b/library/go/core/metrics/solomon/int_gauge.go @@ -82,10 +82,6 @@ func (g *IntGauge) isMemOnly() bool { return g.memOnly } -func (g *IntGauge) setMemOnly() { - g.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (g *IntGauge) MarshalJSON() ([]byte, error) { metricType := g.metricType.String() diff --git a/library/go/core/metrics/solomon/metrics.go b/library/go/core/metrics/solomon/metrics.go index bcf892ec..cf3ad644 100644 --- a/library/go/core/metrics/solomon/metrics.go +++ b/library/go/core/metrics/solomon/metrics.go @@ -198,7 +198,7 @@ func (s Metrics) SplitToChunks(maxChunkSize int) []Metrics { return chunks } -// List return list of metrics +// List return list of metrics. func (s Metrics) List() []Metric { return slices.Clone(s.metrics) } diff --git a/library/go/core/metrics/solomon/registry_opts.go b/library/go/core/metrics/solomon/registry_opts.go index 0859a247..e9abd5a2 100644 --- a/library/go/core/metrics/solomon/registry_opts.go +++ b/library/go/core/metrics/solomon/registry_opts.go @@ -21,7 +21,7 @@ type RegistryOpts struct { Logger log.Logger } -// NewRegistryOpts returns new initialized instance of RegistryOpts +// NewRegistryOpts returns new initialized instance of RegistryOpts. func NewRegistryOpts() *RegistryOpts { return &RegistryOpts{ Separator: '.', @@ -32,19 +32,19 @@ func NewRegistryOpts() *RegistryOpts { } } -// SetUseNameTag overrides current UseNameTag opt +// SetUseNameTag overrides current UseNameTag opt. func (o *RegistryOpts) SetUseNameTag(useNameTag bool) *RegistryOpts { o.UseNameTag = useNameTag return o } -// SetTags overrides existing tags +// SetTags overrides existing tags. func (o *RegistryOpts) SetTags(tags map[string]string) *RegistryOpts { o.Tags = tags return o } -// AddTags merges given tags with existing +// AddTags merges given tags with existing. func (o *RegistryOpts) AddTags(tags map[string]string) *RegistryOpts { for k, v := range tags { o.Tags[k] = v @@ -52,25 +52,25 @@ func (o *RegistryOpts) AddTags(tags map[string]string) *RegistryOpts { return o } -// SetPrefix overrides existing prefix +// SetPrefix overrides existing prefix. func (o *RegistryOpts) SetPrefix(prefix string) *RegistryOpts { o.Prefix = prefix return o } -// AppendPrefix adds given prefix as postfix to existing using separator +// AppendPrefix adds given prefix as postfix to existing using separator. func (o *RegistryOpts) AppendPrefix(prefix string) *RegistryOpts { o.Prefix = registryutil.BuildFQName(string(o.Separator), o.Prefix, prefix) return o } -// SetSeparator overrides existing separator +// SetSeparator overrides existing separator. func (o *RegistryOpts) SetSeparator(separator rune) *RegistryOpts { o.Separator = separator return o } -// SetRated overrides existing rated flag +// SetRated overrides existing rated flag. func (o *RegistryOpts) SetRated(rated bool) *RegistryOpts { o.Rated = rated return o @@ -92,7 +92,7 @@ func (o *RegistryOpts) AddCollectors( return o } -// SetStreamFormat sets default sensors stream format +// SetStreamFormat sets default sensors stream format. func (o *RegistryOpts) SetStreamFormat(format StreamFormat) *RegistryOpts { o.StreamFormat = format return o diff --git a/library/go/core/metrics/solomon/spack.go b/library/go/core/metrics/solomon/spack.go index 7e258be7..31baddc4 100644 --- a/library/go/core/metrics/solomon/spack.go +++ b/library/go/core/metrics/solomon/spack.go @@ -110,11 +110,11 @@ func (s *spackMetric) writeLabel(se *spackEncoder, namesIdx map[string]uint32, v return err } - err = writeULEB128(&s.labels, uint32(namesIdx[name])) + err = writeULEB128(&s.labels, namesIdx[name]) if err != nil { return err } - err = writeULEB128(&s.labels, uint32(valuesIdx[value])) + err = writeULEB128(&s.labels, valuesIdx[value]) if err != nil { return err } @@ -134,17 +134,17 @@ func (s *spackMetric) writeMetric(w io.Writer, version spackVersion) error { return xerrors.Errorf("binary.Write types failed: %w", err) } - err = binary.Write(w, binary.LittleEndian, uint8(s.flags)) + err = binary.Write(w, binary.LittleEndian, s.flags) if err != nil { return xerrors.Errorf("binary.Write flags failed: %w", err) } if version >= version12 { - err = writeULEB128(w, uint32(s.nameValueIndex)) + err = writeULEB128(w, s.nameValueIndex) if err != nil { return xerrors.Errorf("writeULEB128 name value index: %w", err) } } - err = writeULEB128(w, uint32(s.labelsCount)) + err = writeULEB128(w, s.labelsCount) if err != nil { return xerrors.Errorf("writeULEB128 labels count failed: %w", err) } @@ -255,7 +255,6 @@ func (se *spackEncoder) writeLabels() ([]spackMetric, error) { if err := m.writeLabel(se, namesIdx, valuesIdx, name, value); err != nil { return nil, err } - } spackMetrics[idx] = m } @@ -327,7 +326,7 @@ func (se *spackEncoder) writeHeader(w io.Writer) error { ew.binaryWrite(uint16(se.version)) // Version ew.binaryWrite(uint16(24)) // HeaderSize ew.binaryWrite(uint8(0)) // TimePrecision(SECONDS) - ew.binaryWrite(uint8(se.compression)) // CompressionAlg + ew.binaryWrite(se.compression) // CompressionAlg ew.binaryWrite(uint32(se.labelNamePool.Len())) // LabelNamesSize ew.binaryWrite(uint32(se.labelValuePool.Len())) // LabelValuesSize ew.binaryWrite(uint32(len(se.metrics.metrics))) // MetricsCount diff --git a/library/go/core/metrics/solomon/spack_compression.go b/library/go/core/metrics/solomon/spack_compression.go index 004fe015..a1ad73b6 100644 --- a/library/go/core/metrics/solomon/spack_compression.go +++ b/library/go/core/metrics/solomon/spack_compression.go @@ -90,7 +90,7 @@ func (w *lz4CompressionWriteCloser) flushFrame() (written int, err error) { } func (w *lz4CompressionWriteCloser) Write(p []byte) (written int, err error) { - q := p[:] + q := p for len(q) > 0 { space := compressionFrameLength - len(w.buffer) if space == 0 { diff --git a/library/go/core/metrics/solomon/timer.go b/library/go/core/metrics/solomon/timer.go index f63f6ffd..01964192 100644 --- a/library/go/core/metrics/solomon/timer.go +++ b/library/go/core/metrics/solomon/timer.go @@ -61,10 +61,6 @@ func (t *Timer) isMemOnly() bool { return t.memOnly } -func (t *Timer) setMemOnly() { - t.memOnly = true -} - // MarshalJSON implements json.Marshaler. func (t *Timer) MarshalJSON() ([]byte, error) { return json.Marshal(struct { diff --git a/library/go/core/resource/resource.go b/library/go/core/resource/resource.go index 686ea73c..01cd1bdb 100644 --- a/library/go/core/resource/resource.go +++ b/library/go/core/resource/resource.go @@ -45,7 +45,7 @@ func MustGet(key string) []byte { return r } -// Keys returns sorted keys of all registered resources inside the binary +// Keys returns sorted keys of all registered resources inside the binary. func Keys() []string { keys := make([]string, 0, len(resources)) for k := range resources { diff --git a/library/go/core/xerrors/internal/modes/stack_trace_mode.go b/library/go/core/xerrors/internal/modes/stack_trace_mode.go index 04f78ffd..64549574 100644 --- a/library/go/core/xerrors/internal/modes/stack_trace_mode.go +++ b/library/go/core/xerrors/internal/modes/stack_trace_mode.go @@ -19,9 +19,9 @@ func (m StackTraceMode) String() string { const defaultStackTraceMode = StackTraceModeFrames var ( - // Default mode + // Default mode. stackTraceMode = defaultStackTraceMode - // Known modes (used in tests) + // Known modes (used in tests). knownStackTraceModes = []StackTraceMode{ StackTraceModeFrames, StackTraceModeStacks, diff --git a/library/go/core/xerrors/mode.go b/library/go/core/xerrors/mode.go index d20b5c54..df4ca960 100644 --- a/library/go/core/xerrors/mode.go +++ b/library/go/core/xerrors/mode.go @@ -31,7 +31,7 @@ func DisableStackTraces() { modes.SetStackTraceMode(modes.StackTraceModeNothing) } -// newStackTrace returns stacktrace based on current mode and frames count +// newStackTrace returns stacktrace based on current mode and frames count. func newStackTrace(skip int, err error) *xruntime.StackTrace { skip++ m := modes.GetStackTraceMode() diff --git a/library/go/core/xerrors/sentinel.go b/library/go/core/xerrors/sentinel.go index 8ffe7b80..c47b38c8 100644 --- a/library/go/core/xerrors/sentinel.go +++ b/library/go/core/xerrors/sentinel.go @@ -10,17 +10,17 @@ import ( "github.com/doublecloud/transfer/library/go/x/xruntime" ) -// NewSentinel acts as New but does not add stack frame +// NewSentinel acts as New but does not add stack frame. func NewSentinel(text string) *Sentinel { return &Sentinel{error: errors.New(text)} } -// Sentinel error +// Sentinel error. type Sentinel struct { error } -// WithFrame adds stack frame to sentinel error (DEPRECATED) +// WithFrame adds stack frame to sentinel error (DEPRECATED). func (s *Sentinel) WithFrame() error { return &sentinelWithStackTrace{ err: s, @@ -90,7 +90,7 @@ func (e *sentinelWithStackTrace) Is(target error) bool { } // As checks if ew holds the specified error type. Checks only immediate error. -// It does NOT perform target checks as it relies on errors.As to do it +// It does NOT perform target checks as it relies on errors.As to do it. func (e *sentinelWithStackTrace) As(target interface{}) bool { return xreflect.Assign(e.err, target) } @@ -133,7 +133,7 @@ func (e *sentinelWrapper) Format(s fmt.State, v rune) { } } -// Unwrap implements Wrapper interface +// Unwrap implements Wrapper interface. func (e *sentinelWrapper) Unwrap() error { return e.wrapped } @@ -144,7 +144,7 @@ func (e *sentinelWrapper) Is(target error) bool { } // As checks if error holds the specified error type. Checks only immediate error. -// It does NOT perform target checks as it relies on errors.As to do it +// It does NOT perform target checks as it relies on errors.As to do it. func (e *sentinelWrapper) As(target interface{}) bool { return xreflect.Assign(e.err, target) } diff --git a/library/go/poolba/pool.go b/library/go/poolba/pool.go index 2ea2df69..23764e4d 100644 --- a/library/go/poolba/pool.go +++ b/library/go/poolba/pool.go @@ -8,13 +8,13 @@ import ( var errPoolClosed = errors.New("pool closed") -// ConstructorFunc will be called for any uninitialized resource upon fetch from pool +// ConstructorFunc will be called for any uninitialized resource upon fetch from pool. type ConstructorFunc[T any] func(context.Context) (T, error) -// DestructorFunc will be called for any closed resource before it will be returned to pool +// DestructorFunc will be called for any closed resource before it will be returned to pool. type DestructorFunc[T any] func(T) error -// Pool represents thread-safe generic resource pool of fixed size +// Pool represents thread-safe generic resource pool of fixed size. type Pool[T any] struct { // cancel context to be used on pool close closeCtx context.Context @@ -30,7 +30,7 @@ type Pool[T any] struct { coldResources chan *Resource[T] } -// PoolOf returns new generic resource pool of given size +// PoolOf returns new generic resource pool of given size. func PoolOf[T any](size int, opts ...PoolOpt[T]) (*Pool[T], error) { ctx, cancel := context.WithCancel(context.Background()) @@ -70,7 +70,7 @@ func PoolOf[T any](size int, opts ...PoolOpt[T]) (*Pool[T], error) { // Result resource cannot be acquired by another thread, // so it must be eventually returned back to pool via Vacay method. // This method will block code execution until resource is available, -// given context canceled or pool closed +// given context canceled or pool closed. func (p *Pool[T]) Borrow(ctx context.Context) (*Resource[T], error) { // checking hot resources first and moving forward if none available select { @@ -114,7 +114,7 @@ func (p *Pool[T]) Borrow(ctx context.Context) (*Resource[T], error) { } // Close closes pool entirely and destructs all initialized resources. -// Pool cannot be used after close +// Pool cannot be used after close. func (p *Pool[T]) Close() (err error) { p.closeFunc() @@ -134,13 +134,13 @@ func (p *Pool[T]) Close() (err error) { return } -// Resource represents generic reusable resource +// Resource represents generic reusable resource. type Resource[T any] struct { value *T pool *Pool[T] } -// Value returns actual user defined generic resource +// Value returns actual user defined generic resource. func (r *Resource[T]) Value() T { return *r.value } @@ -148,14 +148,14 @@ func (r *Resource[T]) Value() T { // Vacay returns resource back to pool. // It can be reused without initialization later by another thread. // Be aware that resource can be vacated exactly once -// to avoid data race conditions and resource leaks +// to avoid data race conditions and resource leaks. func (r *Resource[T]) Vacay() { r.pool.hotResources <- r } // Close deinitializes resource and returns it back to pool. // Be aware that resource can to closed exactly once -// to avoid data race conditions and resource leaks +// to avoid data race conditions and resource leaks. func (r *Resource[T]) Close() error { err := r.pool.destructor(*r.value) r.value = nil diff --git a/library/go/ptr/ptr.go b/library/go/ptr/ptr.go index 8308251b..33ccc5d4 100644 --- a/library/go/ptr/ptr.go +++ b/library/go/ptr/ptr.go @@ -2,70 +2,70 @@ package ptr import "time" -// Int returns pointer to provided value +// Int returns pointer to provided value. func Int(v int) *int { return &v } -// Int8 returns pointer to provided value +// Int8 returns pointer to provided value. func Int8(v int8) *int8 { return &v } -// Int16 returns pointer to provided value +// Int16 returns pointer to provided value. func Int16(v int16) *int16 { return &v } -// Int32 returns pointer to provided value +// Int32 returns pointer to provided value. func Int32(v int32) *int32 { return &v } -// Int64 returns pointer to provided value +// Int64 returns pointer to provided value. func Int64(v int64) *int64 { return &v } -// Uint returns pointer to provided value +// Uint returns pointer to provided value. func Uint(v uint) *uint { return &v } -// Uint8 returns pointer to provided value +// Uint8 returns pointer to provided value. func Uint8(v uint8) *uint8 { return &v } -// Uint16 returns pointer to provided value +// Uint16 returns pointer to provided value. func Uint16(v uint16) *uint16 { return &v } -// Uint32 returns pointer to provided value +// Uint32 returns pointer to provided value. func Uint32(v uint32) *uint32 { return &v } -// Uint64 returns pointer to provided value +// Uint64 returns pointer to provided value. func Uint64(v uint64) *uint64 { return &v } -// Float32 returns pointer to provided value +// Float32 returns pointer to provided value. func Float32(v float32) *float32 { return &v } -// Float64 returns pointer to provided value +// Float64 returns pointer to provided value. func Float64(v float64) *float64 { return &v } -// Bool returns pointer to provided value +// Bool returns pointer to provided value. func Bool(v bool) *bool { return &v } -// String returns pointer to provided value +// String returns pointer to provided value. func String(v string) *string { return &v } -// Byte returns pointer to provided value +// Byte returns pointer to provided value. func Byte(v byte) *byte { return &v } -// Rune returns pointer to provided value +// Rune returns pointer to provided value. func Rune(v rune) *rune { return &v } -// Complex64 returns pointer to provided value +// Complex64 returns pointer to provided value. func Complex64(v complex64) *complex64 { return &v } -// Complex128 returns pointer to provided value +// Complex128 returns pointer to provided value. func Complex128(v complex128) *complex128 { return &v } -// Time returns pointer to provided value +// Time returns pointer to provided value. func Time(v time.Time) *time.Time { return &v } -// Duration returns pointer to provided value +// Duration returns pointer to provided value. func Duration(v time.Duration) *time.Duration { return &v } -// T returns pointer to provided value +// T returns pointer to provided value. func T[T any](v T) *T { return &v } -// TOrNil returns pointer to provided value or nil if value is empty +// TOrNil returns pointer to provided value or nil if value is empty. func TOrNil[T comparable](v T) *T { if isZero(v) { return nil @@ -74,7 +74,7 @@ func TOrNil[T comparable](v T) *T { return &v } -// From returns value from pointer +// From returns value from pointer. func From[T any](v *T) T { if v == nil { return *new(T) @@ -83,7 +83,7 @@ func From[T any](v *T) T { return *v } -// isZero checks if provided value is empty value for the T +// isZero checks if provided value is empty value for the T. func isZero[T comparable](v T) bool { var t T diff --git a/library/go/slices/filter.go b/library/go/slices/filter.go index 8b383bfc..2f523cda 100644 --- a/library/go/slices/filter.go +++ b/library/go/slices/filter.go @@ -5,7 +5,7 @@ import ( ) // Filter reduces slice values using given function. -// It operates with a copy of given slice +// It operates with a copy of given slice. func Filter[S ~[]T, T any](s S, fn func(T) bool) S { if len(s) == 0 { return s diff --git a/library/go/slices/group_by.go b/library/go/slices/group_by.go index fb61a293..44f14314 100644 --- a/library/go/slices/group_by.go +++ b/library/go/slices/group_by.go @@ -40,7 +40,7 @@ func GroupByUniqueKey[S ~[]T, T any, K comparable](s S, keyGetter func(T) K) (ma return res, nil } -// IndexedEntity stores an entity of original slice with its initial index in that slice +// IndexedEntity stores an entity of original slice with its initial index in that slice. type IndexedEntity[T any] struct { Value T Index int diff --git a/library/go/slices/join.go b/library/go/slices/join.go index 7b72db5e..8b187a9c 100644 --- a/library/go/slices/join.go +++ b/library/go/slices/join.go @@ -5,7 +5,7 @@ import ( "strings" ) -// Join joins slice of any types +// Join joins slice of any types. func Join(s interface{}, glue string) string { if t, ok := s.([]string); ok { return strings.Join(t, glue) diff --git a/library/go/slices/map.go b/library/go/slices/map.go index b2e057aa..9695334d 100644 --- a/library/go/slices/map.go +++ b/library/go/slices/map.go @@ -1,6 +1,6 @@ package slices -// MapP applies given function to every pointer to element of slice +// MapP applies given function to every pointer to element of slice. func MapP[S ~[]T, T, M any](s S, fn func(*T) M) []M { if s == nil { return []M(nil) @@ -15,7 +15,7 @@ func MapP[S ~[]T, T, M any](s S, fn func(*T) M) []M { return res } -// Map applies given function to every value of slice +// Map applies given function to every value of slice. func Map[S ~[]T, T, M any](s S, fn func(T) M) []M { if s == nil { return []M(nil) @@ -30,7 +30,7 @@ func Map[S ~[]T, T, M any](s S, fn func(T) M) []M { return res } -// MapE applies given function to every value of slice and return slice or first error +// MapE applies given function to every value of slice and return slice or first error. func MapE[S ~[]T, T, M any](s S, fn func(T) (M, error)) ([]M, error) { if s == nil { return []M(nil), nil diff --git a/library/go/slices/sort.go b/library/go/slices/sort.go index 78f27c8c..dbbcca85 100644 --- a/library/go/slices/sort.go +++ b/library/go/slices/sort.go @@ -5,7 +5,7 @@ import ( "slices" ) -// Sort is like slices.Sort but returns sorted copy of given slice +// Sort is like slices.Sort but returns sorted copy of given slice. func Sort[T cmp.Ordered](s []T) []T { s2 := make([]T, len(s)) copy(s2, s) @@ -13,22 +13,22 @@ func Sort[T cmp.Ordered](s []T) []T { return s2 } -// SortBy sorts a slice in place using given sortKey +// SortBy sorts a slice in place using given sortKey. func SortBy[S ~[]E, E any, U cmp.Ordered](x S, sortKey func(E) U) { slices.SortFunc(x, func(a, b E) int { return cmp.Compare(sortKey(a), sortKey(b)) }) } -// SortDescBy sorts a slice in place using given sortKey in descending order +// SortDescBy sorts a slice in place using given sortKey in descending order. func SortDescBy[S ~[]E, E any, U cmp.Ordered](x S, sortKey func(E) U) { slices.SortFunc(x, func(a, b E) int { return cmp.Compare(sortKey(b), sortKey(a)) }) } -// SortStableBy sorts a slice in place using given sortKey, uses stable sorting +// SortStableBy sorts a slice in place using given sortKey, uses stable sorting. func SortStableBy[S ~[]E, E any, U cmp.Ordered](x S, sortKey func(E) U) { slices.SortStableFunc(x, func(a, b E) int { return cmp.Compare(sortKey(a), sortKey(b)) }) } -// SortDescStableBy sorts a slice in place using given sortKey in descending order, uses stable sorting +// SortDescStableBy sorts a slice in place using given sortKey in descending order, uses stable sorting. func SortDescStableBy[S ~[]E, E any, U cmp.Ordered](x S, sortKey func(E) U) { slices.SortStableFunc(x, func(a, b E) int { return cmp.Compare(sortKey(b), sortKey(a)) }) } diff --git a/library/go/test/canon/canon.go b/library/go/test/canon/canon.go index 250230bb..191e359b 100644 --- a/library/go/test/canon/canon.go +++ b/library/go/test/canon/canon.go @@ -11,7 +11,7 @@ import ( "github.com/doublecloud/transfer/library/go/test/yatest" ) -var isRunningUnderGotest bool +var isRunningUnderGoTest bool type canonObject struct { TestName string `json:"test_name"` @@ -76,7 +76,7 @@ func saveObject(t *testing.T, item interface{}) { } func SaveJSON(t *testing.T, item interface{}) { - if !isRunningUnderGotest { + if !isRunningUnderGoTest { saveObject(t, item) } } @@ -96,7 +96,7 @@ func getCanonDestination(t *testing.T, canonFilename string) string { } func SaveFile(t *testing.T, canonFilename string, opts ...Option) { - if isRunningUnderGotest { + if isRunningUnderGoTest { return } diff --git a/library/go/test/canon/dctest.go b/library/go/test/canon/dctest.go index dfd4c126..6e38e2b1 100644 --- a/library/go/test/canon/dctest.go +++ b/library/go/test/canon/dctest.go @@ -4,5 +4,5 @@ package canon func init() { - isRunningUnderGotest = true + isRunningUnderGoTest = true } diff --git a/library/go/test/canon/gotest.go b/library/go/test/canon/gotest.go index dfd4c126..6e38e2b1 100644 --- a/library/go/test/canon/gotest.go +++ b/library/go/test/canon/gotest.go @@ -4,5 +4,5 @@ package canon func init() { - isRunningUnderGotest = true + isRunningUnderGoTest = true } diff --git a/library/go/test/yatest/dctest.go b/library/go/test/yatest/dctest.go index 102bae59..227e5e35 100644 --- a/library/go/test/yatest/dctest.go +++ b/library/go/test/yatest/dctest.go @@ -4,7 +4,6 @@ package yatest func doInit() { - isRunningUnderGoTest = true context.Initialized = true context.Runtime.SourceRoot = "" } diff --git a/library/go/test/yatest/env.go b/library/go/test/yatest/env.go index fffd259d..f701318b 100644 --- a/library/go/test/yatest/env.go +++ b/library/go/test/yatest/env.go @@ -2,8 +2,6 @@ package yatest import ( - "bufio" - "encoding/json" "fmt" "os" "path" @@ -46,9 +44,8 @@ type TestContext struct { } var ( - context TestContext - isRunningUnderGoTest bool - initOnce sync.Once + context TestContext + initOnce sync.Once ) func lazyInit() { @@ -61,49 +58,6 @@ func verifyContext() { } } -func initTestContext() { - data, err := os.ReadFile(getenv("YA_TEST_CONTEXT_FILE")) - if err != nil { - panic(err) - } - - err = json.Unmarshal([]byte(data), &context) - if err != nil { - panic(err) - } - setupEnv() - context.Initialized = true -} - -func setupEnv() { - file, err := os.Open(context.Internal.EnvFile) - if err != nil { - return - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - var objmap map[string]json.RawMessage - var val string - err := json.Unmarshal([]byte(line), &objmap) - if err != nil { - panic(err) - } - for k, v := range objmap { - err := json.Unmarshal(v, &val) - if err != nil { - panic(err) - } - err = os.Setenv(k, val) - if err != nil { - panic(err) - } - } - } -} - func HasYaTestContext() bool { lazyInit() return context.Initialized @@ -204,7 +158,7 @@ func OutputPath(dataPath string) string { return filepath.Join(context.Runtime.OutputPath, dataPath) } -// RAMDrivePath returns absolute path to the ramdrive directory +// RAMDrivePath returns absolute path to the ramdrive directory. func RAMDrivePath(dataPath string) string { lazyInit() if path.IsAbs(dataPath) { @@ -215,7 +169,7 @@ func RAMDrivePath(dataPath string) string { return filepath.Join(context.Runtime.RAMDrivePath, dataPath) } -// YtHDDPath returns absolute path to the directory mounted to ext4 fs in YT +// YtHDDPath returns absolute path to the directory mounted to ext4 fs in YT. func YtHDDPath(dataPath string) string { lazyInit() if path.IsAbs(dataPath) { @@ -226,7 +180,7 @@ func YtHDDPath(dataPath string) string { return filepath.Join(context.Runtime.YtHDDPath, dataPath) } -// OutputRAMDrivePath returns absolute path to the ramdrive output directory +// OutputRAMDrivePath returns absolute path to the ramdrive output directory. func OutputRAMDrivePath(dataPath string) string { lazyInit() if path.IsAbs(dataPath) { @@ -237,7 +191,7 @@ func OutputRAMDrivePath(dataPath string) string { return filepath.Join(context.Runtime.TestOutputRAMDrivePath, dataPath) } -// HasRAMDrive returns true if ramdrive is enabled in tests +// HasRAMDrive returns true if ramdrive is enabled in tests. func HasRAMDrive() bool { lazyInit() verifyContext() @@ -257,21 +211,21 @@ func BinaryPath(dataPath string) (string, error) { } } -// ProjectPath returns arcadia-relative path to the test project +// ProjectPath returns arcadia-relative path to the test project. func ProjectPath() string { lazyInit() verifyContext() return context.Runtime.ProjectPath } -// BuildType returns build type that was used to compile the test +// BuildType returns build type that was used to compile the test. func BuildType() string { lazyInit() verifyContext() return context.Build.BuildType } -// BuildFlag returns the value of the requested build flag +// BuildFlag returns the value of the requested build flag. func BuildFlag(name string) (string, bool) { lazyInit() verifyContext() @@ -279,7 +233,7 @@ func BuildFlag(name string) (string, bool) { return val, ok } -// TestParam returns the value of the requested test parameter +// TestParam returns the value of the requested test parameter. func TestParam(name string) (string, bool) { lazyInit() verifyContext() @@ -287,7 +241,7 @@ func TestParam(name string) (string, bool) { return val, ok } -// Sanitizer returns sanitizer name that was used to compile the test +// Sanitizer returns sanitizer name that was used to compile the test. func Sanitizer() string { lazyInit() verifyContext() diff --git a/library/go/x/xruntime/stacktrace.go b/library/go/x/xruntime/stacktrace.go index 5c5e6611..701dbf10 100644 --- a/library/go/x/xruntime/stacktrace.go +++ b/library/go/x/xruntime/stacktrace.go @@ -41,7 +41,7 @@ func NewFrame(skip int) *StackTrace { } func (st *StackTrace) Frames() []runtime.Frame { - frames := runtime.CallersFrames(st.frames[:]) + frames := runtime.CallersFrames(st.frames) if !st.full { if _, ok := frames.Next(); !ok { return nil diff --git a/library/go/yandex/cloud/filter/filters.go b/library/go/yandex/cloud/filter/filters.go index 502326b6..8abfe146 100644 --- a/library/go/yandex/cloud/filter/filters.go +++ b/library/go/yandex/cloud/filter/filters.go @@ -262,10 +262,8 @@ func validateTerm(f grammar.Term, op OperatorType) error { if op != In && op != NotIn { return newSyntaxError(f.Pos, "list values require [ NOT ] IN operator") } - } else { - if op == In || op == NotIn { - return newSyntaxErrorf(f.Pos, "%s operator expect list value, got %s", f.Operator, f.Value.TypeOf()) - } + } else if op == In || op == NotIn { + return newSyntaxErrorf(f.Pos, "%s operator expect list value, got %s", f.Operator, f.Value.TypeOf()) } if f.Value.Null != nil && op != Equals && op != NotEquals { return newSyntaxErrorf(f.Pos, "NULL expects \"=\" or \"!=\" operator, got %s", f.Operator) @@ -289,7 +287,7 @@ func fromParsed(f grammar.Term) (Term, error) { }, nil } -// Parse filter string and returns list of Term +// Parse filter string and returns list of Term. func Parse(filtersString string) ([]Term, error) { parsed, err := grammar.Parse(filtersString) if err != nil { diff --git a/library/go/yandex/cloud/filter/grammar/grammar.go b/library/go/yandex/cloud/filter/grammar/grammar.go index 0d5e9f6d..68b60c52 100644 --- a/library/go/yandex/cloud/filter/grammar/grammar.go +++ b/library/go/yandex/cloud/filter/grammar/grammar.go @@ -167,7 +167,7 @@ func (dt *DateTime) Capture(values []string) error { return nil } -// delayedParse parse Time from captured string +// delayedParse parse Time from captured string. func (dt *DateTime) delayedParse() error { // This also works for parsing time without a time zone, // since `time.Parse` uses UTC for such a case @@ -192,7 +192,7 @@ func (f *Float) Capture(values []string) error { return nil } -// delayedParse parse string into number +// delayedParse parse string into number. func (f *Float) delayedParse() error { fv, err := strconv.ParseFloat(f.value, 64) if err != nil { @@ -218,7 +218,7 @@ func (i *Int) Capture(values []string) error { return nil } -// delayedParse parse string into number +// delayedParse parse string into number. func (i *Int) delayedParse() error { iv, err := strconv.ParseInt(i.value, 10, 64) if err != nil { @@ -244,7 +244,7 @@ func (i *Null) Capture(values []string) error { return nil } -// delayedParse parse string into value +// delayedParse parse string into value. func (i *Null) delayedParse() error { if i.value == "NULL" || i.value == "NIL" { i.V = true diff --git a/library/go/yatool/root.go b/library/go/yatool/root.go index 79630c8f..b0bcebe7 100644 --- a/library/go/yatool/root.go +++ b/library/go/yatool/root.go @@ -53,7 +53,7 @@ func FindArcadiaRoot(arcPath string) (string, error) { } } -// ArcadiaRoot returns the current Arcadia root +// ArcadiaRoot returns the current Arcadia root. func ArcadiaRoot() (string, error) { arcadiaRootOnce.Do(func() { arcadiaRoot, arcadiaRootErr = FindArcadiaRoot(".") diff --git a/library/go/yatool/ya.go b/library/go/yatool/ya.go index f92e6f45..a0da3079 100644 --- a/library/go/yatool/ya.go +++ b/library/go/yatool/ya.go @@ -17,7 +17,7 @@ const ( windowsOS = "windows" ) -// FindYa returns path to ya for arcadia root at the target path +// FindYa returns path to ya for arcadia root at the target path. func FindYa(path string) (string, error) { arcadiaRoot, err := FindArcadiaRoot(path) if err != nil { @@ -27,7 +27,7 @@ func FindYa(path string) (string, error) { return yaForPath(arcadiaRoot) } -// Ya returns "ya" path for current Arcadia root +// Ya returns "ya" path for current Arcadia root. func Ya() (string, error) { yaOnce.Do(func() { var arcadiaRoot string diff --git a/pkg/abstract/async_sink.go b/pkg/abstract/async_sink.go index 5148337a..7f16b73a 100644 --- a/pkg/abstract/async_sink.go +++ b/pkg/abstract/async_sink.go @@ -19,7 +19,7 @@ type AsyncSink interface { // AsyncPushConcurrencyErr indicates a Push has been called on an already closed AsyncSink. This must not happen and means there are concurrency issues in the implementation of a source. var AsyncPushConcurrencyErr = xerrors.NewSentinel("AsyncPush is called after Close") -// PusherFromAsyncSink wraps the given sink into a (synchronous) pusher interface +// PusherFromAsyncSink wraps the given sink into a (synchronous) pusher interface. func PusherFromAsyncSink(asink AsyncSink) Pusher { return func(items []ChangeItem) error { return <-asink.AsyncPush(items) diff --git a/pkg/abstract/change_item_builders.go b/pkg/abstract/change_item_builders.go index 21a416d8..cfb21e72 100644 --- a/pkg/abstract/change_item_builders.go +++ b/pkg/abstract/change_item_builders.go @@ -53,7 +53,7 @@ func MakeTxDone(txSequence uint32, lsn uint64, execTS time.Time, lastPushedGTID, ColumnNames: []string{"query"}, ColumnValues: []interface{}{fmt.Sprintf("-- transaction %v done", lastPushedGTID)}, TableSchema: NewTableSchema([]ColSchema{{}}), - OldKeys: *new(OldKeysType), + OldKeys: OldKeysType{}, TxID: gtidStr, Query: "", Size: RawEventSize(0), diff --git a/pkg/abstract/changeitem/change_item.go b/pkg/abstract/changeitem/change_item.go index 27ca1930..64ae52d7 100644 --- a/pkg/abstract/changeitem/change_item.go +++ b/pkg/abstract/changeitem/change_item.go @@ -20,8 +20,6 @@ var ( _ json.Unmarshaler = (*ChangeItem)(nil) _ yson.Marshaler = (*ChangeItem)(nil) _ yson.Unmarshaler = (*ChangeItem)(nil) - - ysonEncoderOptions *yson.EncoderOptions = nil // for tests ) type ChangeItem struct { @@ -91,10 +89,12 @@ const ( partitionCol = "_partition" ) -var RowEventKinds = set.New(InsertKind, UpdateKind, DeleteKind, MongoUpdateDocumentKind) -var SystemKinds = set.New( - InitTableLoad, InitShardedTableLoad, DoneTableLoad, DoneShardedTableLoad, DropTableKind, TruncateTableKind, - SynchronizeKind, +var ( + RowEventKinds = set.New(InsertKind, UpdateKind, DeleteKind, MongoUpdateDocumentKind) + SystemKinds = set.New( + InitTableLoad, InitShardedTableLoad, DoneTableLoad, DoneShardedTableLoad, DropTableKind, TruncateTableKind, + SynchronizeKind, + ) ) func (c *ChangeItem) Offset() (offset uint64, found bool) { @@ -204,10 +204,10 @@ func (c *ChangeItem) MakeMapKeys() map[string]bool { return keyCols } -// ColumnName is an explicitly declared column name +// ColumnName is an explicitly declared column name. type ColumnName string -// FastTableSchema is a mapping from column name to its schema +// FastTableSchema is a mapping from column name to its schema. type FastTableSchema map[ColumnName]ColSchema // MakeFastTableSchema produces a fast table schema from an array of schemas of each column. Column names are taken from these schemas. @@ -220,7 +220,7 @@ func MakeFastTableSchema(slowTableSchema []ColSchema) FastTableSchema { } // KeysChanged - checks if update changes primary keys -// NOTE: this method is quite inefficient +// NOTE: this method is quite inefficient. func (c *ChangeItem) KeysChanged() bool { if c.Kind != UpdateKind { return false @@ -297,7 +297,7 @@ func (c *ChangeItem) IsToasted() bool { return false } -// OldOrCurrentKeys returns a string representing the values of the columns from the given set of columns, extracted from OldKeys or ColumnValues, if OldKeys are absent +// OldOrCurrentKeys returns a string representing the values of the columns from the given set of columns, extracted from OldKeys or ColumnValues, if OldKeys are absent. func (c *ChangeItem) OldOrCurrentKeysString(keyColumns map[string]bool) string { if (c.Kind == UpdateKind || c.Kind == DeleteKind) && len(c.OldKeys.KeyValues) > 0 { keys := make(map[string]interface{}) @@ -353,7 +353,7 @@ func ContainsNonRowItem(s []ChangeItem) bool { return false } -// FindItemOfKind returns an item in the slice whose kind is among the given ones, or nil if there is no such item in it +// FindItemOfKind returns an item in the slice whose kind is among the given ones, or nil if there is no such item in it. func FindItemOfKind(s []ChangeItem, kinds ...Kind) *ChangeItem { wanted := make(map[Kind]bool) for _, k := range kinds { @@ -368,7 +368,7 @@ func FindItemOfKind(s []ChangeItem, kinds ...Kind) *ChangeItem { } // IsMirror -// mirror - it's special format with hardcoded schema - used for "mirroring" (queue->queue) - transfer messages between queues without changes +// mirror - it's special format with hardcoded schema - used for "mirroring" (queue->queue) - transfer messages between queues without changes. func (c *ChangeItem) IsMirror() bool { if len(c.ColumnNames) != len(RawDataColumns) { return false @@ -437,9 +437,9 @@ func (c ChangeItem) MarshalYSON() ([]byte, error) { writer.MapKeyString("id") writer.Uint64(uint64(c.ID)) writer.MapKeyString("nextlsn") - writer.Uint64(uint64(c.LSN)) + writer.Uint64(c.LSN) writer.MapKeyString("commitTime") - writer.Uint64(uint64(c.CommitTime)) + writer.Uint64(c.CommitTime) writer.MapKeyString("txPosition") writer.Int64(int64(c.Counter)) writer.MapKeyString("kind") @@ -663,7 +663,7 @@ func (c *ChangeItem) SetTableSchema(tableSchema *TableSchema) { } // RemoveColumns mutate change item to skip some columns from it. -// it remove it from column names and column values +// it remove it from column names and column values. func (c *ChangeItem) RemoveColumns(cols ...string) { toDelete := map[string]struct{}{} for _, col := range cols { diff --git a/pkg/abstract/changeitem/kind.go b/pkg/abstract/changeitem/kind.go index ba228268..0cab0515 100644 --- a/pkg/abstract/changeitem/kind.go +++ b/pkg/abstract/changeitem/kind.go @@ -10,10 +10,10 @@ const ( // For each table being uploaded the following ChangeItems should be emited in order // InitShardedTableLoad -> N * (InitTableLoad -> row events -> DoneTableLoad) -> DoneShardedTableLoad // where N is number of table parts if table is sharded (thus N = 1 if table is not sharded) - // Each ChangeItem of those kinds should contain TableSchema for the corresponding table + // Each ChangeItem of those kinds should contain TableSchema for the corresponding table. // InitShardedTableLoad is table header. - // All Init/DoneTableLoad and data items for the same table should be sent strictly after this kind of item + // All Init/DoneTableLoad and data items for the same table should be sent strictly after this kind of item. InitShardedTableLoad = Kind("init_sharded_table_load") // InitTableLoad is table part header. It should be sent for each table part before uploading any data for that part. // If the table is not sharded, it is considered to have a single part so InitTableLoad should be sent diff --git a/pkg/abstract/changeitem/table_id.go b/pkg/abstract/changeitem/table_id.go index 895232e7..8e08da2e 100644 --- a/pkg/abstract/changeitem/table_id.go +++ b/pkg/abstract/changeitem/table_id.go @@ -48,7 +48,7 @@ func (t TableID) Equals(other TableID) bool { // Includes returns true if the given ID identifies a subset of tables identified by the current ID. // -// In other words, it is a result of an operation "is a superset of" +// In other words, it is a result of an operation "is a superset of". func (t TableID) Includes(sub TableID) bool { if t.Equals(sub) { return true diff --git a/pkg/abstract/changeitem/utils.go b/pkg/abstract/changeitem/utils.go index 81431114..410bf5f2 100644 --- a/pkg/abstract/changeitem/utils.go +++ b/pkg/abstract/changeitem/utils.go @@ -5,7 +5,7 @@ import ( "fmt" ) -// MakeMapColNameToIndex returns a mapping of a column name to an index in the given slice +// MakeMapColNameToIndex returns a mapping of a column name to an index in the given slice. func MakeMapColNameToIndex(in []ColSchema) map[string]int { result := make(map[string]int) for i, column := range in { @@ -24,7 +24,7 @@ func KeyNames(in []ColSchema) []string { return result } -// Sniff dat data +// Sniff dat data. func Sniff(input []ChangeItem) string { tables := make(map[TableID]ChangeItem) tablesCntr := make(map[TableID]int) diff --git a/pkg/abstract/coordinator/editor.go b/pkg/abstract/coordinator/editor.go index d8ac3c92..f3b6205f 100644 --- a/pkg/abstract/coordinator/editor.go +++ b/pkg/abstract/coordinator/editor.go @@ -4,7 +4,7 @@ import "github.com/doublecloud/transfer/pkg/abstract/model" // Editor is obsolete interface to update entities on coordinator // -// Deprecated: do not rely on it anymore +// Deprecated: do not rely on it anymore. type Editor interface { // GetEndpointTransfers get all *other* linked transfer to either source or target of provider *transferID* GetEndpointTransfers(transferID string, isSource bool) ([]*model.Transfer, error) diff --git a/pkg/abstract/coordinator/operation.go b/pkg/abstract/coordinator/operation.go index 2c97fa86..195dbb91 100644 --- a/pkg/abstract/coordinator/operation.go +++ b/pkg/abstract/coordinator/operation.go @@ -36,7 +36,7 @@ type OperationStatus interface { // Sharding coordinate multiple worker for transfer operations // transfer utilize MPP aproach, when we have a main (or leader) worker // main worker coordinate secondary workers via single coordinator (API or remote storage) -// main worker is usually first instance of multi-node transfer operation worker +// main worker is usually first instance of multi-node transfer operation worker. type Sharding interface { // GetOperationProgress called by *main* worker to track progress over secondary workers GetOperationProgress(operationID string) (*model.AggregatedProgress, error) @@ -68,7 +68,7 @@ type Sharding interface { UpdateOperationTablesParts(operationID string, tables []*model.OperationTablePart) error } -// Progressable is opt-in interface to show total progress over upload once completed +// Progressable is opt-in interface to show total progress over upload once completed. type Progressable interface { Progress() []*model.OperationTablePart } diff --git a/pkg/abstract/coordinator/transfer.go b/pkg/abstract/coordinator/transfer.go index 45678440..e6db9067 100644 --- a/pkg/abstract/coordinator/transfer.go +++ b/pkg/abstract/coordinator/transfer.go @@ -11,7 +11,7 @@ type TransferHeartbeat struct { LastError string } -// StatusMessageProvider set time-based status for particular transfer +// StatusMessageProvider set time-based status for particular transfer. type StatusMessageProvider interface { // OpenStatusMessage open new line of error status message // for example: add timeout error for SOURCE db category @@ -24,7 +24,7 @@ type StatusMessageProvider interface { } // TransferStatus main coordinator interface used by transfer -// just to start / stop transfer and move it via transfer workflow +// just to start / stop transfer and move it via transfer workflow. type TransferStatus interface { // SetStatus move transfer to certain status SetStatus(transferID string, status model.TransferStatus) error @@ -34,7 +34,7 @@ type TransferStatus interface { TransferHealth(ctx context.Context, transferID string, health *TransferHeartbeat) error } -// TransferState is to manage transfer state, simple K-V structure +// TransferState is to manage transfer state, simple K-V structure. type TransferState interface { // GetTransferState return known transfer state GetTransferState(id string) (map[string]*TransferStateData, error) diff --git a/pkg/abstract/coordinator/transfer_state.go b/pkg/abstract/coordinator/transfer_state.go index 0117867f..1bc7a66f 100644 --- a/pkg/abstract/coordinator/transfer_state.go +++ b/pkg/abstract/coordinator/transfer_state.go @@ -35,7 +35,7 @@ type YtStaticPartState struct { } // TransferStateData contain transfer state, shared across retries / restarts -// can contain any generic information about transfer progress +// can contain any generic information about transfer progress. type TransferStateData struct { // Generic is recommended way, you can put anything json serializable here Generic any diff --git a/pkg/abstract/dterrors/error.go b/pkg/abstract/dterrors/error.go index 34754b50..c4370684 100644 --- a/pkg/abstract/dterrors/error.go +++ b/pkg/abstract/dterrors/error.go @@ -4,7 +4,7 @@ import ( "github.com/doublecloud/transfer/library/go/core/xerrors" ) -// FatalError denotes an error which must stop the transfer completely, forbidding to restart it automatically +// FatalError denotes an error which must stop the transfer completely, forbidding to restart it automatically. type FatalError interface { error IsFatal() @@ -15,7 +15,7 @@ func IsFatal(err error) bool { return xerrors.As(err, &target) } -// baseFatalError is a default implementation of FatalError interface +// baseFatalError is a default implementation of FatalError interface. type baseFatalError struct{ error } func (b baseFatalError) IsFatal() {} diff --git a/pkg/abstract/dterrors/error_test.go b/pkg/abstract/dterrors/error_test.go index 8d96bf2d..56e8c616 100644 --- a/pkg/abstract/dterrors/error_test.go +++ b/pkg/abstract/dterrors/error_test.go @@ -27,7 +27,7 @@ func TestCheckForTableUploadError(t *testing.T) { }) } -// replacement showcase +// replacement showcase. type ValueError int func (v ValueError) Error() string { return "value error" } diff --git a/pkg/abstract/dterrors/errors_test_helper.go b/pkg/abstract/dterrors/errors_test_helper.go index cf5295e4..dd5637f2 100644 --- a/pkg/abstract/dterrors/errors_test_helper.go +++ b/pkg/abstract/dterrors/errors_test_helper.go @@ -27,7 +27,7 @@ func CheckErrorWrapping(t *testing.T, comment string, } // CheckOpaqueErrorWrapping is the same as CheckErrorWrapping, but for opaque errors -// can be used for checking new errors as well +// can be used for checking new errors as well. func CheckOpaqueErrorWrapping(t *testing.T, comment string, predicate func(error) bool, newError func(error) error) { diff --git a/pkg/abstract/filter.go b/pkg/abstract/filter.go index 8e626290..95578516 100644 --- a/pkg/abstract/filter.go +++ b/pkg/abstract/filter.go @@ -17,10 +17,10 @@ type WhereStatement string const NoFilter WhereStatement = WhereStatement("") -// bracketType sorts characters into either opening, closing, or not a bracket +// bracketType sorts characters into either opening, closing, or not a bracket. type bracketType int -// TestVersion is the version of unit tests that this will pass +// TestVersion is the version of unit tests that this will pass. const TestVersion = 2 const ( @@ -109,7 +109,7 @@ func FiltersIntersection(a WhereStatement, b WhereStatement) WhereStatement { if b == NoFilter { return a } - return WhereStatement("(" + a + ") AND (" + b + ")") + return "(" + a + ") AND (" + b + ")" } func NotStatement(a WhereStatement) WhereStatement { diff --git a/pkg/abstract/homo_valuer.go b/pkg/abstract/homo_valuer.go index 14490c60..7625848b 100644 --- a/pkg/abstract/homo_valuer.go +++ b/pkg/abstract/homo_valuer.go @@ -1,6 +1,6 @@ package abstract -// HomoValuer is the same as sql/driver.Valuer, but for homogenous values +// HomoValuer is the same as sql/driver.Valuer, but for homogenous values. type HomoValuer interface { HomoValue() any } diff --git a/pkg/abstract/local_runtime.go b/pkg/abstract/local_runtime.go index 51ab9119..801631b4 100644 --- a/pkg/abstract/local_runtime.go +++ b/pkg/abstract/local_runtime.go @@ -1,18 +1,15 @@ package abstract type LocalRuntime struct { - Host string - CurrentJob int - ShardingUpload ShardUploadParams + Host string `json:"host"` + CurrentJob int `json:"current_job"` + ShardingUpload ShardUploadParams `json:"sharding_upload"` } func (*LocalRuntime) Type() RuntimeType { return LocalRuntimeType } -func (l *LocalRuntime) isRuntime() { -} - func (l *LocalRuntime) NeedRestart(runtime Runtime) bool { return false } @@ -24,7 +21,6 @@ func (l *LocalRuntime) Validate() error { return nil } -func (l *LocalRuntime) isShardingEnabled() {} func (l *LocalRuntime) WorkersNum() int { return l.ShardingUpload.JobCount } func (l *LocalRuntime) ThreadsNumPerWorker() int { return l.ShardingUpload.ProcessCount } func (l *LocalRuntime) CurrentJobIndex() int { return l.CurrentJob } diff --git a/pkg/abstract/middleware.go b/pkg/abstract/middleware.go index 2efc50b7..d24c7034 100644 --- a/pkg/abstract/middleware.go +++ b/pkg/abstract/middleware.go @@ -8,7 +8,7 @@ type Asyncer func(sinker Sinker) AsyncSink // ComposeAsyncMiddleware builds a pipeline of AsyncMiddlewares. // Arguments order should be the same as the desired data flow. -// ComposeAsyncMiddleware(A, B, C)(sink) call is equivalent to A(B(C(sink))) +// ComposeAsyncMiddleware(A, B, C)(sink) call is equivalent to A(B(C(sink))). func ComposeAsyncMiddleware(mw ...AsyncMiddleware) AsyncMiddleware { return func(sink AsyncSink) AsyncSink { for i := len(mw) - 1; i >= 0; i-- { diff --git a/pkg/abstract/model/endpoint.go b/pkg/abstract/model/endpoint.go index 8ef72ca0..e25fc41c 100644 --- a/pkg/abstract/model/endpoint.go +++ b/pkg/abstract/model/endpoint.go @@ -70,19 +70,19 @@ type WithConnectionID interface { } // Abstract2Source if implemented we must try to create abstract2 source for them -// to be deleted in favor of `MakeDataProvider` method +// to be deleted in favor of `MakeDataProvider` method. type Abstract2Source interface { IsAbstract2(Destination) bool } -// IncrementalSource mark as enable incremental snapshot to transfer +// IncrementalSource mark as enable incremental snapshot to transfer. type IncrementalSource interface { IsIncremental() // SupportsStartCursorValue should return true if incremental source may use custom initial cursor value SupportsStartCursorValue() bool } -// TransitionalEndpoint Mark endpoint as type as transitional, so it could be opt-out-ed on snapshot +// TransitionalEndpoint Mark endpoint as type as transitional, so it could be opt-out-ed on snapshot. type TransitionalEndpoint interface { TransitionalWith(right TransitionalEndpoint) bool } @@ -107,7 +107,7 @@ func IsAppendOnlySource(src Source) bool { } } -// DefaultMirrorSource marks source as compatible with default mirror protocol (kafka/yds/eventhub) +// DefaultMirrorSource marks source as compatible with default mirror protocol (kafka/yds/eventhub). type DefaultMirrorSource interface { IsDefaultMirror() bool } @@ -120,7 +120,7 @@ func IsDefaultMirrorSource(src Source) bool { } } -// Parseable provider unified access to parser config +// Parseable provider unified access to parser config. type Parseable interface { Parser() map[string]interface{} } @@ -130,29 +130,29 @@ func IsParseable(src Source) bool { return ok } -// Serializable provider unified access to serializer config +// Serializable provider unified access to serializer config. type Serializable interface { Serializer() (SerializationFormat, bool) } // Runtimeable will force specific runtime for endpoint -// see logfeller for example +// see logfeller for example. type Runtimeable interface { Runtime() abstract.Runtime } // Dashboardeable will force specific dashboard link for endpoint pair -// see logfeller for example +// see logfeller for example. type Dashboardeable interface { DashboardLink(linkedEndpoint EndpointParams) string } -// SourceCompatibility for destination to check is it compatible with transfer source +// SourceCompatibility for destination to check is it compatible with transfer source. type SourceCompatibility interface { Compatible(src Source, transferType abstract.TransferType) error } -// DestinationCompatibility for source to check is it compatible with transfer destination +// DestinationCompatibility for source to check is it compatible with transfer destination. type DestinationCompatibility interface { Compatible(dst Destination) error } @@ -166,7 +166,7 @@ type AsyncPartSource interface { IsAsyncShardPartsSource() } -// UnderlayOnlyEndpoint marks endpoint as available only via our service underlay network +// UnderlayOnlyEndpoint marks endpoint as available only via our service underlay network. type UnderlayOnlyEndpoint interface { IsUnderlayOnlyEndpoint() } @@ -178,17 +178,17 @@ type HackableTarget interface { PostSnapshotHacks() } -// LegacyFillDependentFields for cp-dp backward compatibility, some fields were calculated on backend side before, so we preserve it +// LegacyFillDependentFields for cp-dp backward compatibility, some fields were calculated on backend side before, so we preserve it. type LegacyFillDependentFields interface { FillDependentFields(transfer *Transfer) } -// HostResolver returns a list of hosts to which network availability is required +// HostResolver returns a list of hosts to which network availability is required. type HostResolver interface { HostsNames() ([]string, error) } -// For default every destination is support sharding tables, but this interface can be used to override this behavior +// For default every destination is support sharding tables, but this interface can be used to override this behavior. type ShardeableDestination interface { SupportSharding() bool } diff --git a/pkg/abstract/model/endpoint_registry.go b/pkg/abstract/model/endpoint_registry.go index f82eed09..736eff04 100644 --- a/pkg/abstract/model/endpoint_registry.go +++ b/pkg/abstract/model/endpoint_registry.go @@ -12,13 +12,13 @@ var ( ) // RegisterSource will add new source factory for specific provider type -// this should be placed inside provider `init() func` +// this should be placed inside provider `init() func`. func RegisterSource(typ abstract.ProviderType, fac func() Source) { knownSources[typ] = fac } // RegisterDestination will add new destination factory for specific provider type -// this should be placed inside provide `init() func` +// this should be placed inside provide `init() func`. func RegisterDestination(typ abstract.ProviderType, fac func() Destination) { knownDestinations[typ] = fac } diff --git a/pkg/abstract/model/endpoint_rotator_config.go b/pkg/abstract/model/endpoint_rotator_config.go index 9f24da92..e70b319c 100644 --- a/pkg/abstract/model/endpoint_rotator_config.go +++ b/pkg/abstract/model/endpoint_rotator_config.go @@ -13,7 +13,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// RotationTZ -- rotation is happened in preconfigured timezone, by default - Europe/Moscow +// RotationTZ -- rotation is happened in preconfigured timezone, by default - Europe/Moscow. var RotationTZ *time.Location func init() { @@ -52,7 +52,7 @@ var ( ErrRotatorConfigInvalidKeepPartCount = errors.New("keep part count should be positive (KeepPartCount > 0)") ) -// TODO remove workaround when proper fix rolls out: YCDESIGN-1338 TM-1891 +// TODO remove workaround when proper fix rolls out: YCDESIGN-1338 TM-1891. func (p *RotatorConfig) NilWorkaround() *RotatorConfig { if p == nil { return p @@ -89,7 +89,7 @@ func (p *RotatorConfig) Validate() error { } // breaks up months by partitions with size PartSize -// note, that if PartSize=5, the partitions would be uneven: [0,1,2,3,4], [5,6,7,8,9], [10,11] +// note, that if PartSize=5, the partitions would be uneven: [0,1,2,3,4], [5,6,7,8,9], [10,11]. func (p *RotatorConfig) getMonthPartitioned(m time.Month) time.Month { if p == nil { return m @@ -118,7 +118,7 @@ func (p *RotatorConfig) offsetDate(timestamp time.Time, offset int) time.Time { return timestamp } -// getPartitionBin returns bin representative in which date is falling based on PartSize and PartType parameters +// getPartitionBin returns bin representative in which date is falling based on PartSize and PartType parameters. func (p *RotatorConfig) getPartitionBin(t time.Time) dateBin { if p != nil { switch p.PartType { @@ -134,7 +134,7 @@ func (p *RotatorConfig) getPartitionBin(t time.Time) dateBin { return dateBin(t) } -// formatDate formats date according to PartType parameters +// formatDate formats date according to PartType parameters. func (p *RotatorConfig) dateBinToPartitionName(db dateBin) partitionName { t := time.Time(db) if p != nil { @@ -150,7 +150,7 @@ func (p *RotatorConfig) dateBinToPartitionName(db dateBin) partitionName { return partitionName(t.Format(MonthFormat)) } -// format function produces concrete formatted string given by name and partition +// format function produces concrete formatted string given by name and partition. func (p *RotatorConfig) format(name string, partition partitionName) string { if p.TableNameTemplate == "" { return fmt.Sprintf("%v/%v", name, partition) @@ -160,7 +160,7 @@ func (p *RotatorConfig) format(name string, partition partitionName) string { return strings.ReplaceAll(templ, "{{partition}}", string(partition)) } -// BaseTime defines TTL of tables w.r.t. KeepPartCount window size parameter +// BaseTime defines TTL of tables w.r.t. KeepPartCount window size parameter. func (p *RotatorConfig) BaseTime() time.Time { return p.baseTime(time.Now()) } @@ -174,7 +174,7 @@ func (p *RotatorConfig) baseTime(now time.Time) time.Time { return time.Date(0, 0, 0, 0, 0, 0, 0, RotationTZ) } -// AnnotateWithTime produces rotated name with custom time `v` provided as argument +// AnnotateWithTime produces rotated name with custom time `v` provided as argument. func (p *RotatorConfig) AnnotateWithTime(name string, v time.Time) string { if p == nil { return name @@ -190,19 +190,19 @@ func (p *RotatorConfig) AnnotateWithTime(name string, v time.Time) string { } // Annotate is the default way to acquire rotated name based on the current time -// and structure settings: PartSize, PartType +// and structure settings: PartSize, PartType. func (p *RotatorConfig) Annotate(name string) string { return p.AnnotateWithTime(name, time.Now()) } -// Next returns rotated name with respect to current date +// Next returns rotated name with respect to current date. func (p *RotatorConfig) Next(name string) string { return p.AnnotateWithTime(name, p.offsetDate(time.Now(), 1)) } // AnnotateWithTimeFromColumn gives rotated name according to column with name TimeColumn as the time source // -// if column is absent, default Annotate method will be used +// if column is absent, default Annotate method will be used. func (p *RotatorConfig) AnnotateWithTimeFromColumn(name string, item abstract.ChangeItem) string { if p == nil { return name diff --git a/pkg/abstract/model/endpoint_rotator_config_test.go b/pkg/abstract/model/endpoint_rotator_config_test.go index 52ba65e0..67bc8e05 100644 --- a/pkg/abstract/model/endpoint_rotator_config_test.go +++ b/pkg/abstract/model/endpoint_rotator_config_test.go @@ -1,7 +1,6 @@ package model import ( - "os" "testing" "time" @@ -9,8 +8,9 @@ import ( "github.com/stretchr/testify/require" ) -// test runner +// test runner. func TestRotatorConfig(t *testing.T) { + t.Parallel() t.Run("ScenarioTesting", scenarioTesting) t.Run("NilWorkaround", nilWorkaround) // temporary test t.Run("Validate", validate) @@ -25,15 +25,18 @@ func TestRotatorConfig(t *testing.T) { t.Run("TestTimeParsing", testTimeParsing) } -// some month utility -var monthIds = map[time.Month]int{time.January: 0, time.February: 1, time.March: 2, time.April: 3, time.May: 4, - time.June: 5, time.July: 6, time.August: 7, time.September: 8, time.October: 9, time.November: 10, time.December: 11} -var monthList = []time.Month{time.January, time.February, time.March, time.April, time.May, - time.June, time.July, time.August, time.September, time.October, time.November, time.December} -var monthDayCount = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} -var monthDayCountLeap = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} +// some month utility. +var monthIds = map[time.Month]int{ + time.January: 0, time.February: 1, time.March: 2, time.April: 3, time.May: 4, + time.June: 5, time.July: 6, time.August: 7, time.September: 8, time.October: 9, time.November: 10, time.December: 11, +} -// scenario tests +var monthList = []time.Month{ + time.January, time.February, time.March, time.April, time.May, + time.June, time.July, time.August, time.September, time.October, time.November, time.December, +} + +// scenario tests. func scenarioTesting(t *testing.T) { // this tests based on real user scenarios t.Parallel() @@ -83,10 +86,9 @@ func scenarioTestingDTSUPPORT693(t *testing.T) { timestamp = timestamp.Add(time.Hour) require.Equal(t, rotator.KeepPartCount, len(rotationTables), "Check that there is always window of six tables") } - } -// tests +// tests. func nilWorkaround(t *testing.T) { // TODO delete this test when workaround is not necessary anymore t.Parallel() @@ -120,7 +122,6 @@ func validate(t *testing.T) { require.Error(t, (&RotatorConfig{KeepPartCount: 1, PartType: RotatorPartHour, PartSize: -1, TimeColumn: ""}).Validate(), "PartSize should be positive") require.Error(t, (&RotatorConfig{KeepPartCount: 0, PartType: RotatorPartHour, PartSize: 1, TimeColumn: ""}).Validate(), "KeepPartCount should be positive") require.Error(t, (&RotatorConfig{KeepPartCount: 1, PartType: "kek", PartSize: 1, TimeColumn: ""}).Validate(), "wrong enum value of PartType") - } func getMonthPartitionedTestLight(t *testing.T) { @@ -174,14 +175,13 @@ func getMonthPartitionedTestHeavy(t *testing.T) { require.Equal(t, monthList[i-(i%partSize)], rc.getMonthPartitioned(month)) } } - } func offsetDateTest(t *testing.T) { t.Parallel() t.Run("Hours", offsetDateTestHours) t.Run("Days", offsetDateTestDays) - //t.Run("MonthHeavy", offsetDateTestMonthHeavy) // TODO(@kry127) temporary switched off + // t.Run("MonthHeavy", offsetDateTestMonthHeavy) // TODO(@kry127) temporary switched off t.Run("NilReceiver", offsetDateTestNilReceiver) } @@ -206,8 +206,7 @@ func offsetDateTestHours(t *testing.T) { func offsetDateTestDays(t *testing.T) { t.Parallel() - _ = os.Setenv("TZ", "Europe/Moscow") // this test is timezone aware - defer os.Unsetenv("TZ") + t.Setenv("TZ", "Europe/Moscow") // this test is timezone aware rcDays := RotatorConfig{KeepPartCount: 0, PartType: RotatorPartDay, PartSize: 1, TimeColumn: ""} rcDaysTimestamp := time.Now() @@ -233,103 +232,6 @@ func offsetDateTestNilReceiver(t *testing.T) { } } -func isLeap(year int) bool { - if year%400 == 0 { - return true // 3. but every four hundredth year is and exception again - } - if year%100 == 0 { - return false // 2. except for every hundredth year - } - if year%4 == 0 { - return true // 1. every fourth year is leap - } - return false -} - -func countDaysForYearAcc(year, month, offset int, acc int64) int64 { - switch { - case offset > 0: - dayDiff := 0 - if isLeap(year) { - dayDiff = monthDayCountLeap[month] - } else { - dayDiff = monthDayCount[month] - } - nextMonth := month + 1 - nextYear := year - if nextMonth == len(monthList) { - nextMonth = 0 - nextYear++ - } - return countDaysForYearAcc(nextYear, nextMonth, offset-1, acc+int64(dayDiff)) - case offset < 0: - prevMonth := month - 1 - prevYear := year - if prevMonth < 0 { - prevMonth = len(monthList) - 1 - prevYear-- - } - - dayDiff := 0 - if isLeap(prevYear) { - dayDiff = monthDayCountLeap[prevMonth] - } else { - dayDiff = monthDayCount[prevMonth] - } - return countDaysForYearAcc(prevYear, prevMonth, offset-1, acc-int64(dayDiff)) - default: - return acc - } -} -func countDaysForYear(year, month, offset int) int64 { - return countDaysForYearAcc(year, month, offset, 0) -} - -func offsetDateTestMonthHeavy(t *testing.T) { - t.Parallel() - checkYear := func(t *testing.T, year, partSize int) { - t.Helper() - t.Parallel() - rcMonths := RotatorConfig{KeepPartCount: 0, PartType: RotatorPartMonth, PartSize: partSize, TimeColumn: ""} - nowTimestamp := time.Now() - for offset := 1; offset < 15; offset++ { - for monthID, month := range monthList { - // NOTE UTC parameter in tests! This test will not work for RotationTZ timezone with +1 and -1 correction hours - ts := time.Date(year, month, nowTimestamp.Day(), nowTimestamp.Hour(), nowTimestamp.Minute(), nowTimestamp.Second(), nowTimestamp.Nanosecond(), time.UTC) - offTimestamp := rcMonths.offsetDate(ts, offset) - - expectedHoursOffset := 24 * countDaysForYear(year, monthID, offset*partSize) - require.Equal(t, time.Duration(expectedHoursOffset)*time.Hour, offTimestamp.Sub(ts), "offset=%d for year=%d, month=%v should be %d hours with PartSize=%d", offset, year, month, expectedHoursOffset, partSize) - - } - } - } - - // check some juicy years - // leap years - t.Run("check year 2000 part size 1", func(t *testing.T) { checkYear(t, 2000, 1) }) - t.Run("check year 2000 part size 2", func(t *testing.T) { checkYear(t, 2000, 2) }) - t.Run("check year 2004 part size 3", func(t *testing.T) { checkYear(t, 2004, 3) }) - t.Run("check year 2005 part size 4", func(t *testing.T) { checkYear(t, 2005, 4) }) - t.Run("check year 2024 part size 1", func(t *testing.T) { checkYear(t, 2024, 1) }) - t.Run("check year 2028 part size 1", func(t *testing.T) { checkYear(t, 2028, 1) }) - t.Run("check year 2400 part size 1", func(t *testing.T) { checkYear(t, 2005, 1) }) - t.Run("check year 2400 part size 2", func(t *testing.T) { checkYear(t, 2024, 2) }) - t.Run("check year 2400 part size 3", func(t *testing.T) { checkYear(t, 2028, 3) }) - - // not leap years - t.Run("check year 2021 part size 1", func(t *testing.T) { checkYear(t, 2021, 1) }) - t.Run("check year 2021 part size 2", func(t *testing.T) { checkYear(t, 2021, 2) }) - t.Run("check year 2021 part size 3", func(t *testing.T) { checkYear(t, 2021, 3) }) - t.Run("check year 2021 part size 5", func(t *testing.T) { checkYear(t, 2021, 5) }) - t.Run("check year 2021 part size 6", func(t *testing.T) { checkYear(t, 2021, 6) }) - t.Run("check year 2021 part size 7", func(t *testing.T) { checkYear(t, 2021, 7) }) - t.Run("check year 2100 part size 1", func(t *testing.T) { checkYear(t, 2100, 1) }) - t.Run("check year 2100 part size 5", func(t *testing.T) { checkYear(t, 2100, 5) }) - t.Run("check year 2100 part size 12", func(t *testing.T) { checkYear(t, 2100, 12) }) - -} - func getPartitionBin(t *testing.T) { t.Parallel() var rotatorNil RotatorConfig @@ -358,12 +260,14 @@ func annotateWithTimeFromColumnTest(t *testing.T) { t.Run("NoTimeColumnInConfig", annotateWithTimeFromColumnTestNoTimeColumnInConfig) t.Run("NilReceiver", annotateWithTimeFromColumnTestNilReceiver) } + func annotateWithTimeFromColumnTestWithoutFormat(t *testing.T) { t.Parallel() t.Run("Hours", annotateWithTimeFromColumnTestWithoutFormatHours) t.Run("Days", annotateWithTimeFromColumnTestWithoutFormatDays) t.Run("Months", annotateWithTimeFromColumnTestWithoutFormatMonths) } + func annotateWithTimeFromColumnTestWithFormat(t *testing.T) { t.Parallel() t.Run("Hours", annotateWithTimeFromColumnTestWithTemplateHours) diff --git a/pkg/abstract/model/includeable.go b/pkg/abstract/model/includeable.go index ba9924d1..d934b549 100644 --- a/pkg/abstract/model/includeable.go +++ b/pkg/abstract/model/includeable.go @@ -15,7 +15,7 @@ type Includeable interface { AllIncludes() []string } -// FilteredMap filters IN-PLACE and returns its first argument +// FilteredMap filters IN-PLACE and returns its first argument. func FilteredMap(m abstract.TableMap, incls ...abstract.Includeable) abstract.TableMap { TABLES: for tID := range m { diff --git a/pkg/abstract/model/tmp_policy_config.go b/pkg/abstract/model/tmp_policy_config.go index 5a19f332..c0c9f867 100644 --- a/pkg/abstract/model/tmp_policy_config.go +++ b/pkg/abstract/model/tmp_policy_config.go @@ -38,11 +38,11 @@ const ( ) type TmpPolicyProvider interface { - //Common naive tmp policy: implemented via middleware which one just rename change items during snapshot and call sink method Move(tmp_table -> orig_table) on DoneLoadTable event. - //Sharded snapshots are not supported in this mode. + // Common naive tmp policy: implemented via middleware which one just rename change items during snapshot and call sink method Move(tmp_table -> orig_table) on DoneLoadTable event. + // Sharded snapshots are not supported in this mode. EnsureTmpPolicySupported() error - //Some destinations have their own custom implementations with tmp policy logic (e.g. YT dynamic tables sink - UseStaticTablesOnSnapshot). - //Custom policy is more preferable than common tmp policy. + // Some destinations have their own custom implementations with tmp policy logic (e.g. YT dynamic tables sink - UseStaticTablesOnSnapshot). + // Custom policy is more preferable than common tmp policy. //If custom tmp policy is enabled we won`t drop tables on a destination on cleanup sinker stage. EnsureCustomTmpPolicySupported() error } diff --git a/pkg/abstract/model/transfer.go b/pkg/abstract/model/transfer.go index 9b0befd3..d072e4fe 100644 --- a/pkg/abstract/model/transfer.go +++ b/pkg/abstract/model/transfer.go @@ -38,7 +38,7 @@ const ( // // Zero value is reserved and MUST NOT be used. // - // When incrementing this value, DO ADD a link to the function(s) implementing this fallback to CHANGELOG.md in the current directory + // When incrementing this value, DO ADD a link to the function(s) implementing this fallback to CHANGELOG.md in the current directory. LatestVersion int = 10 // NewTransfersVersion is the version of the typesystem set for new transfers. It must be less or equal to the LatestVersion. // @@ -112,7 +112,7 @@ func (f *Transfer) WithDefault() { // For now, it's called: // - in repository, when building transfer object from the db // - in transitive uploading, when creating synthetic transfer -// - in e2e-tests, when building transfer object from endpoints (helpers: MakeTransfer/InitSrcDst) +// - in e2e-tests, when building transfer object from endpoints (helpers: MakeTransfer/InitSrcDst). func (f *Transfer) FillDependentFields() { if f.TypeSystemVersion == 0 { // the LatestVersion is used here instead of the NewTransfersVersion for test purposes @@ -256,7 +256,7 @@ func (f *Transfer) TransformationFromJSON(value string) error { return nil } -// IsTransitional show transfer that used by kostya and burn our pukans +// IsTransitional show transfer that used by kostya and burn our pukans. func (f *Transfer) IsTransitional() bool { if _, ok := f.Src.(TransitionalEndpoint); ok { return true @@ -405,7 +405,7 @@ func (f *Transfer) Include(tID abstract.TableID) bool { // SystemLabel method is used to access system labels for transfer. // System labels are special reserved labels which are used to control some -// hidden experimental transfer features +// hidden experimental transfer features. func (f *Transfer) SystemLabel(name SystemLabel) (string, error) { labelMap := map[string]string{} if err := json.Unmarshal([]byte(f.Labels), &labelMap); err != nil { diff --git a/pkg/abstract/model/transfer_labels.go b/pkg/abstract/model/transfer_labels.go index cacfd5ef..4f3ca8a6 100644 --- a/pkg/abstract/model/transfer_labels.go +++ b/pkg/abstract/model/transfer_labels.go @@ -3,7 +3,7 @@ package model // SystemLabel contains transfer label names which are reserved to control // hidden experimental transfer features. // -// Each SystemLabel value must be documented +// Each SystemLabel value must be documented. type SystemLabel string // SystemLabelMemThrottle is used to enable/disable MemoryThrorrler middleware. diff --git a/pkg/abstract/operations.go b/pkg/abstract/operations.go index 1509c8c0..b1962284 100644 --- a/pkg/abstract/operations.go +++ b/pkg/abstract/operations.go @@ -40,13 +40,13 @@ type TimeoutableTask interface { HasTimedOut(createdAt time.Time, pingedAt time.Time) bool } -// Any task is either runnable or fake; embed one of these two types in a new task +// Any task is either runnable or fake; embed one of these two types in a new task. type ( runnable struct{} fake struct{} ) -// Runnable tasks {{{ +// Runnable tasks {{{. type ( Activate struct { runnable @@ -106,7 +106,7 @@ var RunnableTasks []RunnableTask = makeRunnableTasks() // }}} -// Fake tasks {{{ +// Fake tasks {{{. type ( Replication struct{ fake } Termination struct{ fake } @@ -172,7 +172,7 @@ func (p UpdateTransferParams) AddedTables() ([]TableDescription, error) { var AllTasks []Task = makeAllTasks() -// Visitors {{{ +// Visitors {{{. type TaskVisitor interface { RunnableVisitor FakeVisitor @@ -208,7 +208,7 @@ type FakeVisitor interface { // }}} -// Method implementations {{{ +// Method implementations {{{. func (t runnable) isTaskType() {} func (t runnable) isRunnable() {} func (t fake) isTaskType() {} diff --git a/pkg/abstract/runtime.go b/pkg/abstract/runtime.go index a4487ce6..12424e77 100644 --- a/pkg/abstract/runtime.go +++ b/pkg/abstract/runtime.go @@ -55,10 +55,10 @@ func KnownRuntime(r RuntimeType) bool { return ok } -// Parallelism params +// Parallelism params. type ShardUploadParams struct { - JobCount int //Workers count - ProcessCount int //Threads count + JobCount int // Workers count + ProcessCount int // Threads count } func NewShardUploadParams(jobCount int, processCount int) *ShardUploadParams { diff --git a/pkg/abstract/sink.go b/pkg/abstract/sink.go index 63a91a41..64afe70a 100644 --- a/pkg/abstract/sink.go +++ b/pkg/abstract/sink.go @@ -10,7 +10,7 @@ type ( // // All its methods are guaranteed to be called non-concurrently (synchronously). // -// TODO: rename to Sink +// TODO: rename to Sink. type Sinker interface { io.Closer // Push writes the given items into destination synchronously. If its result is nil, the items are considered to be successfully written to the destination. @@ -18,5 +18,5 @@ type Sinker interface { Push(items []ChangeItem) error } -// TODO: Drop by making transformers a common middleware +// TODO: Drop by making transformers a common middleware. type SinkOption func(sinker Sinker) Sinker diff --git a/pkg/abstract/storage.go b/pkg/abstract/storage.go index 0500e00e..d107e276 100644 --- a/pkg/abstract/storage.go +++ b/pkg/abstract/storage.go @@ -156,20 +156,17 @@ overFQTNRunes: if !previousWasDoubleQuote { return nil, xerrors.Errorf("missing ending \" in position [%d]", len(identifier)-1) } - previousWasDoubleQuote = false - insideDoubleQuotedIdentifier = false } result = append(result, idPartBuilder.String()) idPartBuilder.Reset() - identifierInProgress = false return result, nil } type TableInfo struct { - EtaRow uint64 - IsView bool - Schema *TableSchema + EtaRow uint64 `json:"eta_row"` + IsView bool `json:"is_view"` + Schema *TableSchema `json:"schema"` } type TableMap map[TableID]TableInfo @@ -279,7 +276,7 @@ func (t *TableDescription) String() string { return fmt.Sprintf("%s [filter %q offset %d]", t.Fqtn(), t.Filter, t.Offset) } -// TableIDsIntersection returns an intersection of two lists of TableIDs +// TableIDsIntersection returns an intersection of two lists of TableIDs. func TableIDsIntersection(a []TableID, b []TableID) []TableID { if len(b) == 0 { return a @@ -318,7 +315,7 @@ type Storage interface { TableExists(table TableID) (bool, error) } -// PositionalStorage some storages may provide specific position for snapshot consistency +// PositionalStorage some storages may provide specific position for snapshot consistency. type PositionalStorage interface { // Position provide info about snapshot read position Position(ctx context.Context) (*LogPosition, error) @@ -330,12 +327,12 @@ type LogPosition struct { TxID string } -// SchemaStorage allow to resolve DB Schema from storage +// SchemaStorage allow to resolve DB Schema from storage. type SchemaStorage interface { LoadSchema() (DBSchema, error) } -// SampleableStorage is for dataplane tests +// SampleableStorage is for dataplane tests. type SampleableStorage interface { Storage @@ -346,12 +343,12 @@ type SampleableStorage interface { TableAccessible(table TableDescription) bool } -// ShardingStorage is for in table sharding +// ShardingStorage is for in table sharding. type ShardingStorage interface { ShardTable(ctx context.Context, table TableDescription) ([]TableDescription, error) } -// Storage has data, that need to be shared with all workers +// Storage has data, that need to be shared with all workers. type ShardingContextStorage interface { // ShardingContext Return shared data, used on *MAIN* worker; // Take care, method return OperationState_ShardedUploadState, but only fill field Context; diff --git a/pkg/abstract/test_result.go b/pkg/abstract/test_result.go index 2d7c631e..8b70f52a 100644 --- a/pkg/abstract/test_result.go +++ b/pkg/abstract/test_result.go @@ -5,16 +5,16 @@ import ( "github.com/doublecloud/transfer/pkg/abstract/changeitem" ) -// CheckType test check type +// CheckType test check type. type CheckType string -// CheckResult describe particular check result that was performed against endpoint / transfer +// CheckResult describe particular check result that was performed against endpoint / transfer. type CheckResult struct { Error error Success bool } -// TestResult aggregated result of test for endpoint +// TestResult aggregated result of test for endpoint. type TestResult struct { Checks map[CheckType]CheckResult Schema TableMap @@ -27,7 +27,7 @@ func (t *TestResult) Add(extraChecks ...CheckType) { } } -// Combine combines the two checkResult maps into one +// Combine combines the two checkResult maps into one. func (t *TestResult) Combine(partialResults *TestResult) { for checkType, checkVal := range partialResults.Checks { t.Checks[checkType] = checkVal diff --git a/pkg/abstract/typed_change_item.go b/pkg/abstract/typed_change_item.go index 1219e715..57a52f5b 100644 --- a/pkg/abstract/typed_change_item.go +++ b/pkg/abstract/typed_change_item.go @@ -10,11 +10,13 @@ import ( "github.com/valyala/fastjson" ) -type TypedChangeItem ChangeItem -type TypedValue struct { - Type string `json:"type"` - Value any `json:"value"` -} +type ( + TypedChangeItem ChangeItem + TypedValue struct { + Type string `json:"type"` + Value any `json:"value"` + } +) func (t *TypedChangeItem) MarshalJSON() ([]byte, error) { _, err := t.packTypedValues() @@ -66,22 +68,6 @@ func (t *TypedChangeItem) packTypedValues() ([]TypedValue, error) { return res, nil } -func (t *TypedChangeItem) unpackTypedValues(v *fastjson.Value) ([]interface{}, error) { - items, err := v.Array() - if err != nil { - return nil, xerrors.Errorf("unable to get array: %w", err) - } - var res []interface{} - for _, item := range items { - extractedV, err := unpackTypedValue(item) - if err != nil { - return nil, xerrors.Errorf("unable to unpack: %w", err) - } - res = append(res, extractedV) - } - return res, nil -} - func extractVal[T any](val []byte) (T, error) { var result T if err := json.Unmarshal(val, &result); err != nil { @@ -231,7 +217,7 @@ func packTypedValue(val any) (TypedValue, error) { items = append(items, packedV) } if len(errs) > 0 { - return *new(TypedValue), xerrors.Errorf("many errs: %w", errs) + return TypedValue{}, xerrors.Errorf("many errs: %w", errs) } return TypedValue{Value: items, Type: typeNameFromValue(val)}, nil case map[string]interface{}: @@ -243,7 +229,7 @@ func packTypedValue(val any) (TypedValue, error) { errs = util.AppendErr(errs, err) } if len(errs) > 0 { - return *new(TypedValue), xerrors.Errorf("many errs: %w", errs) + return TypedValue{}, xerrors.Errorf("many errs: %w", errs) } return TypedValue{Value: data, Type: typeNameFromValue(val)}, nil case [][]interface{}: // Clickhouse nested arrays @@ -259,7 +245,7 @@ func packTypedValue(val any) (TypedValue, error) { data = append(data, items) } if len(errs) > 0 { - return *new(TypedValue), xerrors.Errorf("many errs: %w", errs) + return TypedValue{}, xerrors.Errorf("many errs: %w", errs) } return TypedValue{Value: data, Type: typeNameFromValue(val)}, nil case *TableSchema: diff --git a/pkg/abstract/typed_change_item_test.go b/pkg/abstract/typed_change_item_test.go index 324f64ad..ab5da480 100644 --- a/pkg/abstract/typed_change_item_test.go +++ b/pkg/abstract/typed_change_item_test.go @@ -3,7 +3,6 @@ package abstract import ( "encoding/json" "fmt" - "os" "testing" "time" @@ -11,8 +10,7 @@ import ( ) func TestTypedChangeItem(t *testing.T) { - _ = os.Setenv("TZ", "NZ") // see: https://github.com/golang/go/issues/45960 - defer os.Unsetenv("TZ") + t.Setenv("TZ", "NZ") // see: https://github.com/golang/go/issues/45960 ci := &TypedChangeItem{ ID: 291975574, CommitTime: 1601382119000000000, diff --git a/pkg/abstract/typesystem/fallback.go b/pkg/abstract/typesystem/fallback.go index e0130277..31dcf964 100644 --- a/pkg/abstract/typesystem/fallback.go +++ b/pkg/abstract/typesystem/fallback.go @@ -44,14 +44,14 @@ func (f Fallback) String() string { // AddFallbackSourceFactory registers a fallbacks for a source of some type // -// This method is expected to be called in the `init()` function of a module which introduces a fallback when some additional behaviour is expected +// This method is expected to be called in the `init()` function of a module which introduces a fallback when some additional behaviour is expected. func AddFallbackSourceFactory(factory FallbackFactory) { registerFallbackFactory(&SourceFallbackFactories, factory) } // AddFallbackTargetFactory registers a fallbacks for a target of some type // -// This method is expected to be called in the `init()` function of a module which introduces a fallback when some additional behaviour is expected +// This method is expected to be called in the `init()` function of a module which introduces a fallback when some additional behaviour is expected. func AddFallbackTargetFactory(factory FallbackFactory) { registerFallbackFactory(&TargetFallbackFactories, factory) } diff --git a/pkg/abstract/typesystem/values/type_checkers.go b/pkg/abstract/typesystem/values/type_checkers.go index c7e0e9e9..88e2f35d 100644 --- a/pkg/abstract/typesystem/values/type_checkers.go +++ b/pkg/abstract/typesystem/values/type_checkers.go @@ -35,7 +35,7 @@ func jsonSerializeable(val interface{}) bool { // OneofValueTypeCheckers returns checkers permitting different representations for a particular YT type // -// TODO: TM-4130: switch to StrictValueTypeCheckers() +// TODO: TM-4130: switch to StrictValueTypeCheckers(). func OneofValueTypeCheckers() map[schema.Type]ValueTypeChecker { return map[schema.Type]ValueTypeChecker{ schema.TypeBoolean: strict(false), @@ -59,7 +59,7 @@ func OneofValueTypeCheckers() map[schema.Type]ValueTypeChecker { } } -// StrictValueTypeCheckers returns checkers permitting only the single correct representation for a particular YT type +// StrictValueTypeCheckers returns checkers permitting only the single correct representation for a particular YT type. func StrictValueTypeCheckers() map[schema.Type]ValueTypeChecker { return map[schema.Type]ValueTypeChecker{ schema.TypeBoolean: strict(false), diff --git a/pkg/base/events/cleanup.go b/pkg/base/events/cleanup.go index 728765b0..7d886208 100644 --- a/pkg/base/events/cleanup.go +++ b/pkg/base/events/cleanup.go @@ -4,7 +4,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// TODO: Remove after new cleanup design based on TableLoadEvents +// TODO: Remove after new cleanup design based on TableLoadEvents. type CleanupEvent abstract.TableID func (c *CleanupEvent) String() string { diff --git a/pkg/base/events/table_events.go b/pkg/base/events/table_events.go index 4665fedf..b1010ee9 100644 --- a/pkg/base/events/table_events.go +++ b/pkg/base/events/table_events.go @@ -76,7 +76,7 @@ func (b *insertsBatch) MarshalYSON(w *yson.Writer) error { case types.UInt32Value: w.Uint64(uint64(*v.UInt32Value())) case types.UInt64Value: - w.Uint64(uint64(*v.UInt64Value())) + w.Uint64(*v.UInt64Value()) case types.FloatValue: w.Float64(float64(*v.FloatValue())) case types.DoubleValue: diff --git a/pkg/base/events/table_load.go b/pkg/base/events/table_load.go index 233087c4..ddd0922f 100644 --- a/pkg/base/events/table_load.go +++ b/pkg/base/events/table_load.go @@ -10,7 +10,7 @@ import ( type TableLoadState int -// It is important for serialization not to use iota +// It is important for serialization not to use iota. const ( InitShardedTableLoad = TableLoadState(4) TableLoadBegin = TableLoadState(1) diff --git a/pkg/base/events/transaction.go b/pkg/base/events/transaction.go index 8c78b957..01fc2b13 100644 --- a/pkg/base/events/transaction.go +++ b/pkg/base/events/transaction.go @@ -6,7 +6,7 @@ import ( type TransactionState int -// It is important for serialization not to use iota +// It is important for serialization not to use iota. const ( TransactionBegin TransactionState = TransactionState(1) TransactionCommit TransactionState = TransactionState(2) diff --git a/pkg/base/transfer.go b/pkg/base/transfer.go index 6b7946a6..836f78be 100644 --- a/pkg/base/transfer.go +++ b/pkg/base/transfer.go @@ -106,7 +106,7 @@ func NewBatchFromBatches(batches []EventBatch) EventBatch { return &iteratorOverBatched{batches: batches, iter: -1} } -// Support legacy +// Support legacy. type SupportsOldChangeItem interface { Event ToOldChangeItem() (*abstract.ChangeItem, error) @@ -118,13 +118,13 @@ type LogPosition interface { Compare(otherPosition LogPosition) (int, error) } -// Support legacy +// Support legacy. type SupportsOldLSN interface { LogPosition ToOldLSN() (uint64, error) // To ChangeItem.LSN } -// Support legacy +// Support legacy. type SupportsOldCommitTime interface { LogPosition ToOldCommitTime() (uint64, error) // To ChangeItem.CommitTime @@ -147,7 +147,7 @@ type TransactionalEvent interface { Transaction() Transaction } -// For any event source, replication, snapshot, etc +// For any event source, replication, snapshot, etc. type EventSource interface { Running() bool Start(ctx context.Context, target EventTarget) error @@ -159,7 +159,7 @@ type Discoverable interface { } // Reporting of progress for not endless event sources -// Like snapshots, schema creation, etc +// Like snapshots, schema creation, etc. type ProgressableEventSource interface { EventSource Progress() (EventSourceProgress, error) @@ -197,7 +197,7 @@ func (progress *DefaultEventSourceProgress) Total() uint64 { return progress.total } -// abstract.AsyncSink for abstract2 +// abstract.AsyncSink for abstract2. type EventTarget interface { io.Closer AsyncPush(input EventBatch) chan error diff --git a/pkg/cobraaux/cobraaux.go b/pkg/cobraaux/cobraaux.go index 3fcd1846..cd135e9b 100644 --- a/pkg/cobraaux/cobraaux.go +++ b/pkg/cobraaux/cobraaux.go @@ -6,7 +6,7 @@ import ( ) // RegisterCommand is like parent.AddCommand(child), but also -// makes chaining of PersistentPreRunE and PersistentPreRun +// makes chaining of PersistentPreRunE and PersistentPreRun. func RegisterCommand(parent, child *cobra.Command) { parentPpre := parent.PersistentPreRunE childPpre := child.PersistentPreRunE diff --git a/pkg/coordinator/s3coordinator/coordinator_s3.go b/pkg/coordinator/s3coordinator/coordinator_s3.go index 801ea250..24f8383f 100644 --- a/pkg/coordinator/s3coordinator/coordinator_s3.go +++ b/pkg/coordinator/s3coordinator/coordinator_s3.go @@ -113,7 +113,7 @@ func (c *CoordinatorS3) RemoveTransferState(transferID string, keys []string) er return nil } -// GetOperationProgress do nothing +// GetOperationProgress do nothing. func (c *CoordinatorS3) GetOperationProgress(operationID string) (*model.AggregatedProgress, error) { return model.NewAggregatedProgress(), nil } diff --git a/pkg/csv/error.go b/pkg/csv/error.go index 800da08c..322276b3 100644 --- a/pkg/csv/error.go +++ b/pkg/csv/error.go @@ -4,7 +4,6 @@ import "github.com/doublecloud/transfer/library/go/core/xerrors" var ( errInvalidDelimiter = xerrors.NewSentinel("csv: invalid delimiter") - errInvalidEscape = xerrors.NewSentinel("csv: escape char used outside of quoted text") errDoubleQuotesDisabled = xerrors.NewSentinel("csv: found double quotes while double quote feature disabled") errQuotingDisabled = xerrors.NewSentinel("csv: found quote char while feature disabled") ) diff --git a/pkg/csv/reader.go b/pkg/csv/reader.go index edf4af34..ca29c260 100644 --- a/pkg/csv/reader.go +++ b/pkg/csv/reader.go @@ -131,7 +131,7 @@ func (r *Reader) readSingleLine() ([]string, error) { return entries, nil } -// readAndDecodeLine reads until \n from buffer and returns a string decoded based on configured Encoding +// readAndDecodeLine reads until \n from buffer and returns a string decoded based on configured Encoding. func (r *Reader) readAndDecodeLine() (string, error) { var err error read, err := r.reader.ReadBytes('\n') @@ -159,7 +159,7 @@ func (r *Reader) readAndDecodeLine() (string, error) { line = string(converted) } - if r.QuoteChar == 0 && strings.Contains(string(line), string('"')) { + if r.QuoteChar == 0 && strings.Contains(line, string('"')) { return "", errQuotingDisabled } diff --git a/pkg/csv/reader_test.go b/pkg/csv/reader_test.go index 59c11de4..56a7c76d 100644 --- a/pkg/csv/reader_test.go +++ b/pkg/csv/reader_test.go @@ -10,7 +10,7 @@ import ( "golang.org/x/text/encoding/charmap" ) -// this function needed to check simultaneously: ReadAll() & ValidateOneLine() - they always should give the same result +// this function needed to check simultaneously: ReadAll() & ValidateOneLine() - they always should give the same result. func csvReaderReadAll(t *testing.T, reader *Reader) ([][]string, error) { result, errExpected := reader.ReadAll() for _, el := range result { diff --git a/pkg/csv/splitter.go b/pkg/csv/splitter.go index d4396538..30b44267 100644 --- a/pkg/csv/splitter.go +++ b/pkg/csv/splitter.go @@ -13,15 +13,15 @@ import ( type splitterState int const ( - // Normal state, the default one upon initialization + // Normal state, the default one upon initialization. outsideQuote = splitterState(iota) - // We are inside double quotes and did not find the closing quote yet + // We are inside double quotes and did not find the closing quote yet. quoteOpen // We are inside double quotes, found the closing quote and we are yet to // decide whether it is the closing double quote or an escape sequence - // (i.e. "", two double quotes inside double quotes) + // (i.e. "", two double quotes inside double quotes). closingQuote ) diff --git a/pkg/dbaas/host_port.go b/pkg/dbaas/host_port.go index 1c74894d..960281c9 100644 --- a/pkg/dbaas/host_port.go +++ b/pkg/dbaas/host_port.go @@ -12,7 +12,7 @@ import ( // If port present in 'host' string - it overrides 'port' from config // At least it's very useful for testing - you can to upraise locally pgHA installation // And it also can be useful for any user, who has postgres servers on >1 port -// It can be used with any HA installation - for now it's used only with pg +// It can be used with any HA installation - for now it's used only with pg. func ResolveHostPortWithOverride(host string, port uint16) (string, uint16, error) { if host == "" { return host, port, nil diff --git a/pkg/dblog/tablequery/storage.go b/pkg/dblog/tablequery/storage.go index b3e1a880..89b3b222 100644 --- a/pkg/dblog/tablequery/storage.go +++ b/pkg/dblog/tablequery/storage.go @@ -6,7 +6,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// StorageTableQueryable is storage with table query loading +// StorageTableQueryable is storage with table query loading. type StorageTableQueryable interface { abstract.SampleableStorage diff --git a/pkg/dblog/utils.go b/pkg/dblog/utils.go index 66059ef7..ed295eea 100644 --- a/pkg/dblog/utils.go +++ b/pkg/dblog/utils.go @@ -114,9 +114,8 @@ func ResolvePrimaryKeyColumns( ctx context.Context, storage abstract.Storage, tableID abstract.TableID, - IsSupportedKeyType func(keyType string) bool, + isSupportedKeyType func(keyType string) bool, ) ([]string, error) { - schema, err := storage.TableSchema(ctx, tableID) if err != nil { return nil, xerrors.Errorf("unable to get table schema tableID: %s, err: %w", tableID, err) @@ -129,7 +128,7 @@ func ResolvePrimaryKeyColumns( primaryKey = append(primaryKey, column.ColumnName) } - if !IsSupportedKeyType(column.OriginalType) { + if !isSupportedKeyType(column.OriginalType) { return nil, xerrors.Errorf("unsupported by data-transfer incremental snapshot") } } diff --git a/pkg/debezium/common/field_receiver.go b/pkg/debezium/common/field_receiver.go index fed30b43..db8bb0b3 100644 --- a/pkg/debezium/common/field_receiver.go +++ b/pkg/debezium/common/field_receiver.go @@ -141,7 +141,7 @@ type Int16ToInt8 interface { } // IntToInt32 -// special case - generalization Int*ToInt32 +// special case - generalization Int*ToInt32. type IntToInt32 interface { Do(int64, *OriginalTypeInfo, *Schema, bool) (int32, error) } @@ -155,7 +155,7 @@ type IntToUint16 interface { } // IntToString -// special case - generalization Int*ToString +// special case - generalization Int*ToString. type IntToString interface { Do(int64, *OriginalTypeInfo, *Schema, bool) (string, error) } @@ -235,19 +235,19 @@ type StructToString interface { // two workarounds // AnyToDouble -// special for io.debezium.data.VariableScaleDecimal +// special for io.debezium.data.VariableScaleDecimal. type AnyToDouble interface { Do(interface{}, *OriginalTypeInfo, *Schema, bool) (json.Number, error) } // AnyToAny -// it's: ArrayToAny & StructToAny +// it's: ArrayToAny & StructToAny. type AnyToAny interface { Do(interface{}, *OriginalTypeInfo, *Schema, bool) (interface{}, error) // string ret_val: result_type } // ContainsColSchemaAdditionalInfo -// modifies ColSchema, adds special parameters +// modifies ColSchema, adds special parameters. type ContainsColSchemaAdditionalInfo interface { AddInfo(*Schema, *abstract.ColSchema) } diff --git a/pkg/debezium/emitter_value_converter.go b/pkg/debezium/emitter_value_converter.go index 34fd9ff9..0dd1b4fb 100644 --- a/pkg/debezium/emitter_value_converter.go +++ b/pkg/debezium/emitter_value_converter.go @@ -76,7 +76,7 @@ type emitType int const ( // It's when 1 changeItem generates 1 debezium event, for example: // - insert - // - update (with updating non-pkey) + // - update (with updating non-pkey). regularEmitType = emitType(0) @@ -89,7 +89,7 @@ const ( // // Delete generates 2 debezium events: // - delete row - // - tombstone event + // - tombstone event. deleteEventEmitType = emitType(1) tombstoneEventEmitType = emitType(2) @@ -135,7 +135,7 @@ func arrColSchemaToFieldsDescrKeys(arrColSchema []abstract.ColSchema, snapshot b return fields, nil } -// add - function for adding (with conversion) one field +// add - function for adding (with conversion) one field. func add(colSchema *abstract.ColSchema, colName string, colVal interface{}, originalType string, ignoreUnknownSources, snapshot bool, connectorParameters map[string]string, result *debeziumcommon.Values) error { if strings.HasPrefix(originalType, "pg:") { if strings.HasSuffix(originalType, "[]") { @@ -204,7 +204,7 @@ func makeValuesWithUnpack(tableSchema []abstract.ColSchema, connectorParameters return result, nil } -// makeValues - is used for build dict for both: 'after' and 'before' +// makeValues - is used for build dict for both: 'after' and 'before'. func makeValues(tableSchema []abstract.ColSchema, connectorParameters map[string]string, names []string, vals []interface{}, keysOnly, ignoreUnknownSources, snapshot bool) (*debeziumcommon.Values, error) { mapColToIndex := make(map[string]int) for i, col := range tableSchema { @@ -255,7 +255,7 @@ func (m *Emitter) GetPackers() (packer.Packer, packer.Packer) { // makeKey - builds 'key' for kafka // for inserts/updates(without pkey changing) - just pkeys from 'after' -// for deletes/update(with pkey changing) - extracts pkeys from OldKeys +// for deletes/update(with pkey changing) - extracts pkeys from OldKeys. func (m *Emitter) makeKey(changeItem *abstract.ChangeItem, useAfter bool, ignoreUnknownSources, snapshot bool) (*debeziumcommon.Values, error) { if useAfter || changeItem.OldKeys.KeyNames == nil { return buildKV(changeItem, m.connectorParameters, true, ignoreUnknownSources, snapshot) @@ -273,7 +273,7 @@ func (m *Emitter) makeKey(changeItem *abstract.ChangeItem, useAfter bool, ignore return result, nil } -// hasPreviousValues - happens when replica_identity=full (for pg), and same things for other databases +// hasPreviousValues - happens when replica_identity=full (for pg), and same things for other databases. func hasPreviousValues(changeItem *abstract.ChangeItem) bool { pkeysNum := 0 for _, el := range changeItem.TableSchema.Columns() { @@ -284,7 +284,7 @@ func hasPreviousValues(changeItem *abstract.ChangeItem) bool { return len(changeItem.OldKeys.KeyNames) > pkeysNum } -// BuildKVMap - builds 'after' k-v map +// BuildKVMap - builds 'after' k-v map. func BuildKVMap(changeItem *abstract.ChangeItem, connectorParameters map[string]string, snapshot bool) (map[string]interface{}, error) { values, err := buildKV(changeItem, connectorParameters, false, true, snapshot) if err != nil { @@ -293,7 +293,7 @@ func BuildKVMap(changeItem *abstract.ChangeItem, connectorParameters map[string] return values.V, nil } -// buildKV - builds 'after' k-v map +// buildKV - builds 'after' k-v map. func buildKV(changeItem *abstract.ChangeItem, connectorParameters map[string]string, keysOnly, ignoreUnknownSources, snapshot bool) (*debeziumcommon.Values, error) { mapColNameToIndex := make(map[string]int) for i, col := range changeItem.TableSchema.Columns() { @@ -389,7 +389,7 @@ func (m *Emitter) ToKafkaSchemaKey(changeItem *abstract.ChangeItem, snapshot boo return result, nil } -// ToKafkaPayloadKey - generate schema for a key message +// ToKafkaPayloadKey - generate schema for a key message. func (m *Emitter) ToKafkaPayloadKey(changeItem *abstract.ChangeItem, snapshot bool, emitType emitType) ([]byte, error) { afterKey, err := m.makeKey(changeItem, emitType == insertEventEmitType, m.ignoreUnknownSources, snapshot) if err != nil { @@ -398,7 +398,7 @@ func (m *Emitter) ToKafkaPayloadKey(changeItem *abstract.ChangeItem, snapshot bo return util.JSONMarshalUnescape(afterKey.V) } -// ToKafkaSchemaVal - generate a schema for a val message +// ToKafkaSchemaVal - generate a schema for a val message. func (m *Emitter) ToKafkaSchemaVal(changeItem *abstract.ChangeItem, snapshot bool) ([]byte, error) { fieldsVal, err := arrColSchemaToFieldsDescr(changeItem.TableSchema.Columns(), snapshot, m.connectorParameters) if err != nil { @@ -449,7 +449,7 @@ func (m *Emitter) ToKafkaPayloadVal(changeItem *abstract.ChangeItem, payloadTSMS return util.JSONMarshalUnescape(payloadObj) } -// valPayload - generate a payload for a val message +// valPayload - generate a payload for a val message. func (m *Emitter) valPayload(changeItem *abstract.ChangeItem, payloadTSMS time.Time, snapshot bool, emitType emitType) (map[string]interface{}, error) { op, err := kindToOp(changeItem.Kind, snapshot, emitType) if err != nil { @@ -622,7 +622,7 @@ func (m *Emitter) emitOneDebeziumMessage( return string(key), &valStr, nil } -// EmitKV - main exported method - generates kafka key & kafka value +// EmitKV - main exported method - generates kafka key & kafka value. func (m *Emitter) emitKV(changeItem *abstract.ChangeItem, payloadTSMS time.Time, snapshot bool, sessionPackers packer.SessionPackers) ([]debeziumcommon.KeyValue, error) { if changeItem.Kind != abstract.InsertKind && changeItem.Kind != abstract.UpdateKind && changeItem.Kind != abstract.DeleteKind { return []debeziumcommon.KeyValue{}, nil @@ -673,7 +673,7 @@ func (m *Emitter) emitKV(changeItem *abstract.ChangeItem, payloadTSMS time.Time, return []debeziumcommon.KeyValue{{DebeziumKey: key, DebeziumVal: val}}, nil } -// EmitKV - main exported method - generates kafka key & kafka value +// EmitKV - main exported method - generates kafka key & kafka value. func (m *Emitter) EmitKV(changeItem *abstract.ChangeItem, payloadTSMS time.Time, snapshot bool, sessionPackers packer.SessionPackers) ([]debeziumcommon.KeyValue, error) { result, err := m.emitKV(changeItem, payloadTSMS, snapshot, sessionPackers) if err != nil { diff --git a/pkg/debezium/packer/lightning_cache/lightning_cache.go b/pkg/debezium/packer/lightning_cache/lightning_cache.go index 7f94998a..161cd40b 100644 --- a/pkg/debezium/packer/lightning_cache/lightning_cache.go +++ b/pkg/debezium/packer/lightning_cache/lightning_cache.go @@ -58,7 +58,7 @@ func handleChangeItems(input []abstract.ChangeItem, schemaIDCache *SessionPacker } // NewLightningCache -// after NewLightningCache there are all schemaID/finalSchemas should be known +// after NewLightningCache there are all schemaID/finalSchemas should be known. func NewLightningCache(emitter *debezium.Emitter, input []abstract.ChangeItem, isSnapshot bool) (packer.SessionPackers, error) { schemaIDCache, err := NewSessionPackersSchemaIDCache(emitter.GetPackers()) if err != nil { diff --git a/pkg/debezium/packer/packer.go b/pkg/debezium/packer/packer.go index c23590f2..69b3f31e 100644 --- a/pkg/debezium/packer/packer.go +++ b/pkg/debezium/packer/packer.go @@ -8,7 +8,7 @@ type BuilderFunc = func(changeItem *abstract.ChangeItem) ([]byte, error) // SchemaIDResolver // If there are packer with schemaRegistry and wrapped into schemaID resolver -// It would be called ResolveSchemaID+PackWithSchemaID except 'Pack' +// It would be called ResolveSchemaID+PackWithSchemaID except 'Pack'. type SchemaIDResolver interface { ResolveSchemaID(schema []byte, table abstract.TableID) (uint32, error) PackWithSchemaID(schemaID uint32, payload []byte) ([]byte, error) diff --git a/pkg/debezium/parameters/parameters.go b/pkg/debezium/parameters/parameters.go index 5808fc32..e8830b1e 100644 --- a/pkg/debezium/parameters/parameters.go +++ b/pkg/debezium/parameters/parameters.go @@ -13,7 +13,7 @@ import ( // Parameters names. const ( - // automatically filled settings, user can override it, if user wants: + // automatically filled settings, user can override it, if user wants:. DatabaseDBName = "database.dbname" // "db" in debezium payload - it's just field in "source". Present in pg-connector, absent in mysql-connector TopicPrefix = "topic.prefix" // "name" in debezium payload - it's prefix for topic_name & it's field "name" in "source" (used to be called "db.server.name", but was renamed) @@ -25,7 +25,7 @@ const ( BatchingMaxSize = "dt.batching.max.size" WriteIntoOneFullTopicName = "dt.write.into.one.topic" - // other settings: + // other settings:. TimePrecisionMode = "time.precision.mode" DecimalHandlingMode = "decimal.handling.mode" diff --git a/pkg/debezium/pg/emitter.go b/pkg/debezium/pg/emitter.go index 734d893f..3eb49b6c 100644 --- a/pkg/debezium/pg/emitter.go +++ b/pkg/debezium/pg/emitter.go @@ -486,7 +486,7 @@ func AddPg(v *debeziumcommon.Values, colSchema *abstract.ColSchema, colName stri case `pg:citext`: v.AddVal(colName, colVal.(string)) case `pg:hstore`: - result := "" + var result string var err error switch t := colVal.(type) { case map[string]interface{}: diff --git a/pkg/debezium/pg/tests/emitter_chain_test.go b/pkg/debezium/pg/tests/emitter_chain_test.go index db0d3c51..92566e41 100644 --- a/pkg/debezium/pg/tests/emitter_chain_test.go +++ b/pkg/debezium/pg/tests/emitter_chain_test.go @@ -73,7 +73,7 @@ func runTwoConversions(t *testing.T, pgSnapshotChangeItem []byte, isWipeOriginal // - debeziumMsg -> // - changeItem (without original_type_info) -> // - debeziumMsg -> -// - changeItem +// - changeItem. func TestEmitterCommonWithWipe(t *testing.T) { pgSnapshotChangeItem, err := os.ReadFile(yatest.SourcePath("transfer_manager/go/pkg/debezium/pg/tests/testdata/emitter_chain_test__canon_change_item_original.txt")) require.NoError(t, err) @@ -95,7 +95,7 @@ func TestEmitterCommonWithWipe(t *testing.T) { // - debeziumMsg -> // - changeItem (with original_type_info) -> // - debeziumMsg -> -// - changeItem +// - changeItem. func TestEmitterCommonWithoutWipe(t *testing.T) { pgSnapshotChangeItem, err := os.ReadFile(yatest.SourcePath("transfer_manager/go/pkg/debezium/pg/tests/testdata/emitter_chain_test__canon_change_item_original.txt")) require.NoError(t, err) diff --git a/pkg/debezium/pg/tests/receiver_test.go b/pkg/debezium/pg/tests/receiver_test.go index 57286db4..b1801881 100644 --- a/pkg/debezium/pg/tests/receiver_test.go +++ b/pkg/debezium/pg/tests/receiver_test.go @@ -94,7 +94,7 @@ func fixTableName(in string) string { //--------------------------------------------------------------------------------------------------------------------- // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val bigint); -// INSERT INTO public.basic_types (id, val) VALUES (1, 2); +// INSERT INTO public.basic_types (id, val) VALUES (1, 2);. var debeziumMsg00 = `{"payload":{"after":{"id":1,"val":2},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":23725240,"name":"fullfillment","schema":"public","sequence":"[\\"23725184\\",\\"23725240\\"]","snapshot":"false","table":"basic_types","ts_ms":1643471295802,"txId":555,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643471296262},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int64"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int64"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem00 = `{"id":555,"nextlsn":23725240,"commitTime":1643471295802000000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types00","columnnames":["id","val"],"columnvalues":[1,2],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"int64","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bigint"}],"oldkeys":{},"tx_id":"","query":""}` @@ -108,7 +108,7 @@ func TestReceive00(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val bit(1)); -// INSERT INTO public.basic_types (id, val) VALUES (1, b'1'); +// INSERT INTO public.basic_types (id, val) VALUES (1, b'1');. var debeziumMsg01 = `{"payload":{"after":{"id":1,"val":true},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":23737744,"name":"fullfillment","schema":"public","sequence":"[\\"23737688\\",\\"23737744\\"]","snapshot":"false","table":"basic_types","ts_ms":1643471367220,"txId":558,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643471367288},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"boolean"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"boolean"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem01 = `{"id":558,"nextlsn":24522216,"commitTime":1643471788895334000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types01","columnnames":["id","val"],"columnvalues":[1,"1"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"string","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bit(1)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -122,7 +122,7 @@ func TestReceive01(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val bit(8)); -// INSERT INTO public.basic_types (id, val) VALUES (1, b'10111011'); +// INSERT INTO public.basic_types (id, val) VALUES (1, b'10111011');. var debeziumMsg02 = `{"payload":{"after":{"id":1,"val":"uw=="},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":23868184,"name":"fullfillment","schema":"public","sequence":"[\"23868184\",\"23868184\"]","snapshot":"false","table":"basic_types","ts_ms":1643471395280,"txId":561,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643471395355},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"8"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"8"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem02 = `{"id":561,"nextlsn":24522216,"commitTime":1643471788895579000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types02","columnnames":["id","val"],"columnvalues":[1,"10111011"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"string","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bit(8)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -136,7 +136,7 @@ func TestReceive02(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val bit varying(8)); -// INSERT INTO public.basic_types (id, val) VALUES (1, b'10111011'); +// INSERT INTO public.basic_types (id, val) VALUES (1, b'10111011');. var debeziumMsg03 = `{"payload":{"after":{"id":1,"val":"uw=="},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24088720,"name":"fullfillment","schema":"public","sequence":"[\"24088720\",\"24088720\"]","snapshot":"false","table":"basic_types","ts_ms":1643633778837,"txId":564,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643633779486},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"8"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"8"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem03 = `{"id":564,"nextlsn":24531320,"commitTime":1643633963155601000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types03","columnnames":["id","val"],"columnvalues":[1,"10111011"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"string","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bit varying(8)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -150,7 +150,7 @@ func TestReceive03(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val boolean); -// INSERT INTO public.basic_types (id, val) VALUES (1, true); +// INSERT INTO public.basic_types (id, val) VALUES (1, true);. var debeziumMsg04 = `{"payload":{"after":{"id":1,"val":true},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24209576,"name":"fullfillment","schema":"public","sequence":"[\"24209520\",\"24209576\"]","snapshot":"false","table":"basic_types","ts_ms":1643634511251,"txId":567,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643634511784},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"boolean"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"boolean"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem04 = `{"id":567,"nextlsn":24540344,"commitTime":1643634681324924000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types04","columnnames":["id","val"],"columnvalues":[1,true],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"boolean","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:boolean"}],"oldkeys":{},"tx_id":"","query":""}` @@ -164,7 +164,7 @@ func TestReceive04(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val bytea); -// INSERT INTO public.basic_types (id, val) VALUES (1, decode('CAFEBABE', 'hex')); +// INSERT INTO public.basic_types (id, val) VALUES (1, decode('CAFEBABE', 'hex'));. var debeziumMsg05 = `{"payload":{"after":{"id":1,"val":"yv66vg=="},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24358936,"name":"fullfillment","schema":"public","sequence":"[\"24358936\",\"24358936\"]","snapshot":"false","table":"basic_types","ts_ms":1643635175600,"txId":570,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643635176057},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"bytes"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"bytes"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem05 = `{"id":570,"nextlsn":24572984,"commitTime":1643634881019849000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types05","columnnames":["id","val"],"columnvalues":[1,"yv66vg=="],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"string","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bytea"}],"oldkeys":{},"tx_id":"","query":""}` @@ -178,7 +178,7 @@ func TestReceive05(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val character(1)); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'z'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'z');. var debeziumMsg06 = `{"payload":{"after":{"id":1,"val":"z"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24483600,"name":"fullfillment","schema":"public","sequence":"[\"24483544\",\"24483600\"]","snapshot":"false","table":"basic_types","ts_ms":1643635383640,"txId":573,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643635384053},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem06 = `{"id":573,"nextlsn":24582008,"commitTime":1643635514020408000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types06","columnnames":["id","val"],"columnvalues":[1,"z"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:character(1)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -192,7 +192,7 @@ func TestReceive06(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val character(4)); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'abcd'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'abcd');. var debeziumMsg07 = `{"payload":{"after":{"id":1,"val":"abcd"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24594536,"name":"fullfillment","schema":"public","sequence":"[\"24594536\",\"24594536\"]","snapshot":"false","table":"basic_types","ts_ms":1643636799976,"txId":576,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643636800108},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem07 = `{"id":576,"nextlsn":24591032,"commitTime":1643636872675869000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types07","columnnames":["id","val"],"columnvalues":[1,"abcd"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:character(4)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -206,7 +206,7 @@ func TestReceive07(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val character varying(256)); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'blablabla'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'blablabla');. var debeziumMsg08 = `{"payload":{"after":{"id":1,"val":"blablabla"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24708568,"name":"fullfillment","schema":"public","sequence":"[\"24708512\",\"24708568\"]","snapshot":"false","table":"basic_types","ts_ms":1643637018349,"txId":579,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643637018651},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem08 = `{"id":579,"nextlsn":24600872,"commitTime":1643637188079327000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types08","columnnames":["id","val"],"columnvalues":[1,"blablabla"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:character varying(256)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -220,7 +220,7 @@ func TestReceive08(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val cidr); -// INSERT INTO public.basic_types (id, val) VALUES (1, '10.1/16'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '10.1/16');. var debeziumMsg09 = `{"payload":{"after":{"id":1,"val":"10.1.0.0/16"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24852568,"name":"fullfillment","schema":"public","sequence":"[\"24852512\",\"24852568\"]","snapshot":"false","table":"basic_types","ts_ms":1643637458256,"txId":582,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643637458555},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem09 = `{"id":582,"nextlsn":24618528,"commitTime":1643637543329859000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types09","columnnames":["id","val"],"columnvalues":[1,"10.1.0.0/16"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:cidr"}],"oldkeys":{},"tx_id":"","query":""}` @@ -234,7 +234,7 @@ func TestReceive09(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val date); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'January 8, 1999'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'January 8, 1999');. var debeziumMsg10 = `{"payload":{"after":{"id":1,"val":10599},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24975832,"name":"fullfillment","schema":"public","sequence":"[\"24975832\",\"24975832\"]","snapshot":"false","table":"basic_types","ts_ms":1643637772033,"txId":585,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643637772336},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Date","optional":true,"type":"int32","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem10 = `{"id":585,"nextlsn":24627552,"commitTime":1643659128505565000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types10","columnnames":["id","val"],"columnvalues":[1,"1999-01-08T00:00:00Z"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:date"}],"oldkeys":{},"tx_id":"","query":""}` @@ -248,7 +248,7 @@ func TestReceive10(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val double precision); -// INSERT INTO public.basic_types (id, val) VALUES (1, 3.14e-100); +// INSERT INTO public.basic_types (id, val) VALUES (1, 3.14e-100);. var debeziumMsg11 = `{"payload":{"after":{"id":1,"val":3.14e-100},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":25116240,"name":"fullfillment","schema":"public","sequence":"[\"25116184\",\"25116240\"]","snapshot":"false","table":"basic_types","ts_ms":1643660790574,"txId":588,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643660790733},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"double"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"double"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem11 = `{"id":588,"nextlsn":25051056,"commitTime":1643660670442569000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types11","columnnames":["id","val"],"columnvalues":[1,3.14e-100],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"double","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:double precision"}],"oldkeys":{},"tx_id":"","query":""}` @@ -262,7 +262,7 @@ func TestReceive11(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val inet); -// INSERT INTO public.basic_types (id, val) VALUES (1, '192.168.1.5'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '192.168.1.5');. var debeziumMsg12 = `{"payload":{"after":{"id":1,"val":"192.168.1.5"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":25264200,"name":"fullfillment","schema":"public","sequence":"[\"25264200\",\"25264200\"]","snapshot":"false","table":"basic_types","ts_ms":1643661524350,"txId":591,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643661524595},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem12 = `{"id":591,"nextlsn":25051056,"commitTime":1643660670210670000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types12","columnnames":["id","val"],"columnvalues":[1,"192.168.1.5/32"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:inet"}],"oldkeys":{},"tx_id":"","query":""}` @@ -276,7 +276,7 @@ func TestReceive12(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val int4range); -// INSERT INTO public.basic_types (id, val) VALUES (1, '[3,7)'::int4range); +// INSERT INTO public.basic_types (id, val) VALUES (1, '[3,7)'::int4range);. var debeziumMsg13 = `{"payload":{"after":{"id":1,"val":"[3,7)"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":25412936,"name":"fullfillment","schema":"public","sequence":"[\"25412936\",\"25412936\"]","snapshot":"false","table":"basic_types","ts_ms":1643666022708,"txId":594,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643666022893},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem13 = `{"id":594,"nextlsn":25051056,"commitTime":1643660670248166000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types13","columnnames":["id","val"],"columnvalues":[1,"[3,7)"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:int4range"}],"oldkeys":{},"tx_id":"","query":""}` @@ -290,7 +290,7 @@ func TestReceive13(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val int8range); -// INSERT INTO public.basic_types (id, val) VALUES (1, '[3,7)'::int8range); +// INSERT INTO public.basic_types (id, val) VALUES (1, '[3,7)'::int8range);. var debeziumMsg14 = `{"payload":{"after":{"id":1,"val":"[3,7)"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":25644480,"name":"fullfillment","schema":"public","sequence":"[\"25644480\",\"25644480\"]","snapshot":"false","table":"basic_types","ts_ms":1643666270935,"txId":598,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643666271277},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem14 = `{"id":598,"nextlsn":25051056,"commitTime":1643660670399582000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types14","columnnames":["id","val"],"columnvalues":[1,"[3,7)"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:int8range"}],"oldkeys":{},"tx_id":"","query":""}` @@ -304,7 +304,7 @@ func TestReceive14(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val integer); -// INSERT INTO public.basic_types (id, val) VALUES (1, -8388605); +// INSERT INTO public.basic_types (id, val) VALUES (1, -8388605);. var debeziumMsg15 = `{"payload":{"after":{"id":1,"val":-8388605},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":25770128,"name":"fullfillment","schema":"public","sequence":"[\"25770072\",\"25770128\"]","snapshot":"false","table":"basic_types","ts_ms":1643666479618,"txId":601,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643666480074},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int32"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int32"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem15 = `{"id":601,"nextlsn":25051056,"commitTime":1643660670333075000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types15","columnnames":["id","val"],"columnvalues":[1,-8388605],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"int32","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"}],"oldkeys":{},"tx_id":"","query":""}` @@ -318,7 +318,7 @@ func TestReceive15(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val interval); -// INSERT INTO public.basic_types (id, val) VALUES (1, interval '1 day 01:00:00'); +// INSERT INTO public.basic_types (id, val) VALUES (1, interval '1 day 01:00:00');. var debeziumMsg16 = `{"payload":{"after":{"id":1,"val":90000000000},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":25868768,"name":"fullfillment","schema":"public","sequence":"[\"25868768\",\"25868768\"]","snapshot":"false","table":"basic_types","ts_ms":1643666651700,"txId":605,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643666651858},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroDuration","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroDuration","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem16 = `{"id":605,"nextlsn":25051056,"commitTime":1643660670310867000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types16","columnnames":["id","val"],"columnvalues":[1,"1 day 01:00:00.000000"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:interval"}],"oldkeys":{},"tx_id":"","query":""}` @@ -332,7 +332,7 @@ func TestReceive16(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val json); -// INSERT INTO public.basic_types (id, val) VALUES (1, '{"k1": "v1"}'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '{"k1": "v1"}');. var debeziumMsg17 = `{"payload":{"after":{"id":1,"val":"{\"k1\":\"v1\"}"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26009584,"name":"fullfillment","schema":"public","sequence":"[\"26009584\",\"26009584\"]","snapshot":"false","table":"basic_types","ts_ms":1643669861140,"txId":610,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643669861322},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Json","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Json","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem17 = `{"id":610,"nextlsn":25051056,"commitTime":1643660670260338000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types17","columnnames":["id","val"],"columnvalues":[1,{"k1":"v1"}],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:json"}],"oldkeys":{},"tx_id":"","query":""}` @@ -346,7 +346,7 @@ func TestReceive17(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val jsonb); -// INSERT INTO public.basic_types (id, val) VALUES (1, '{"k2": "v2"}'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '{"k2": "v2"}');. var debeziumMsg18 = `{"payload":{"after":{"id":1,"val":"{\"k2\":\"v2\"}"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26194920,"name":"fullfillment","schema":"public","sequence":"[\"26194920\",\"26194920\"]","snapshot":"false","table":"basic_types","ts_ms":1643670300335,"txId":616,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643670300736},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Json","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Json","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem18 = `{"id":616,"nextlsn":25051056,"commitTime":1643660670307562000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types18","columnnames":["id","val"],"columnvalues":[1,{"k2":"v2"}],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:jsonb"}],"oldkeys":{},"tx_id":"","query":""}` @@ -360,7 +360,7 @@ func TestReceive18(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val macaddr); -// INSERT INTO public.basic_types (id, val) VALUES (1, '08:00:2b:01:02:03'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '08:00:2b:01:02:03');. var debeziumMsg19 = `{"payload":{"after":{"id":1,"val":"08:00:2b:01:02:03"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26373168,"name":"fullfillment","schema":"public","sequence":"[\"26373168\",\"26373168\"]","snapshot":"false","table":"basic_types","ts_ms":1643670468486,"txId":620,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643670468566},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem19 = `{"id":620,"nextlsn":25051056,"commitTime":1643660670399509000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types19","columnnames":["id","val"],"columnvalues":[1,"08:00:2b:01:02:03"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:macaddr"}],"oldkeys":{},"tx_id":"","query":""}` @@ -374,7 +374,7 @@ func TestReceive19(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val money); -// INSERT INTO public.basic_types (id, val) VALUES (1, 99.98); +// INSERT INTO public.basic_types (id, val) VALUES (1, 99.98);. var debeziumMsg20 = `{"payload":{"after":{"id":1,"val":"Jw4="},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26393328,"name":"fullfillment","schema":"public","sequence":"[\"26393272\",\"26393328\"]","snapshot":"false","table":"basic_types","ts_ms":1643670557369,"txId":623,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643670557677},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"scale":"2"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"scale":"2"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem20 = `{"id":623,"nextlsn":25051056,"commitTime":1643660670347491000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types20","columnnames":["id","val"],"columnvalues":[1,"$99.98"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:money"}],"oldkeys":{},"tx_id":"","query":""}` @@ -388,7 +388,7 @@ func TestReceive20(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val numeric(18,2)); -// INSERT INTO public.basic_types (id, val) VALUES (1, -100.00); +// INSERT INTO public.basic_types (id, val) VALUES (1, -100.00);. var debeziumMsg202 = `{"payload":{"after":{"id":1,"val":"2PA="},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26393328,"name":"fullfillment","schema":"public","sequence":"[\"26393272\",\"26393328\"]","snapshot":"false","table":"basic_types","ts_ms":1643670557369,"txId":623,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643670557677},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"18","scale":"2"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"18","scale":"2"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem202 = `{"id":623,"nextlsn":25051056,"commitTime":1643660670347491000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types20","columnnames":["id","val"],"columnvalues":[1,-100.00],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"double","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric(18,2)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -402,7 +402,7 @@ func TestReceive202(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val numeric); -// INSERT INTO public.basic_types (id, val) VALUES (1, 1267650600228229401496703205376); +// INSERT INTO public.basic_types (id, val) VALUES (1, 1267650600228229401496703205376);. var debeziumMsg21 = `{"payload":{"after":{"id":1,"val":{"scale":0,"value":"EAAAAAAAAAAAAAAAAA=="}},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26550024,"name":"fullfillment","schema":"public","sequence":"[\"26549968\",\"26550024\"]","snapshot":"false","table":"basic_types","ts_ms":1643736695780,"txId":626,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643736696145},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"doc":"Variable scaled decimal","field":"val","fields":[{"field":"scale","optional":false,"type":"int32"},{"field":"value","optional":false,"type":"bytes"}],"name":"io.debezium.data.VariableScaleDecimal","optional":true,"type":"struct","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"doc":"Variable scaled decimal","field":"val","fields":[{"field":"scale","optional":false,"type":"int32"},{"field":"value","optional":false,"type":"bytes"}],"name":"io.debezium.data.VariableScaleDecimal","optional":true,"type":"struct","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem21 = `{"id":626,"nextlsn":25051056,"commitTime":1643749932407187000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types21","columnnames":["id","val"],"columnvalues":[1,1267650600228229401496703205376e0],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"double","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric"}],"oldkeys":{},"tx_id":"","query":""}` @@ -416,7 +416,7 @@ func TestReceive21(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val numrange); -// INSERT INTO public.basic_types (id, val) VALUES (1, numrange(1.9,1.91)); +// INSERT INTO public.basic_types (id, val) VALUES (1, numrange(1.9,1.91));. var debeziumMsg22 = `{"payload":{"after":{"id":1,"val":"[1.9,1.91)"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26697384,"name":"fullfillment","schema":"public","sequence":"[\"26697328\",\"26697384\"]","snapshot":"false","table":"basic_types","ts_ms":1643750902810,"txId":629,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643750903030},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem22 = `{"id":629,"nextlsn":25051056,"commitTime":1643749932437964000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types22","columnnames":["id","val"],"columnvalues":[1,"[19e-1,191e-2)"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numrange"}],"oldkeys":{},"tx_id":"","query":""}` @@ -430,7 +430,7 @@ func TestReceive22(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val oid); -// INSERT INTO public.basic_types (id, val) VALUES (1, 2); +// INSERT INTO public.basic_types (id, val) VALUES (1, 2);. var debeziumMsg23 = `{"payload":{"after":{"id":1,"val":2},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":26925136,"name":"fullfillment","schema":"public","sequence":"[\"26925136\",\"26925136\"]","snapshot":"false","table":"basic_types","ts_ms":1643752629674,"txId":634,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643752629815},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int64"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int64"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem23 = `{"id":634,"nextlsn":25051056,"commitTime":1643752954350432000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types23","columnnames":["id","val"],"columnvalues":[1,2],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"int32","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:oid"}],"oldkeys":{},"tx_id":"","query":""}` @@ -460,7 +460,7 @@ func TestReceive24(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val citext); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'Tom'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'Tom');. var debeziumMsg25 = `{"payload":{"after":{"id":1,"val":"Tom"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":27710672,"name":"fullfillment","schema":"public","sequence":"[\"27710672\",\"27710672\"]","snapshot":"false","table":"basic_types","ts_ms":1643754482020,"txId":642,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643754482604},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem25 = `{"id":642,"nextlsn":25051056,"commitTime":1643752954420386000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types25","columnnames":["id","val"],"columnvalues":[1,"Tom"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:citext"}],"oldkeys":{},"tx_id":"","query":""}` @@ -474,7 +474,7 @@ func TestReceive25(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val hstore); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'a=>1,b=>2'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'a=>1,b=>2');. var debeziumMsg26 = `{"payload":{"after":{"id":1,"val":"{\"a\":\"1\",\"b\":\"2\"}"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":28360384,"name":"fullfillment","schema":"public","sequence":"[\"28360384\",\"28360384\"]","snapshot":"false","table":"basic_types","ts_ms":1643754993258,"txId":647,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643754993415},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Json","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Json","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem26 = `{"id":647,"nextlsn":25051056,"commitTime":1643752954509425000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types26","columnnames":["id","val"],"columnvalues":[1,{"a":"1","b":"2"}],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:hstore"}],"oldkeys":{},"tx_id":"","query":""}` @@ -504,7 +504,7 @@ func TestReceive27(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val smallint); -// INSERT INTO public.basic_types (id, val) VALUES (1, -32768); +// INSERT INTO public.basic_types (id, val) VALUES (1, -32768);. var debeziumMsg28 = `{"payload":{"after":{"id":1,"val":-32768},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":28702048,"name":"fullfillment","schema":"public","sequence":"[\"28701992\",\"28702048\"]","snapshot":"false","table":"basic_types","ts_ms":1643755963173,"txId":654,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643755963263},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int16"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"int16"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem28 = `{"id":654,"nextlsn":25051056,"commitTime":1643752954579894000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types28","columnnames":["id","val"],"columnvalues":[1,-32768],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"int16","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:smallint"}],"oldkeys":{},"tx_id":"","query":""}` @@ -518,7 +518,7 @@ func TestReceive28(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val text); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'text_example'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'text_example');. var debeziumMsg29 = `{"payload":{"after":{"id":1,"val":"text_example"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":28856736,"name":"fullfillment","schema":"public","sequence":"[\"28856680\",\"28856736\"]","snapshot":"false","table":"basic_types","ts_ms":1643756263096,"txId":657,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643756263293},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem29 = `{"id":657,"nextlsn":25051056,"commitTime":1643752954596446000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types29","columnnames":["id","val"],"columnvalues":[1,"text_example"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:text"}],"oldkeys":{},"tx_id":"","query":""}` @@ -532,7 +532,7 @@ func TestReceive29(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54');. var debeziumMsg30 = `{"payload":{"after":{"id":1,"val":1098181434000000},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":29001064,"name":"fullfillment","schema":"public","sequence":"[\"29001008\",\"29001064\"]","snapshot":"false","table":"basic_types","ts_ms":1643756441744,"txId":660,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643756441920},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem30 = `{"id":660,"nextlsn":24668192,"commitTime":1674573416677871000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types00","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types00","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types00","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -563,7 +563,7 @@ func TestReceive31(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val time without time zone); -// INSERT INTO public.basic_types (id, val) VALUES (1, '04:05:06'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '04:05:06');. var debeziumMsg32 = `{"payload":{"after":{"id":1,"val":14706000000},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":29296040,"name":"fullfillment","schema":"public","sequence":"[\"29296040\",\"29296040\"]","snapshot":"false","table":"basic_types","ts_ms":1643758220207,"txId":666,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643758220366},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem32 = `{"id":666,"nextlsn":24586096,"commitTime":1674763072222677000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types","part":"","columnnames":["id","val"],"columnvalues":[1,"04:05:06"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -593,7 +593,7 @@ func TestReceive33(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val tsrange); -// INSERT INTO public.basic_types (id, val) VALUES (1, '[2010-01-02 10:00, 2010-01-02 11:00)'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '[2010-01-02 10:00, 2010-01-02 11:00)');. var debeziumMsg34 = `{"payload":{"after":{"id":1,"val":"[\"2010-01-02 10:00:00\",\"2010-01-02 11:00:00\")"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":29895648,"name":"fullfillment","schema":"public","sequence":"[\"29895648\",\"29895648\"]","snapshot":"false","table":"basic_types","ts_ms":1643760870144,"txId":680,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643760870376},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem34 = `{"id":680,"nextlsn":25051056,"commitTime":1643752954611655000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types34","columnnames":["id","val"],"columnvalues":[1,"[2010-01-02 10:00:00,2010-01-02 11:00:00)"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:tsrange"}],"oldkeys":{},"tx_id":"","query":""}` @@ -607,7 +607,7 @@ func TestReceive34(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val tstzrange); -// INSERT INTO public.basic_types (id, val) VALUES (1, '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange); +// INSERT INTO public.basic_types (id, val) VALUES (1, '[2010-01-01 01:00:00 -05, 2010-01-01 02:00:00 -08)'::tstzrange);. var debeziumMsg35 = `{"payload":{"after":{"id":1,"val":"[\"2010-01-01 06:00:00+00\",\"2010-01-01 10:00:00+00\")"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":30048312,"name":"fullfillment","schema":"public","sequence":"[\"30048312\",\"30048312\"]","snapshot":"false","table":"basic_types","ts_ms":1643763067972,"txId":683,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643763068317},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","optional":true,"type":"string"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem35 = `{"id":683,"nextlsn":25051056,"commitTime":1643752954463913000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types35","columnnames":["id","val"],"columnvalues":[1,"[2010-01-01 06:00:00Z,2010-01-01 10:00:00Z)"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:tstzrange"}],"oldkeys":{},"tx_id":"","query":""}` @@ -621,7 +621,7 @@ func TestReceive35(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val uuid); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11');. var debeziumMsg36 = `{"payload":{"after":{"id":1,"val":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":29607560,"name":"fullfillment","schema":"public","sequence":"[\"29607504\",\"29607560\"]","snapshot":"false","table":"basic_types","ts_ms":1643759997449,"txId":672,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643759997643},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Uuid","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Uuid","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem36 = `{"id":672,"nextlsn":25051056,"commitTime":1643752954296104000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types36","columnnames":["id","val"],"columnvalues":[1,"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:uuid"}],"oldkeys":{},"tx_id":"","query":""}` @@ -635,7 +635,7 @@ func TestReceive36(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val xml); -// INSERT INTO public.basic_types (id, val) VALUES (1, 'bar'); +// INSERT INTO public.basic_types (id, val) VALUES (1, 'bar');. var debeziumMsg37 = `{"payload":{"after":{"id":1,"val":"bar"},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":29462048,"name":"fullfillment","schema":"public","sequence":"[\"29462048\",\"29462048\"]","snapshot":"false","table":"basic_types","ts_ms":1643759636016,"txId":669,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643759636085},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Xml","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Xml","optional":true,"type":"string","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem37 = `{"id":669,"nextlsn":25051056,"commitTime":1643752954400380000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types37","columnnames":["id","val"],"columnvalues":[1,"\u003cfoo\u003ebar\u003c/foo\u003e"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:xml"}],"oldkeys":{},"tx_id":"","query":""}` @@ -649,7 +649,7 @@ func TestReceive37(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIME(1)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '04:05:06.1'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '04:05:06.1');. var debeziumMsg38 = `{"payload":{"after":{"id":1,"val":14706100},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":23831768,"name":"fullfillment","schema":"public","sequence":"[\"23831768\",\"23831768\"]","snapshot":"false","table":"basic_types","ts_ms":1644239926590,"txId":558,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644239926751},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Time","optional":true,"type":"int32","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Time","optional":true,"type":"int32","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem38 = `{"id":558,"nextlsn":24586096,"commitTime":1674764123764984000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types","part":"","columnnames":["id","val"],"columnvalues":[1,"04:05:06.1"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time(1) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -663,7 +663,7 @@ func TestReceive38(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIME(6)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '04:05:06.123456'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '04:05:06.123456');. var debeziumMsg38_2 = `{"payload":{"after":{"id":1,"val":14706123456},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":34795416,"name":"fullfillment","schema":"public","snapshot":"false","table":"basic_types","ts_ms":1674769011570,"txId":766,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1674769011873},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem38_2 = `{"id":766,"nextlsn":24586096,"commitTime":1674768915799271000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types33","part":"","columnnames":["id","val"],"columnvalues":[1,"04:05:06.123456"],"table_schema":[{"table_schema":"public","table_name":"basic_types33","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types33","path":"","name":"val","type":"utf8","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time(6) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -677,7 +677,7 @@ func TestReceive38_2(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(1)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.9'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.9');. var debeziumMsg39 = `{"payload":{"after":{"id":1,"val":1098181434900},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":23730168,"name":"fullfillment","schema":"public","sequence":"[\"23730168\",\"23730168\"]","snapshot":"false","table":"basic_types","ts_ms":1644243907787,"txId":555,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644243908374},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem39 = `{"id":555,"nextlsn":24586096,"commitTime":1674765249216244000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types12","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.9+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types12","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types12","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(1) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -694,7 +694,7 @@ func TestReceive39(t *testing.T) { //--------------------------------------------------------------------------------------------------------------------- // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(1)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.9'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.9');. var debeziumMsg41 = `{"payload":{"after":{"id":1,"val":1098181434900},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24158968,"name":"fullfillment","schema":"public","sequence":"[\"24158912\",\"24158968\"]","snapshot":"false","table":"basic_types","ts_ms":1644247943535,"txId":579,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644247943618},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem41 = `{"id":579,"nextlsn":24586096,"commitTime":1674766325363996000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types13","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.9+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types13","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types13","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(1) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -709,7 +709,7 @@ func TestReceive41(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(2)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.98'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.98');. var debeziumMsg42 = `{"payload":{"after":{"id":1,"val":1098181434980},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24171376,"name":"fullfillment","schema":"public","sequence":"[\"24171376\",\"24171376\"]","snapshot":"false","table":"basic_types","ts_ms":1644247978813,"txId":582,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644247979016},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem42 = `{"id":582,"nextlsn":24586096,"commitTime":1674766436356239000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types14","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.98+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types14","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types14","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(2) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -724,7 +724,7 @@ func TestReceive42(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(3)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.987'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.987');. var debeziumMsg43 = `{"payload":{"after":{"id":1,"val":1098181434987},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24183824,"name":"fullfillment","schema":"public","sequence":"[\"24183824\",\"24183824\"]","snapshot":"false","table":"basic_types","ts_ms":1644248007860,"txId":585,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644248008123},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.Timestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem43 = `{"id":585,"nextlsn":24586096,"commitTime":1674766569036602000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types15","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.987+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types15","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types15","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(3) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -739,7 +739,7 @@ func TestReceive43(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(4)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.9'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.9');. var debeziumMsg44 = `{"payload":{"after":{"id":1,"val":1098181434987600},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24196216,"name":"fullfillment","schema":"public","sequence":"[\"24196160\",\"24196216\"]","snapshot":"false","table":"basic_types","ts_ms":1644248032513,"txId":588,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644248032750},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem44 = `{"id":588,"nextlsn":24586096,"commitTime":1674767015484937000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types20","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.9+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types20","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types20","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(4) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -754,7 +754,7 @@ func TestReceive44(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(5)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.98765'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.98765');. var debeziumMsg45 = `{"payload":{"after":{"id":1,"val":1098181434987650},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24208672,"name":"fullfillment","schema":"public","sequence":"[\"24208616\",\"24208672\"]","snapshot":"false","table":"basic_types","ts_ms":1644248057696,"txId":591,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644248057828},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem45 = `{"id":591,"nextlsn":24586096,"commitTime":1674767128031202000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types21","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.98765+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types21","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types21","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(5) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -769,7 +769,7 @@ func TestReceive45(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val TIMESTAMP(6)); -// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.987654'); +// INSERT INTO public.basic_types (id, val) VALUES (1, '2004-10-19 10:23:54.987654');. var debeziumMsg46 = `{"payload":{"after":{"id":1,"val":1098181434987654},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24221064,"name":"fullfillment","schema":"public","sequence":"[\"24221008\",\"24221064\"]","snapshot":"false","table":"basic_types","ts_ms":1644248084027,"txId":594,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1644248084436},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem46 = `{"id":594,"nextlsn":24586096,"commitTime":1674767224153149000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types22","part":"","columnnames":["id","val"],"columnvalues":[1,"2004-10-19T10:23:54.987654+04:00"],"table_schema":[{"table_schema":"public","table_name":"basic_types22","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":true,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types22","path":"","name":"val","type":"timestamp","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(6) without time zone"}],"oldkeys":{},"tx_id":"","query":""}` @@ -784,7 +784,7 @@ func TestReceive46(t *testing.T) { } // CREATE TABLE public.basic_types (id INT PRIMARY KEY, val bit varying(16)); -// INSERT INTO public.basic_types (id, val) VALUES (1, b'1111111100000000'); +// INSERT INTO public.basic_types (id, val) VALUES (1, b'1111111100000000');. var debeziumMsg47 = `{"payload":{"after":{"id":1,"val":"AP8="},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24088720,"name":"fullfillment","schema":"public","sequence":"[\"24088720\",\"24088720\"]","snapshot":"false","table":"basic_types","ts_ms":1643633778837,"txId":564,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":1643633779486},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":false,"type":"int32"},{"field":"val","name":"io.debezium.data.Bits","optional":true,"parameters":{"length":"16"},"type":"bytes","version":1}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false,incremental"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"sequence","optional":true,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItem47 = `{"id":564,"nextlsn":24531320,"commitTime":1643633963155601000,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types03","columnnames":["id","val"],"columnvalues":[1,"1111111100000000"],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"id","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"val","type":"string","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bit varying(16)"}],"oldkeys":{},"tx_id":"","query":""}` @@ -799,7 +799,7 @@ func TestReceive47(t *testing.T) { //--- -// create/insert from tests/e2e/pg2pg/all_datatypes_serde_via_debezium_arr +// create/insert from tests/e2e/pg2pg/all_datatypes_serde_via_debezium_arr. var debeziumMsgArr = `{"payload":{"after":{"arr_bl":[true,true],"arr_c":["1","1"],"arr_character_varying_":["varc","varc"],"arr_d":[3.14e-100,3.14e-100],"arr_date_":[10599,10599],"arr_decimal_":[{"scale":0,"value":"AeJA"},{"scale":0,"value":"AeJA"}],"arr_decimal_5":["MDk=","MDk="],"arr_decimal_5_2":["ME8=","ME8="],"arr_f":[1.45e-10,1.45e-10],"arr_i":[1,1],"arr_id":[1,2],"arr_int":[1,2],"arr_it":["192.168.100.128/25","192.168.100.128/25"],"arr_numeric_":[{"scale":0,"value":"EAAAAAAAAAAAAAAAAA=="},{"scale":14,"value":"EAAAAAAAAAAAAAAAAA=="}],"arr_numeric_5":["MDk=","MDk="],"arr_numeric_5_2":["ME8=","ME8="],"arr_oid_":[1,2],"arr_real_":[1.45e-10,1.45e-10],"arr_si":[1,2],"arr_str":["varchar_example","varchar_example"],"arr_t":["text_example","text_example"],"arr_time1":[14706100000,14706100000],"arr_time6":[14706123000,14706123000],"arr_time_":[14706000000,14706000000],"arr_time_with_time_zone_":["08:51:02Z","08:51:02Z"],"arr_timestamp":[1098181434000000,1098181434000000],"arr_timestamp1":[1098181434900000,1098181434900000],"arr_timestamp6":[1098181434987654,1098181434987654],"arr_timestamptz_":["2004-10-19T08:23:54Z","2004-10-19T08:23:54Z"],"arr_timetz1":["17:30:25Z","17:30:25Z"],"arr_timetz6":["17:30:25Z","17:30:25Z"],"arr_timetz_":["08:51:02Z","08:51:02Z"],"arr_timetz__":["17:30:25Z","17:30:25Z"],"arr_tst":["2004-10-19T09:23:54Z","2004-10-19T09:23:54Z"],"arr_uid":["a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"],"i":1},"before":null,"op":"c","source":{"connector":"postgresql","db":"pguser","lsn":24544072,"name":"fullfillment","schema":"public","snapshot":"false","table":"basic_types","ts_ms":1652025148443,"txId":0,"version":"1.8.0.Final","xmin":null},"transaction":null,"ts_ms":0},"schema":{"fields":[{"field":"before","fields":[{"field":"i","optional":false,"type":"int32"},{"field":"arr_bl","items":{"optional":true,"type":"boolean"},"optional":true,"type":"array"},{"field":"arr_si","items":{"optional":true,"type":"int16"},"optional":true,"type":"array"},{"field":"arr_int","items":{"optional":true,"type":"int32"},"optional":true,"type":"array"},{"field":"arr_id","items":{"optional":true,"type":"int64"},"optional":true,"type":"array"},{"field":"arr_oid_","items":{"optional":true,"type":"int64"},"optional":true,"type":"array"},{"field":"arr_real_","items":{"optional":true,"type":"float"},"optional":true,"type":"array"},{"field":"arr_d","items":{"optional":true,"type":"double"},"optional":true,"type":"array"},{"field":"arr_c","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_str","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_character_varying_","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_timestamptz_","items":{"name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_tst","items":{"name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timetz_","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_time_with_time_zone_","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_uid","items":{"name":"io.debezium.data.Uuid","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_it","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_f","items":{"optional":true,"type":"double"},"optional":true,"type":"array"},{"field":"arr_i","items":{"optional":true,"type":"int32"},"optional":true,"type":"array"},{"field":"arr_t","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_date_","items":{"name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},"optional":true,"type":"array"},{"field":"arr_time_","items":{"name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_time1","items":{"name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_time6","items":{"name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_timetz__","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timetz1","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timetz6","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timestamp1","items":{"name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_timestamp6","items":{"name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_timestamp","items":{"name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_numeric_","items":{"doc":"Variable scaled decimal","fields":[{"field":"scale","optional":false,"type":"int32"},{"field":"value","optional":false,"type":"bytes"}],"name":"io.debezium.data.VariableScaleDecimal","optional":true,"type":"struct","version":1},"optional":true,"type":"array"},{"field":"arr_numeric_5","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"0"},"type":"bytes","version":1},"optional":true,"type":"array"},{"field":"arr_numeric_5_2","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"2"},"type":"bytes","version":1},"optional":true,"type":"array"},{"field":"arr_decimal_","items":{"doc":"Variable scaled decimal","fields":[{"field":"scale","optional":false,"type":"int32"},{"field":"value","optional":false,"type":"bytes"}],"name":"io.debezium.data.VariableScaleDecimal","optional":true,"type":"struct","version":1},"optional":true,"type":"array"},{"field":"arr_decimal_5","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"0"},"type":"bytes","version":1},"optional":true,"type":"array"},{"field":"arr_decimal_5_2","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"2"},"type":"bytes","version":1},"optional":true,"type":"array"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"i","optional":false,"type":"int32"},{"field":"arr_bl","items":{"optional":true,"type":"boolean"},"optional":true,"type":"array"},{"field":"arr_si","items":{"optional":true,"type":"int16"},"optional":true,"type":"array"},{"field":"arr_int","items":{"optional":true,"type":"int32"},"optional":true,"type":"array"},{"field":"arr_id","items":{"optional":true,"type":"int64"},"optional":true,"type":"array"},{"field":"arr_oid_","items":{"optional":true,"type":"int64"},"optional":true,"type":"array"},{"field":"arr_real_","items":{"optional":true,"type":"float"},"optional":true,"type":"array"},{"field":"arr_d","items":{"optional":true,"type":"double"},"optional":true,"type":"array"},{"field":"arr_c","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_str","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_character_varying_","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_timestamptz_","items":{"name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_tst","items":{"name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timetz_","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_time_with_time_zone_","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_uid","items":{"name":"io.debezium.data.Uuid","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_it","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_f","items":{"optional":true,"type":"double"},"optional":true,"type":"array"},{"field":"arr_i","items":{"optional":true,"type":"int32"},"optional":true,"type":"array"},{"field":"arr_t","items":{"optional":true,"type":"string"},"optional":true,"type":"array"},{"field":"arr_date_","items":{"name":"io.debezium.time.Date","optional":true,"type":"int32","version":1},"optional":true,"type":"array"},{"field":"arr_time_","items":{"name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_time1","items":{"name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_time6","items":{"name":"io.debezium.time.MicroTime","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_timetz__","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timetz1","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timetz6","items":{"name":"io.debezium.time.ZonedTime","optional":true,"type":"string","version":1},"optional":true,"type":"array"},{"field":"arr_timestamp1","items":{"name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_timestamp6","items":{"name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_timestamp","items":{"name":"io.debezium.time.MicroTimestamp","optional":true,"type":"int64","version":1},"optional":true,"type":"array"},{"field":"arr_numeric_","items":{"doc":"Variable scaled decimal","fields":[{"field":"scale","optional":false,"type":"int32"},{"field":"value","optional":false,"type":"bytes"}],"name":"io.debezium.data.VariableScaleDecimal","optional":true,"type":"struct","version":1},"optional":true,"type":"array"},{"field":"arr_numeric_5","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"0"},"type":"bytes","version":1},"optional":true,"type":"array"},{"field":"arr_numeric_5_2","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"2"},"type":"bytes","version":1},"optional":true,"type":"array"},{"field":"arr_decimal_","items":{"doc":"Variable scaled decimal","fields":[{"field":"scale","optional":false,"type":"int32"},{"field":"value","optional":false,"type":"bytes"}],"name":"io.debezium.data.VariableScaleDecimal","optional":true,"type":"struct","version":1},"optional":true,"type":"array"},{"field":"arr_decimal_5","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"0"},"type":"bytes","version":1},"optional":true,"type":"array"},{"field":"arr_decimal_5_2","items":{"name":"org.apache.kafka.connect.data.Decimal","optional":true,"parameters":{"connect.decimal.precision":"5","scale":"2"},"type":"bytes","version":1},"optional":true,"type":"array"}],"name":"fullfillment.public.basic_types.Value","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"version","optional":false,"type":"string"},{"field":"connector","optional":false,"type":"string"},{"field":"name","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"},{"default":"false","field":"snapshot","name":"io.debezium.data.Enum","optional":true,"parameters":{"allowed":"true,last,false"},"type":"string","version":1},{"field":"db","optional":false,"type":"string"},{"field":"table","optional":false,"type":"string"},{"field":"lsn","optional":true,"type":"int64"},{"field":"schema","optional":false,"type":"string"},{"field":"txId","optional":true,"type":"int64"},{"field":"xmin","optional":true,"type":"int64"}],"name":"io.debezium.connector.postgresql.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":true,"type":"int64"},{"field":"transaction","fields":[{"field":"id","optional":false,"type":"string"},{"field":"total_order","optional":false,"type":"int64"},{"field":"data_collection_order","optional":false,"type":"int64"}],"optional":true,"type":"struct"}],"name":"fullfillment.public.basic_types.Envelope","optional":false,"type":"struct"}}` var canonChangeItemArr = `{"id":0,"nextlsn":0,"commitTime":0,"txPosition":0,"kind":"insert","schema":"public","table":"basic_types","columnnames":["i","arr_bl","arr_si","arr_int","arr_id","arr_oid_","arr_real_","arr_d","arr_c","arr_str","arr_character_varying_","arr_timestamptz_","arr_tst","arr_timetz_","arr_time_with_time_zone_","arr_uid","arr_it","arr_f","arr_i","arr_t","arr_date_","arr_time_","arr_time1","arr_time6","arr_timetz__","arr_timetz1","arr_timetz6","arr_timestamp1","arr_timestamp6","arr_timestamp","arr_numeric_","arr_numeric_5","arr_numeric_5_2","arr_decimal_","arr_decimal_5","arr_decimal_5_2"],"columnvalues":[1,[true,true],[1,2],[1,2],[1,2],[1,2],[1.45e-10,1.45e-10],[3.14e-100,3.14e-100],["1","1"],["varchar_example","varchar_example"],["varc","varc"],["2004-10-19T08:23:54Z","2004-10-19T08:23:54Z"],["2004-10-19T09:23:54Z","2004-10-19T09:23:54Z"],["08:51:02Z","08:51:02Z"],["08:51:02Z","08:51:02Z"],["a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"],["192.168.100.128/25","192.168.100.128/25"],[1.45e-10,1.45e-10],[1,1],["text_example","text_example"],["1999-01-08T00:00:00Z","1999-01-08T00:00:00Z"],["04:05:06.000000","04:05:06.000000"],["04:05:06.100000","04:05:06.100000"],["04:05:06.123000","04:05:06.123000"],["17:30:25Z","17:30:25Z"],["17:30:25Z","17:30:25Z"],["17:30:25Z","17:30:25Z"],["2004-10-19T10:23:54.900000Z","2004-10-19T10:23:54.900000Z"],["2004-10-19T10:23:54.987654Z","2004-10-19T10:23:54.987654Z"],["2004-10-19T10:23:54.000000Z","2004-10-19T10:23:54.000000Z"],[1267650600228229401496703205376e0,12676506002282294.01496703205376e0],["12345","12345"],["123.67","123.67"],[123456e0,123456e0],["12345","12345"],["123.67","123.67"]],"table_schema":[{"table_schema":"public","table_name":"basic_types","path":"","name":"i","type":"int32","key":true,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_bl","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:boolean[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_si","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:smallint[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_int","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_id","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:bigint[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_oid_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:oid[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_real_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:real[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_d","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:double precision[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_c","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:character(1)[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_str","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:character varying(256)[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_character_varying_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:character varying(5)[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timestamptz_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_tst","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timetz_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_time_with_time_zone_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_uid","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:uuid[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_it","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:inet[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_f","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:double precision[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_i","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:integer[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_t","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:text[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_date_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:date[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_time_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time without time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_time1","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time(1) without time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_time6","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time(6) without time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timetz__","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timetz1","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time(1) with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timetz6","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:time(6) with time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timestamp1","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(1) without time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timestamp6","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp(6) without time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_timestamp","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:timestamp without time zone[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_numeric_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_numeric_5","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric(5,0)[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_numeric_5_2","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric(5,2)[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_decimal_","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_decimal_5","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric(5,0)[]"},{"table_schema":"public","table_name":"basic_types","path":"","name":"arr_decimal_5_2","type":"any","key":false,"fake_key":false,"required":false,"expression":"","original_type":"pg:numeric(5,2)[]"}],"oldkeys":{},"tx_id":"","query":""}` diff --git a/pkg/debezium/typeutil/helpers.go b/pkg/debezium/typeutil/helpers.go index 6322110c..693c6048 100644 --- a/pkg/debezium/typeutil/helpers.go +++ b/pkg/debezium/typeutil/helpers.go @@ -17,29 +17,6 @@ import ( "github.com/doublecloud/transfer/pkg/util" ) -//--------------------------------------------------------------------------------------------------------------------- -// it's fixed in higher version of debezium (in 1.1 this bug is present, in 1.8 absent) -// so, actual function - changeItemsBitsToDebeziumHonest - -func changeItemsBitsToDebeziumWA(bits string) string { - bufSize := imitateDebeziumBufSize(len(bits)) - return changeItemsBitsStringToDebezium(bits, bufSize) -} - -func imitateDebeziumBufSize(bitsCount int) int { - if bitsCount < 16 { - return 2 - } else if bitsCount < 32 { - return 4 - } else if bitsCount < 64 { - return 8 - } else { - return int(math.Ceil(float64(bitsCount) / 8)) // honest count - } -} - -//--------------------------------------------------------------------------------------------------------------------- - func ChangeItemsBitsToDebeziumHonest(bits string) string { honestBufSize := int(math.Ceil(float64(len(bits)) / 8)) return changeItemsBitsStringToDebezium(bits, honestBufSize) @@ -62,7 +39,7 @@ func changeItemsBitsStringToDebezium(bits string, bufSize int) string { bitNum := 7 - (i % 8) if val == '1' { foundOne = true - buf[byteNum] = buf[byteNum] | (1 << bitNum) + buf[byteNum] |= (1 << bitNum) } } if !foundOne { @@ -73,7 +50,7 @@ func changeItemsBitsStringToDebezium(bits string, bufSize int) string { } func GetBitLength(pgType string) (string, error) { - result := "" + var result string if strings.HasPrefix(pgType, "pg:bit(") { rightIndex := strings.Index(pgType[7:], ")") if rightIndex == -1 { @@ -116,10 +93,12 @@ func GetTimeDivider(originalTypeWithoutProvider string) (int, error) { } } -var reTimeWithoutTZ = regexp.MustCompile(`^time\((\d)\) without time zone`) -var reTimestampWithoutTZ = regexp.MustCompile(`^timestamp\((\d)\) without time zone`) -var reMysqlTime = regexp.MustCompile(`^mysql:timestamp\((\d)\)`) -var reMysqlDatetime = regexp.MustCompile(`^mysql:datetime\((\d)\)`) +var ( + reTimeWithoutTZ = regexp.MustCompile(`^time\((\d)\) without time zone`) + reTimestampWithoutTZ = regexp.MustCompile(`^timestamp\((\d)\) without time zone`) + reMysqlTime = regexp.MustCompile(`^mysql:timestamp\((\d)\)`) + reMysqlDatetime = regexp.MustCompile(`^mysql:datetime\((\d)\)`) +) func GetTimePrecision(colTypeStr string) int { precision := -1 @@ -237,7 +216,7 @@ func MysqlDecimalFloatToValue(val float64, colType string) (string, error) { } arr[0] = arr[0][len(arr[0])-integerLen:] for i := len(arr[1]); i < scale; i++ { - arr[1] = arr[1] + "0" + arr[1] += "0" } arr[1] = arr[1][0:scale] return arr[0] + arr[1], nil @@ -447,10 +426,12 @@ func DecimalToDebeziumPrimitives(decimal string, connectorParameters map[string] } } -var pgTimestampLayout0 = "2006-01-02T15:04:05Z" -var pgTimestampLayout1 = "2006-01-02 15:04:05Z" -var pgTimestampLayout2 = "2006-01-02T15:04:05-07:00" -var pgTimestampLayout3 = "2006-01-02 15:04:05-07" +var ( + pgTimestampLayout0 = "2006-01-02T15:04:05Z" + pgTimestampLayout1 = "2006-01-02 15:04:05Z" + pgTimestampLayout2 = "2006-01-02T15:04:05-07:00" + pgTimestampLayout3 = "2006-01-02 15:04:05-07" +) func ParsePgDateTimeWithTimezone(in string) (time.Time, error) { var result time.Time @@ -664,10 +645,12 @@ func ParsePgDateTimeWithTimezone2(l, r string) (time.Time, time.Time, error) { return lTime, rTime, nil } -var regexHour = *regexp.MustCompile(`(\d+):(\d+):(\d+)`) -var regexYear = *regexp.MustCompile(`(\d+) years?`) -var regexMonth = *regexp.MustCompile(`(\d+) (?:mon|months?)`) -var regexDay = *regexp.MustCompile(`(\d+) days?`) +var ( + regexHour = *regexp.MustCompile(`(\d+):(\d+):(\d+)`) + regexYear = *regexp.MustCompile(`(\d+) years?`) + regexMonth = *regexp.MustCompile(`(\d+) (?:mon|months?)`) + regexDay = *regexp.MustCompile(`(\d+) days?`) +) func ExtractPostgresIntervalArray(interval string) ([]string, error) { result := make([]string, 7) @@ -706,13 +689,15 @@ func ExtractPostgresIntervalArray(interval string) ([]string, error) { return result, nil } -var timeWithoutTZ0 = "15:04:05" -var timeWithoutTZ1 = "15:04:05.0" -var timeWithoutTZ2 = "15:04:05.00" -var timeWithoutTZ3 = "15:04:05.000" -var timeWithoutTZ4 = "15:04:05.0000" -var timeWithoutTZ5 = "15:04:05.00000" -var timeWithoutTZ6 = "15:04:05.000000" +var ( + timeWithoutTZ0 = "15:04:05" + timeWithoutTZ1 = "15:04:05.0" + timeWithoutTZ2 = "15:04:05.00" + timeWithoutTZ3 = "15:04:05.000" + timeWithoutTZ4 = "15:04:05.0000" + timeWithoutTZ5 = "15:04:05.00000" + timeWithoutTZ6 = "15:04:05.000000" +) func ParseTimeWithoutTZ(timeStr string) (time.Time, error) { var layout string @@ -742,13 +727,15 @@ func ParseTimeWithoutTZ(timeStr string) (time.Time, error) { return timeVal, nil } -var timestampWithoutTZ0 = "2006-01-02T15:04:05Z" -var timestampWithoutTZ1 = "2006-01-02T15:04:05.0Z" -var timestampWithoutTZ2 = "2006-01-02T15:04:05.00Z" -var timestampWithoutTZ3 = "2006-01-02T15:04:05.000Z" -var timestampWithoutTZ4 = "2006-01-02T15:04:05.0000Z" -var timestampWithoutTZ5 = "2006-01-02T15:04:05.00000Z" -var timestampWithoutTZ6 = "2006-01-02T15:04:05.000000Z" +var ( + timestampWithoutTZ0 = "2006-01-02T15:04:05Z" + timestampWithoutTZ1 = "2006-01-02T15:04:05.0Z" + timestampWithoutTZ2 = "2006-01-02T15:04:05.00Z" + timestampWithoutTZ3 = "2006-01-02T15:04:05.000Z" + timestampWithoutTZ4 = "2006-01-02T15:04:05.0000Z" + timestampWithoutTZ5 = "2006-01-02T15:04:05.00000Z" + timestampWithoutTZ6 = "2006-01-02T15:04:05.000000Z" +) func ParseTimestamp(timeStr string) (time.Time, error) { var layout string @@ -832,24 +819,26 @@ func BufToChangeItemsBits(in []byte) string { return result } -var yearMS = int64(31557600000000) -var monthMS = int64(2629800000000) -var dayMS = int64(3600 * 24 * 1000 * 1000) -var hourMS = int64(3600 * 1000 * 1000) -var minuteMS = int64(60 * 1000 * 1000) -var secondMS = int64(1000 * 1000) +var ( + yearMS = int64(31557600000000) + monthMS = int64(2629800000000) + dayMS = int64(3600 * 24 * 1000 * 1000) + hourMS = int64(3600 * 1000 * 1000) + minuteMS = int64(60 * 1000 * 1000) + secondMS = int64(1000 * 1000) +) func EmitPostgresInterval(val int64) string { y := val / yearMS least := val - y*yearMS month := least / monthMS - least = least - month*monthMS + least -= month * monthMS d := least / dayMS - least = least - d*dayMS + least -= d * dayMS h := least / hourMS - least = least - h*hourMS + least -= h * hourMS min := least / minuteMS - least = least - min*minuteMS + least -= min * minuteMS s := least / secondMS ms := least - s*secondMS @@ -1043,7 +1032,7 @@ func ShrinkMysqlBit(colVal interface{}, colType string) ([]uint8, error) { var reParametrizedType = regexp.MustCompile(`^.*\((\d)\).*`) -// MysqlFitBinaryLength - used for types: "mysql:binary", "mysql:varbinary", "mysql:longblob", "mysql:mediumblob", "mysql:blob", "mysql:tinyblob" +// MysqlFitBinaryLength - used for types: "mysql:binary", "mysql:varbinary", "mysql:longblob", "mysql:mediumblob", "mysql:blob", "mysql:tinyblob". func MysqlFitBinaryLength(colType string, colVal interface{}) (interface{}, error) { reMatch := reParametrizedType.FindStringSubmatch(colType) if len(reMatch) == 2 { @@ -1106,8 +1095,10 @@ func OriginalTypeWithoutProvider(originalType string) string { return originalType[index+1:] } -var pgTimeWithoutTimeZoneParam = *regexp.MustCompile(`pg:time\((\d)\) without time zone`) -var pgNumeric = *regexp.MustCompile(`pg:numeric\(\d+,\d+\)`) +var ( + pgTimeWithoutTimeZoneParam = *regexp.MustCompile(`pg:time\((\d)\) without time zone`) + pgNumeric = *regexp.MustCompile(`pg:numeric\(\d+,\d+\)`) +) func PgTimeWithoutTimeZonePrecision(originalType string) int { if originalType == "pg:time without time zone" { diff --git a/pkg/debezium/typeutil/helpers_test.go b/pkg/debezium/typeutil/helpers_test.go index 611e4d3d..4797183e 100644 --- a/pkg/debezium/typeutil/helpers_test.go +++ b/pkg/debezium/typeutil/helpers_test.go @@ -419,11 +419,11 @@ func TestDateTest(t *testing.T) { }) t.Run("datetime", func(t *testing.T) { date := time.Date(2020, 2, 2, 0, 0, 2, 0, time.UTC) - require.Equal(t, date, TimeFromDatetime(int64(DatetimeToSecs(date)))) + require.Equal(t, date, TimeFromDatetime(DatetimeToSecs(date))) }) t.Run("timestamp", func(t *testing.T) { date := time.Date(2020, 2, 2, 0, 0, 0, 2000, time.UTC) - require.Equal(t, date, TimeFromTimestamp(int64(DatetimeToMicrosecs(date)))) + require.Equal(t, date, TimeFromTimestamp(DatetimeToMicrosecs(date))) }) } diff --git a/pkg/errors/categories/category.go b/pkg/errors/categories/category.go index bf5d02e0..e03601ad 100644 --- a/pkg/errors/categories/category.go +++ b/pkg/errors/categories/category.go @@ -13,6 +13,6 @@ const ( Source Category = "source" // Target is for errors originating from the target endpoint (its configuration, runtime properties, etc.) Target Category = "target" - // Internal is for errors caused by the Transfer service's infrastructure and unfixable by the user + // Internal is for errors caused by the Transfer service's infrastructure and unfixable by the user. Internal Category = "internal" ) diff --git a/pkg/errors/categorized.go b/pkg/errors/categorized.go index c77503d5..cc85e22c 100644 --- a/pkg/errors/categorized.go +++ b/pkg/errors/categorized.go @@ -5,7 +5,7 @@ import ( "github.com/doublecloud/transfer/pkg/errors/categories" ) -// Categorized is an error with an attached category +// Categorized is an error with an attached category. type Categorized interface { error xerrors.Wrapper @@ -18,7 +18,7 @@ type categorizedImpl struct { category categories.Category } -// CategorizedErrorf produces a xerrors-wrapped error with a given assigned category +// CategorizedErrorf produces a xerrors-wrapped error with a given assigned category. func CategorizedErrorf(category categories.Category, format string, a ...any) error { errorf := xerrors.Errorf(format, a...) var categorized Categorized = nil diff --git a/pkg/errors/coded/error.go b/pkg/errors/coded/error.go index 91a3e565..df53ff63 100644 --- a/pkg/errors/coded/error.go +++ b/pkg/errors/coded/error.go @@ -4,7 +4,7 @@ import ( "github.com/doublecloud/transfer/library/go/core/xerrors" ) -// CodedError is an error with an attached code +// CodedError is an error with an attached code. type CodedError interface { error // Code is an uniq identifier of some specific error that can be referred from documentation or code / sdk. @@ -16,7 +16,7 @@ type codedImpl struct { code Code } -// CategorizedErrorf produces a xerrors-wrapped error with a given assigned category +// CategorizedErrorf produces a xerrors-wrapped error with a given assigned category. func Errorf(code Code, format string, a ...any) CodedError { return &codedImpl{ error: xerrors.Errorf(format, a...), diff --git a/pkg/errors/coded/registry.go b/pkg/errors/coded/registry.go index a45bd19c..741edeea 100644 --- a/pkg/errors/coded/registry.go +++ b/pkg/errors/coded/registry.go @@ -9,7 +9,7 @@ import ( ) // Code define provider defined stable code. Each provider has own code-registry, but we have global registry to dedup them -// in case we have duplicate we will panic at start +// in case we have duplicate we will panic at start. type Code string func (c Code) ID() string { diff --git a/pkg/functions/cloud_functions.go b/pkg/functions/cloud_functions.go index 638f5372..95807f4c 100644 --- a/pkg/functions/cloud_functions.go +++ b/pkg/functions/cloud_functions.go @@ -202,6 +202,7 @@ func (e *Executor) Do(data []abstract.ChangeItem) ([]abstract.ChangeItem, error) e.logger.Error("failed to invoke function", log.Error(err)) return xerrors.Errorf("failed to invoke function: %w", err) } + defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { var errorResp interface{} if err = json.NewDecoder(resp.Body).Decode(&errorResp); err == nil { @@ -298,7 +299,7 @@ func (e *Executor) Do(data []abstract.ChangeItem) ([]abstract.ChangeItem, error) ColumnNames: data[i].ColumnNames, ColumnValues: data[i].ColumnValues, TableSchema: data[i].TableSchema, - OldKeys: *new(abstract.OldKeysType), + OldKeys: abstract.OldKeysType{}, TxID: "", Query: "", Size: data[i].Size, diff --git a/pkg/instanceutil/metadata_service.go b/pkg/instanceutil/metadata_service.go index 02957f07..87f6b132 100644 --- a/pkg/instanceutil/metadata_service.go +++ b/pkg/instanceutil/metadata_service.go @@ -51,7 +51,7 @@ func GetIdentityDocument() (string, error) { func makeGoogleMetadataRequest(param GoogleCEMetaDataParam, recursive bool) (*http.Request, error) { url := fmt.Sprintf("http://%v/computeMetadata/v1/instance/%v?recursive=%v", InstanceMetadataAddr, param, recursive) - request, err := http.NewRequest("GET", url, nil) + request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err } @@ -91,7 +91,7 @@ const ( func GetAmazonEC2MetaData(param AmazonEC2MetaDataParam) (string, error) { url := fmt.Sprintf("http://%v/latest/meta-data/%v", InstanceMetadataAddr, param) - request, err := http.NewRequest("GET", url, nil) + request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return "", xerrors.Errorf("cannot get metadata: %w", err) } @@ -101,7 +101,7 @@ func GetAmazonEC2MetaData(param AmazonEC2MetaDataParam) (string, error) { func GetAmazonEC2UserData(out interface{}) error { // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-add-user-data.html url := fmt.Sprintf("http://%v/latest/user-data", InstanceMetadataAddr) - request, err := http.NewRequest("GET", url, nil) + request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return xerrors.Errorf("cannot make request for user data: %w", err) } diff --git a/pkg/maplock/mutex.go b/pkg/maplock/mutex.go index 656df1df..af8e06a6 100644 --- a/pkg/maplock/mutex.go +++ b/pkg/maplock/mutex.go @@ -7,7 +7,7 @@ import ( ) // Mutex is the mutex with synchronized map -// it's for reducing unnecessary locks among different keys +// it's for reducing unnecessary locks among different keys. type Mutex struct { locks map[interface{}]interface{} m *sync.Mutex @@ -36,25 +36,25 @@ func (m *Mutex) TryLock(key interface{}) (gotLock bool) { } // Unlock unlocks for the key -// please call Unlock only after having aquired the lock +// please call Unlock only after having aquired the lock. func (m *Mutex) Unlock(key interface{}) { m.m.Lock() delete(m.locks, key) m.m.Unlock() } -// borrowed from grpc +// borrowed from grpc. func (m *Mutex) backoff(retries int) time.Duration { if retries == 0 { return time.Duration(m.baseDelay) * time.Nanosecond } - backoff, max := m.baseDelay, m.maxDelay - for backoff < max && retries > 0 { + backoff, maxDelay := m.baseDelay, m.maxDelay + for backoff < maxDelay && retries > 0 { backoff *= m.factor retries-- } - if backoff > max { - backoff = max + if backoff > maxDelay { + backoff = maxDelay } backoff *= 1 + m.jitter*(rand.Float64()*2-1) if backoff < 0 { @@ -63,12 +63,12 @@ func (m *Mutex) backoff(retries int) time.Duration { return time.Duration(backoff) * time.Nanosecond } -// NewMapMutex returns a mapmutex with default configs +// NewMapMutex returns a mapmutex with default configs. func NewMapMutex() *Mutex { return NewCustomizedMapMutex(200, 100000000, 10, 1.1, 0.2) } -// NewCustomizedMapMutex returns a customized mapmutex +// NewCustomizedMapMutex returns a customized mapmutex. func NewCustomizedMapMutex(mRetry int, mDelay, bDelay, factor, jitter float64) *Mutex { return &Mutex{ locks: make(map[interface{}]interface{}), diff --git a/pkg/maplock/mutex_test.go b/pkg/maplock/mutex_test.go index da12c025..60ec63e1 100644 --- a/pkg/maplock/mutex_test.go +++ b/pkg/maplock/mutex_test.go @@ -80,33 +80,33 @@ func BenchmarkMutex1000_100_20_20(b *testing.B) { lockByOneMutex(1000, 10 func BenchmarkMapWithMutex1000_100_20_20(b *testing.B) { lockByMapWithMutex(1000, 100, 20, 20) } func BenchmarkMapMutex1000_100_20_20(b *testing.B) { lockByMapMutex(1000, 100, 20, 20) } -// less key, more conflict for map key +// less key, more conflict for map key. func BenchmarkMutex1000_20_20_20(b *testing.B) { lockByOneMutex(1000, 20, 20, 20) } func BenchmarkMapWithMutex1000_20_20_20(b *testing.B) { lockByMapWithMutex(1000, 20, 20, 20) } func BenchmarkMapMutex1000_20_20_20(b *testing.B) { lockByMapMutex(1000, 20, 20, 20) } -// less key, more goroutine, more conflict for map key +// less key, more goroutine, more conflict for map key. func BenchmarkMutex1000_20_40_20(b *testing.B) { lockByOneMutex(1000, 20, 40, 20) } func BenchmarkMapWithMutex1000_20_40_20(b *testing.B) { lockByMapWithMutex(1000, 20, 40, 20) } func BenchmarkMapMutex1000_20_40_20(b *testing.B) { lockByMapMutex(1000, 20, 40, 20) } // even we want to use map to avoid unnecessary lock -// if case of only 2 entries, a lot of locking occurs +// if case of only 2 entries, a lot of locking occurs. func BenchmarkMutex1000_2_40_20(b *testing.B) { lockByOneMutex(1000, 2, 40, 20) } func BenchmarkMapWithMutex1000_2_40_20(b *testing.B) { lockByMapWithMutex(1000, 2, 40, 20) } func BenchmarkMapMutex1000_2_40_20(b *testing.B) { lockByMapMutex(1000, 2, 40, 20) } -// longer time per job, more conflict for map key +// longer time per job, more conflict for map key. func BenchmarkMutex1000_20_40_60(b *testing.B) { lockByOneMutex(1000, 20, 40, 60) } func BenchmarkMapWithMutex1000_20_40_60(b *testing.B) { lockByMapWithMutex(1000, 20, 40, 60) } func BenchmarkMapMutex1000_20_40_60(b *testing.B) { lockByMapMutex(1000, 20, 40, 60) } -// much more actions +// much more actions. func BenchmarkMutex10000_20_40_20(b *testing.B) { lockByOneMutex(10000, 20, 40, 20) } func BenchmarkMapWithMutex10000_20_40_20(b *testing.B) { lockByMapWithMutex(10000, 20, 40, 20) } func BenchmarkMapMutex10000_20_40_20(b *testing.B) { lockByMapMutex(10000, 20, 40, 20) } -// load should be larger than 0 +// load should be larger than 0. func splitLoad(load, buckets int) []int { result := make([]int, buckets) avg := load / buckets diff --git a/pkg/metering/initializer_os.go b/pkg/metering/initializer_os.go index 87841b9e..6d6d8191 100644 --- a/pkg/metering/initializer_os.go +++ b/pkg/metering/initializer_os.go @@ -15,7 +15,6 @@ func Agent() MeteringAgent { } func InitializeWithTags(transfer *model.Transfer, task *model.TransferOperation, runtimeTags map[string]interface{}) { - return } func WithAgent(agent MeteringAgent) MeteringAgent { diff --git a/pkg/metering/rows_metric.go b/pkg/metering/rows_metric.go index 044def33..f963048e 100644 --- a/pkg/metering/rows_metric.go +++ b/pkg/metering/rows_metric.go @@ -51,10 +51,6 @@ const ( bucketg10Mb = SizeBucket(10*1024*1024 + 1) ) -func (*RowsMetricState) isMetricState() {} - -func (*RowsMetric) isMetric() {} - func (rm *RowsMetric) Reset() *RowsMetricState { rm.statsMu.Lock() prevIntervalFinishTS := time.Now() @@ -292,7 +288,7 @@ func getPreview(baseOpts *MeteringOpts) bool { if tagPresent { return preview.(bool) } - //should never happen, but may for some freezed transfers + // should never happen, but may for some freezed transfers return true } diff --git a/pkg/middlewares/async/bufferer/bufferer.go b/pkg/middlewares/async/bufferer/bufferer.go index 0a78157e..31aa3c01 100644 --- a/pkg/middlewares/async/bufferer/bufferer.go +++ b/pkg/middlewares/async/bufferer/bufferer.go @@ -68,7 +68,7 @@ func newBufferer(sink abstract.Sinker, logger log.Logger, r metrics.Registry, tr return result } -// apiState contains the state connecting an asynchronously running goroutine which actually performs Push and callers of the middleware +// apiState contains the state connecting an asynchronously running goroutine which actually performs Push and callers of the middleware. type apiState struct { sync.RWMutex sync.WaitGroup @@ -138,7 +138,7 @@ func (b *bufferer) AsyncPush(items []abstract.ChangeItem) chan error { return result } -// bufferFlushState inherits sync.WaitGroup and contains all the information to do and control flush +// bufferFlushState inherits sync.WaitGroup and contains all the information to do and control flush. type buffererFlushState struct { sync.WaitGroup diff --git a/pkg/middlewares/async/measurer.go b/pkg/middlewares/async/measurer.go index ef8f2c96..d212977e 100644 --- a/pkg/middlewares/async/measurer.go +++ b/pkg/middlewares/async/measurer.go @@ -10,7 +10,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// Measurer calculate the size of items passing through it +// Measurer calculate the size of items passing through it. func Measurer(logger log.Logger) func(abstract.AsyncSink) abstract.AsyncSink { return func(s abstract.AsyncSink) abstract.AsyncSink { return newMeasurer(s, logger) diff --git a/pkg/middlewares/error_tracker.go b/pkg/middlewares/error_tracker.go index c060afb6..18131fa9 100644 --- a/pkg/middlewares/error_tracker.go +++ b/pkg/middlewares/error_tracker.go @@ -8,7 +8,7 @@ import ( "github.com/doublecloud/transfer/pkg/stats" ) -// ErrorTracker do nothing except tracking error / success pushes into metrics +// ErrorTracker do nothing except tracking error / success pushes into metrics. func ErrorTracker(mtrcs metrics.Registry) func(abstract.Sinker) abstract.Sinker { return func(s abstract.Sinker) abstract.Sinker { return newErrorTracker(s, mtrcs) diff --git a/pkg/middlewares/interval_throttler.go b/pkg/middlewares/interval_throttler.go index c89bca68..54d5e54d 100644 --- a/pkg/middlewares/interval_throttler.go +++ b/pkg/middlewares/interval_throttler.go @@ -7,7 +7,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// IntervalThrottler blocks Push until a given interval since the finish of the previous Push passes +// IntervalThrottler blocks Push until a given interval since the finish of the previous Push passes. func IntervalThrottler(logger log.Logger, interval time.Duration) func(abstract.Sinker) abstract.Sinker { return func(s abstract.Sinker) abstract.Sinker { return newIntervalThrottler(s, logger, interval) diff --git a/pkg/middlewares/memthrottle/middleware.go b/pkg/middlewares/memthrottle/middleware.go index 28f652f9..db260675 100644 --- a/pkg/middlewares/memthrottle/middleware.go +++ b/pkg/middlewares/memthrottle/middleware.go @@ -16,7 +16,7 @@ import ( // defaultMinReserve should cover Go runtime resources + all background tasks // like sending logs, metrics, upload state, etc. -// It may also help to survive spontaneous large batch from source +// It may also help to survive spontaneous large batch from source. const defaultMinReserve = 200 * humanize.MiByte type Config struct { @@ -28,7 +28,7 @@ type Config struct { // DefaultConfig makes default config for memory throttler algo // // totalBytes parameter should define total memory amount available -// to the process, not VM size or smth +// to the process, not VM size or smth. func DefaultConfig(totalBytes uint64) Config { return Config{ totalBytes: totalBytes, @@ -48,7 +48,7 @@ func DefaultConfig(totalBytes uint64) Config { // in single AsyncPush call. It's believed that all sources should produce reasonably small batches in future // // It use simple adaptive algorithm to determine and control flush threshold -// depending on current memory usage +// depending on current memory usage. func MemoryThrottler(cfg Config, lgr log.Logger) abstract.AsyncMiddleware { return func(sink abstract.AsyncSink) abstract.AsyncSink { return newMemoryThrottler(cfg, lgr, sink) diff --git a/pkg/middlewares/nonrow_separator.go b/pkg/middlewares/nonrow_separator.go index 8715ea35..9df26c97 100644 --- a/pkg/middlewares/nonrow_separator.go +++ b/pkg/middlewares/nonrow_separator.go @@ -5,7 +5,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// NonRowSeparator separates non-row items and pushes each of them in a distinct call to Push (which will contain just one item) +// NonRowSeparator separates non-row items and pushes each of them in a distinct call to Push (which will contain just one item). func NonRowSeparator() func(abstract.Sinker) abstract.Sinker { return func(s abstract.Sinker) abstract.Sinker { return newNonRowSeparator(s) diff --git a/pkg/middlewares/retrier.go b/pkg/middlewares/retrier.go index 1775bf12..3664ec3c 100644 --- a/pkg/middlewares/retrier.go +++ b/pkg/middlewares/retrier.go @@ -12,7 +12,7 @@ import ( ) // Retrier retries Push operations automatically using the hardcoded delay and interval. Retries can be interrupted using the given context. -// Push operations containing non-row items are NOT retried +// Push operations containing non-row items are NOT retried. func Retrier(logger log.Logger, ctx context.Context) func(abstract.Sinker) abstract.Sinker { return func(s abstract.Sinker) abstract.Sinker { return newRetrier(s, logger, ctx) diff --git a/pkg/middlewares/statistician.go b/pkg/middlewares/statistician.go index 7bc0136b..ad528c07 100644 --- a/pkg/middlewares/statistician.go +++ b/pkg/middlewares/statistician.go @@ -9,7 +9,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// Statistician tracks the traffic of sinker +// Statistician tracks the traffic of sinker. func Statistician(logger log.Logger, stats *stats.WrapperStats) func(abstract.Sinker) abstract.Sinker { return func(s abstract.Sinker) abstract.Sinker { return newStatistician(s, logger, stats) diff --git a/pkg/middlewares/table_temporator.go b/pkg/middlewares/table_temporator.go index 3b13cdd3..8067695f 100644 --- a/pkg/middlewares/table_temporator.go +++ b/pkg/middlewares/table_temporator.go @@ -10,7 +10,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// TableTemporator provides support for temporary policy +// TableTemporator provides support for temporary policy. func TableTemporator(logger log.Logger, transferID string, config model.TmpPolicyConfig) func(abstract.Movable) abstract.Sinker { return func(m abstract.Movable) abstract.Sinker { return newTemporator(m, logger, transferID, config) diff --git a/pkg/middlewares/type_strictness_tracker.go b/pkg/middlewares/type_strictness_tracker.go index c23d5bdd..bcf81290 100644 --- a/pkg/middlewares/type_strictness_tracker.go +++ b/pkg/middlewares/type_strictness_tracker.go @@ -70,7 +70,7 @@ func newStrictnessChecker() *strictnessChecker { } } -// HasNonStrictTypes checks whether an item contain types which do not fulfill the strictness guarantee +// HasNonStrictTypes checks whether an item contain types which do not fulfill the strictness guarantee. func (c *strictnessChecker) HasNonStrictTypes(item *abstract.ChangeItem) bool { tSchema, ok := c.tableSchemas[item.TableID()] if !ok { diff --git a/pkg/parsequeue/waitable_parsequeue.go b/pkg/parsequeue/waitable_parsequeue.go index df2d4272..b58447d5 100644 --- a/pkg/parsequeue/waitable_parsequeue.go +++ b/pkg/parsequeue/waitable_parsequeue.go @@ -21,7 +21,7 @@ func (p *WaitableParseQueue[TData]) Add(message TData) error { // Wait waits when all messages, added via .Add() will be acked // -// Should be called mutually exclusive with Add()/Close() +// Should be called mutually exclusive with Add()/Close(). func (p *WaitableParseQueue[TData]) Wait() { p.inflightWG.Wait() } diff --git a/pkg/parsers/abstract.go b/pkg/parsers/abstract.go index 7a3d8bc8..1ba2f221 100644 --- a/pkg/parsers/abstract.go +++ b/pkg/parsers/abstract.go @@ -6,7 +6,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// Message is struct describing incoming message +// Message is struct describing incoming message. type Message struct { // Offset is server sequence of message in topic. Must be monotone growing. Offset uint64 @@ -38,7 +38,7 @@ type Parser interface { } // WrappedParser parser can be layered by wrapping them in extra layers. -// For wrapped parsers we should add extra method for extracting actual parser +// For wrapped parsers we should add extra method for extracting actual parser. type WrappedParser interface { Parser Unwrap() Parser diff --git a/pkg/parsers/generic/generic_parser.go b/pkg/parsers/generic/generic_parser.go index ac7b6f6b..2282fa8b 100644 --- a/pkg/parsers/generic/generic_parser.go +++ b/pkg/parsers/generic/generic_parser.go @@ -790,7 +790,7 @@ func ElemIDX(columns []string) int { return len(columns) - 1 } -// can return in moment: valid value & error +// can return in moment: valid value & error. func (p *GenericParser) extractTimestamp(row map[string]interface{}, defaultTime time.Time) (time.Time, error) { if p.auxOpts.TimeField != nil && row[p.auxOpts.TimeField.Col] != nil { resultTime, done, err := p.extractTimeValue(row[p.auxOpts.TimeField.Col], defaultTime) diff --git a/pkg/parsers/registry/audittrailsv1/engine/parser.go b/pkg/parsers/registry/audittrailsv1/engine/parser.go index 5f59daa8..e029fb3b 100644 --- a/pkg/parsers/registry/audittrailsv1/engine/parser.go +++ b/pkg/parsers/registry/audittrailsv1/engine/parser.go @@ -240,7 +240,7 @@ func getNotElasticFields() []abstract.ColSchema { } } -// there are a lot of unused columns, which appear after renaming in `ingest_pipeline.json` +// there are a lot of unused columns, which appear after renaming in `ingest_pipeline.json`. func getElasticFields() *abstract.TableSchema { return abstract.NewTableSchema([]abstract.ColSchema{ makeColSchemaPK("event_id", "", "string"), diff --git a/pkg/parsers/registry/cloudevents/engine/protobuf.go b/pkg/parsers/registry/cloudevents/engine/protobuf.go index a127a631..b0a19bc7 100644 --- a/pkg/parsers/registry/cloudevents/engine/protobuf.go +++ b/pkg/parsers/registry/cloudevents/engine/protobuf.go @@ -5,7 +5,7 @@ import ( "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" "github.com/doublecloud/transfer/library/go/core/xerrors" - "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/proto" ) func unpackCloudEventsProtoMessage(buf []byte) (*cloudEventsProtoFields, []byte, string, error) { diff --git a/pkg/parsers/registry/cloudevents/engine/testutils/testutils.go b/pkg/parsers/registry/cloudevents/engine/testutils/testutils.go index a35becdd..c18ec43c 100644 --- a/pkg/parsers/registry/cloudevents/engine/testutils/testutils.go +++ b/pkg/parsers/registry/cloudevents/engine/testutils/testutils.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" ) func ChangeRegistryURL(t *testing.T, buf []byte, newURL string) []byte { diff --git a/pkg/parsers/registry/confluentschemaregistry/engine/parser.go b/pkg/parsers/registry/confluentschemaregistry/engine/parser.go index 754f55a1..067213f9 100644 --- a/pkg/parsers/registry/confluentschemaregistry/engine/parser.go +++ b/pkg/parsers/registry/confluentschemaregistry/engine/parser.go @@ -122,7 +122,7 @@ func (p *ConfluentSrImpl) DoBatch(batch parsers.MessageBatch) []abstract.ChangeI return result } -func NewConfluentSchemaRegistryImpl(srURL string, caCert string, username string, password string, SendSrNotFoundToUnparsed bool, logger log.Logger) *ConfluentSrImpl { +func NewConfluentSchemaRegistryImpl(srURL string, caCert string, username string, password string, sendSrNotFoundToUnparsed bool, logger log.Logger) *ConfluentSrImpl { client, err := confluent.NewSchemaRegistryClientWithTransport(srURL, caCert, logger) if err != nil { logger.Warnf("Unable to create schema registry client: %v", err) @@ -132,7 +132,7 @@ func NewConfluentSchemaRegistryImpl(srURL string, caCert string, username string return &ConfluentSrImpl{ logger: logger, SchemaRegistryClient: client, - SendSrNotFoundToUnparsed: SendSrNotFoundToUnparsed, + SendSrNotFoundToUnparsed: sendSrNotFoundToUnparsed, inMDBuilder: newMDBuilder(), } } diff --git a/pkg/parsers/registry/debezium/engine/parser.go b/pkg/parsers/registry/debezium/engine/parser.go index 9abbdb99..ab0d406b 100644 --- a/pkg/parsers/registry/debezium/engine/parser.go +++ b/pkg/parsers/registry/debezium/engine/parser.go @@ -26,7 +26,7 @@ type DebeziumImpl struct { // DoOne message with multiple debezium events inside. // Contains multiple debezium events only if messages are -// serialized using schema registry and started with magic zero-byte +// serialized using schema registry and started with magic zero-byte. func (p *DebeziumImpl) DoOne(partition abstract.Partition, buf []byte, offset uint64, writeTime time.Time) ([]byte, abstract.ChangeItem) { msgLen := len(buf) if len(buf) != 0 { diff --git a/pkg/parsers/registry/json/engine/fallback_timestamp_as_datetime.go b/pkg/parsers/registry/json/engine/fallback_timestamp_as_datetime.go index 7c4f35ff..3069d84e 100644 --- a/pkg/parsers/registry/json/engine/fallback_timestamp_as_datetime.go +++ b/pkg/parsers/registry/json/engine/fallback_timestamp_as_datetime.go @@ -12,7 +12,7 @@ import ( // isParsedItem - check if table is the result of generic-parser with opts.AddDedupeKeys columns // that means we have 4 our system columns in the end of ColumnNames/TableSchema: _timestamp/_partition/_offset/_idx // check if _timestamp is schema.TypeTimestamp - just in case. -// Actually type_system should guarantee it's always be called on old generic_parser, and always should be true +// Actually type_system should guarantee it's always be called on old generic_parser, and always should be true. func isParsedItem(ci *abstract.ChangeItem) bool { if len(ci.TableSchema.Columns()) <= 4 { return false @@ -31,7 +31,7 @@ func isParsedItem(ci *abstract.ChangeItem) bool { // isUnparsedItem - check if table is the result of 'unparsed' case of generic-parser // that means we have 6 our system columns: _timestamp/_partition/_offset/_idx/unparsed_row/reason // check if _timestamp is schema.TypeTimestamp - just in case. -// Actually type_system should guarantee it's always be called on old generic_parser, and always should be true +// Actually type_system should guarantee it's always be called on old generic_parser, and always should be true. func isUnparsedItem(ci *abstract.ChangeItem) bool { if len(ci.TableSchema.Columns()) != 6 { return false diff --git a/pkg/parsers/registry/protobuf/protoparser/proto_parser_config.go b/pkg/parsers/registry/protobuf/protoparser/proto_parser_config.go index d481e854..b9f3fdc9 100644 --- a/pkg/parsers/registry/protobuf/protoparser/proto_parser_config.go +++ b/pkg/parsers/registry/protobuf/protoparser/proto_parser_config.go @@ -52,7 +52,7 @@ type ProtoParserConfig struct { AddSyntheticKeys bool } -// SetDescriptors sets ProtoMessageDesc & ScannerMessageDesc +// SetDescriptors sets ProtoMessageDesc & ScannerMessageDesc. func (c *ProtoParserConfig) SetDescriptors(descFileContent []byte, messageName string, pkgType MessagePackageType) error { rootMsgDesc, err := extractMessageDesc(descFileContent, messageName) if err != nil { diff --git a/pkg/parsers/registry/protobuf/protoparser/proto_parser_test.go b/pkg/parsers/registry/protobuf/protoparser/proto_parser_test.go index 4d96b6d2..2659dbba 100644 --- a/pkg/parsers/registry/protobuf/protoparser/proto_parser_test.go +++ b/pkg/parsers/registry/protobuf/protoparser/proto_parser_test.go @@ -397,7 +397,7 @@ func TestDoStdDataTypesEqualValues(t *testing.T) { checkEqualStdValues(t, got[0], stdDataTypesFilled.ProtoReflect()) } -// not deep equal: don't check embedded complicated types +// not deep equal: don't check embedded complicated types. func checkEqualStdValues(t *testing.T, ci abstract.ChangeItem, msg protoreflect.Message) { fields := msg.Descriptor().Fields() diff --git a/pkg/parsers/registry/raw2table/engine/parser_test.go b/pkg/parsers/registry/raw2table/engine/parser_test.go index df0aa560..0d1e79ce 100644 --- a/pkg/parsers/registry/raw2table/engine/parser_test.go +++ b/pkg/parsers/registry/raw2table/engine/parser_test.go @@ -20,7 +20,7 @@ func TestDo(t *testing.T) { } stringAndByteData = append(stringAndByteData, 0xC0, 0xC1, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9) - tu := time.Unix(0, int64(time.Now().Unix())) + tu := time.Unix(0, time.Now().Unix()) testCases := []struct { testCaseName string diff --git a/pkg/parsers/scanner/protoseq_scanner.go b/pkg/parsers/scanner/protoseq_scanner.go index 718f8db1..48b6c30e 100644 --- a/pkg/parsers/scanner/protoseq_scanner.go +++ b/pkg/parsers/scanner/protoseq_scanner.go @@ -10,14 +10,14 @@ import ( ) var ( - // magic protoseq sequence: 1FF7F77EBEA65E9E37A6F62EFEAE47A7B76EBFAF169E9F37F657F766A706AFF7 + // magic protoseq sequence: 1FF7F77EBEA65E9E37A6F62EFEAE47A7B76EBFAF169E9F37F657F766A706AFF7. magicSeq = []byte{ 31, 247, 247, 126, 190, 166, 94, 158, 55, 166, 246, 46, 254, 174, 71, 167, 183, 110, 191, 175, 22, 158, 159, 55, 246, 87, 247, 102, 167, 6, 175, 247, } - // 64mb - const from push-client + // 64mb - const from push-client. maxRecordSize uint32 = 64 * 1024 * 1024 ) diff --git a/pkg/parsers/tests/generic_parser_test.go b/pkg/parsers/tests/generic_parser_test.go index 703e581f..24d73f11 100644 --- a/pkg/parsers/tests/generic_parser_test.go +++ b/pkg/parsers/tests/generic_parser_test.go @@ -140,24 +140,6 @@ func TestParser_TableSplitterSpecChar(t *testing.T) { require.Equal(t, test6TableName[:len(test6TableExpectedPrefix)], test6TableExpectedPrefix) } -func changeItemFactoryForMetrica(tableName string, appVersionName string, receiveTimestamp int64, apiKey int32, deviceID interface{}) *abstract.ChangeItem { - table1Schema := abstract.NewTableSchema(abstract.TableColumns{ - abstract.MakeTypedColSchema("AppVersionName", string(schema.TypeString), true), - abstract.MakeTypedColSchema("ReceiveTimestamp", string(schema.TypeInt64), false), - abstract.MakeTypedColSchema("APIKey", string(schema.TypeInt32), false), - abstract.MakeTypedColSchema("DeviceID", string(schema.TypeAny), false), - }) - item1 := abstract.ChangeItem{ - Kind: "Insert", - Schema: "db", - Table: tableName, - ColumnNames: []string{"AppVersionName", "ReceiveTimestamp", "APIKey", "DeviceID"}, - ColumnValues: []interface{}{appVersionName, receiveTimestamp, apiKey, deviceID}, - TableSchema: table1Schema, - } - return &item1 -} - func testTableSplitterOnChangeItem( t *testing.T, originalTableName string, @@ -224,7 +206,6 @@ func TestGenericParser_DoSensitive(t *testing.T) { fieldsWithSecretErasure++ } } - } logger.Log.Debug("Example of secret erasure", log.Any("res", res)) require.Greater(t, fieldsWithSecretErasure, 0, "there should be at least one erasured secret") @@ -245,7 +226,6 @@ func TestGenericParser_DoSensitiveDisabled(t *testing.T) { fieldsWithSecretErasure++ } } - } logger.Log.Debug("Example of keeping secrets unerasured to increase performance", log.Any("res", res)) require.Equal(t, 0, fieldsWithSecretErasure, "secrets should not be processed in order to increase performance") @@ -440,9 +420,9 @@ func TestGenericParser_Parse_vs_Do(t *testing.T) { return } require.NoError(t, err) - //genericParserImpl := GetGenericParserImpl(parser) + // genericParserImpl := GetGenericParserImpl(parser) res := parser.Do(samples.Data[k], *new(abstract.Partition)) - //batch := genericParserImpl.Parse(samples.Data[k], *new(abstract.Partition)) + // batch := genericParserImpl.Parse(samples.Data[k], *new(abstract.Partition)) var changes []abstract.ChangeItem //for batch.Next() { // ev, err := batch.Event() @@ -458,7 +438,7 @@ func TestGenericParser_Parse_vs_Do(t *testing.T) { abstract.Dump(res) t.Logf("new parser: %v len", len(changes)) abstract.Dump(changes) - //require.Equal(t, len(res), len(changes)) + // require.Equal(t, len(res), len(changes)) }) } } diff --git a/pkg/parsers/utils.go b/pkg/parsers/utils.go index 7cebbe64..b7572e3f 100644 --- a/pkg/parsers/utils.go +++ b/pkg/parsers/utils.go @@ -19,10 +19,8 @@ import ( ) type schema struct { - Path ypath.Path - Fields []abstract.ColSchema `json:"fields"` - revision int - dead chan bool + Path ypath.Path + Fields []abstract.ColSchema `json:"fields"` } func adjustType(s string) string { diff --git a/pkg/pgha/pg.go b/pkg/pgha/pg.go index f4e80630..73762e04 100644 --- a/pkg/pgha/pg.go +++ b/pkg/pgha/pg.go @@ -93,7 +93,7 @@ func (pg *PgHA) ConnStringByHost(host string) string { pg.user, pg.password) if pg.ssl { - q = q + " sslmode=verify-full" + q += " sslmode=verify-full" } return q } @@ -216,7 +216,7 @@ func NewFromHosts(dbName, user, password string, hosts []string, port int, ssl b password, ) if ssl { - connStr = connStr + " sslmode=verify-full" + connStr += " sslmode=verify-full" } dsn, err := pgx.ParseConfig(connStr) if err != nil { diff --git a/pkg/predicate/parser.go b/pkg/predicate/parser.go index 98e19815..f8ae5e39 100644 --- a/pkg/predicate/parser.go +++ b/pkg/predicate/parser.go @@ -40,7 +40,7 @@ func NewParser(r io.Reader) *Parser { // Parse starts scanning & parsing process (main entry point). // It returns an expression (AST) which you can use for the final evaluation -// of the conditions/statements +// of the conditions/statements. func (p *Parser) Parse() (Expr, error) { return p.parseExpr() } @@ -135,21 +135,22 @@ func (p *Parser) scanWithMapping() (Token, string) { case scanner.Ident: ttU := strings.ToUpper(tt) - if ttU == "AND" { + switch ttU { + case "AND": tok = AND - } else if ttU == "OR" { + case "OR": tok = OR - } else if ttU == "NOT" { + case "NOT": _, tmp := p.scan() if tmp == "(" { p.unscan() tok = NOT } - } else if ttU == "TRUE" { + case "TRUE": tok = TRUE - } else if ttU == "FALSE" { + case "FALSE": tok = FALSE - } else { + default: tok = ILLEGAL } } @@ -162,7 +163,7 @@ func (p *Parser) unscan() { p.buf.buffSize = 1 } -// parseExpr is an entry point to parsing +// parseExpr is an entry point to parsing. func (p *Parser) parseExpr() (Expr, error) { // Parse a non-binary expression type to start. // This variable will always be the root of the expression tree. @@ -201,7 +202,6 @@ func (p *Parser) parseExpr() (Expr, error) { expr = &BinaryExpr{LHS: expr, RHS: rhs, Op: op} } } - } // parseUnaryExpr parses an non-binary expression. diff --git a/pkg/predicate/token.go b/pkg/predicate/token.go index 1bb96fde..d2649092 100644 --- a/pkg/predicate/token.go +++ b/pkg/predicate/token.go @@ -4,12 +4,12 @@ package predicate type Token int const ( - // ILLEGAL token represent illegal token found in the statement + // ILLEGAL token represent illegal token found in the statement. ILLEGAL Token = iota - // EOF token represents end of statement + // EOF token represents end of statement. EOF - // Literals + // Literals. NUMBER // 12345.67 STRING // "abc" TRUE // true @@ -80,11 +80,3 @@ func (tok Token) Precedence() int { // isOperator returns true for operator tokens. func (tok Token) isOperator() bool { return tok > operatorBegin && tok < operatorEnd } - -// tokstr returns a literal if provided, otherwise returns the token string. -func tokstr(tok Token, lit string) string { - if lit != "" { - return lit - } - return tok.String() -} diff --git a/pkg/providers/airbyte/provider.go b/pkg/providers/airbyte/provider.go index 56f8d592..a8b20064 100644 --- a/pkg/providers/airbyte/provider.go +++ b/pkg/providers/airbyte/provider.go @@ -16,7 +16,7 @@ func init() { providers.Register(ProviderType, New) } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Activator = (*Provider)(nil) _ providers.Snapshot = (*Provider)(nil) diff --git a/pkg/providers/airbyte/provider_model.go b/pkg/providers/airbyte/provider_model.go index 35815341..0f57e54a 100644 --- a/pkg/providers/airbyte/provider_model.go +++ b/pkg/providers/airbyte/provider_model.go @@ -147,7 +147,7 @@ func (b *JSONProperty) UnmarshalJSON(data []byte) error { return xerrors.Errorf("unable to parse json property: %w", errs) } -// JSONProperty type can be array or single string, so had to implement custom unmarshal +// JSONProperty type can be array or single string, so had to implement custom unmarshal. type JSONProperty struct { Type []string `json:"type"` Format string `json:"format"` diff --git a/pkg/providers/airbyte/storage.go b/pkg/providers/airbyte/storage.go index c24123aa..8ae5d73b 100644 --- a/pkg/providers/airbyte/storage.go +++ b/pkg/providers/airbyte/storage.go @@ -6,7 +6,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "os" "os/exec" "sort" @@ -166,7 +166,7 @@ func (a *Storage) LoadTable(ctx context.Context, table abstract.TableDescription if err := a.storeState(table.ID(), currentState); err != nil { return xerrors.Errorf("unable to store incremental state: %w", err) } - data, err := ioutil.ReadAll(stderr) + data, err := io.ReadAll(stderr) if err != nil { return xerrors.Errorf("%s stderr read all failed: %w", table.ID().String(), err) } @@ -323,7 +323,7 @@ func (a *Storage) writeFile(fileName, fileData string) error { fullPath := fmt.Sprintf("%v/%v", a.config.DataDir(), fileName) a.logger.Debugf("%s -> \n%s", fileName, fileData) defer a.logger.Infof("file(%s) %s written", format.SizeInt(len(fileData)), fullPath) - return ioutil.WriteFile( + return os.WriteFile( fullPath, []byte(fileData), 0664, diff --git a/pkg/providers/airbyte/storage_incremental.go b/pkg/providers/airbyte/storage_incremental.go index 8542043c..9e4b252d 100644 --- a/pkg/providers/airbyte/storage_incremental.go +++ b/pkg/providers/airbyte/storage_incremental.go @@ -23,7 +23,7 @@ func (a *Storage) GetIncrementalState(ctx context.Context, incremental []abstrac }), nil } -// SetInitialState should have done nothing, since state handled inside loadTable method +// SetInitialState should have done nothing, since state handled inside loadTable method. func (a *Storage) SetInitialState(tables []abstract.TableDescription, incrementalTables []abstract.IncrementalTable) { for i, table := range tables { for _, incremental := range incrementalTables { diff --git a/pkg/providers/bigquery/provider.go b/pkg/providers/bigquery/provider.go index af7c4260..0e602f2c 100644 --- a/pkg/providers/bigquery/provider.go +++ b/pkg/providers/bigquery/provider.go @@ -26,7 +26,7 @@ func destinationModelFactory() model.Destination { const ProviderType = abstract.ProviderType("bigquery") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) ) diff --git a/pkg/providers/clickhouse/a2_cluster_tables.go b/pkg/providers/clickhouse/a2_cluster_tables.go index 51defec8..9bb01076 100644 --- a/pkg/providers/clickhouse/a2_cluster_tables.go +++ b/pkg/providers/clickhouse/a2_cluster_tables.go @@ -115,7 +115,7 @@ func newClusterTablesFromDescription(storage ClickhouseStorage, config *model.Ch return objs, nil } -// Hack: convert base.DataObjectFilter to abstract.IncludeTableList to CH storage +// Hack: convert base.DataObjectFilter to abstract.IncludeTableList to CH storage. type filterWrapper struct { baseFilter base.DataObjectFilter } diff --git a/pkg/providers/clickhouse/a2_data_provider.go b/pkg/providers/clickhouse/a2_data_provider.go index 371248c8..901b518f 100644 --- a/pkg/providers/clickhouse/a2_data_provider.go +++ b/pkg/providers/clickhouse/a2_data_provider.go @@ -24,7 +24,7 @@ type DataProvider struct { storage ClickhouseStorage } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ base.SnapshotProvider = (*DataProvider)(nil) ) diff --git a/pkg/providers/clickhouse/async/model/parts/part.go b/pkg/providers/clickhouse/async/model/parts/part.go index c70677b7..52cf0d8e 100644 --- a/pkg/providers/clickhouse/async/model/parts/part.go +++ b/pkg/providers/clickhouse/async/model/parts/part.go @@ -4,7 +4,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// Parts is a collection of table parts +// Parts is a collection of table parts. type Parts interface { Add(id abstract.TablePartID, part Part) Part(id abstract.TablePartID) Part diff --git a/pkg/providers/clickhouse/async/sink.go b/pkg/providers/clickhouse/async/sink.go index 6510021c..7ecc118d 100644 --- a/pkg/providers/clickhouse/async/sink.go +++ b/pkg/providers/clickhouse/async/sink.go @@ -168,7 +168,7 @@ func (s *sink) mergePart(partID abstract.TablePartID) error { } } -// flushPart commits all uncommited part data but allows to continue writing to the same part +// flushPart commits all uncommited part data but allows to continue writing to the same part. func (s *sink) flushPart(partID abstract.TablePartID) chan error { return s.withBacklog(partID, false, func() error { s.lgr.Info("Need to synchronize, flushing part", diff --git a/pkg/providers/clickhouse/async/streamer.go b/pkg/providers/clickhouse/async/streamer.go index 027f23cb..d0247181 100644 --- a/pkg/providers/clickhouse/async/streamer.go +++ b/pkg/providers/clickhouse/async/streamer.go @@ -39,7 +39,7 @@ type chV2Streamer struct { // BlockMarshallingError is a wrapper for clickhouse-go/v2 *proto.BlockError // *proto.BlockError occurs if the driver failed to build clickhouse native proto block. -// Usually it happens due to incorrect input types or values +// Usually it happens due to incorrect input types or values. type BlockMarshallingError struct { err *proto.BlockError code coded.Code diff --git a/pkg/providers/clickhouse/columntypes/types.go b/pkg/providers/clickhouse/columntypes/types.go index 586aba27..6d7f8f4c 100644 --- a/pkg/providers/clickhouse/columntypes/types.go +++ b/pkg/providers/clickhouse/columntypes/types.go @@ -114,7 +114,7 @@ func Restore(column abstract.ColSchema, val interface{}) interface{} { } } -// BaseType strips modifiers from the given ClickHouse type, if possible +// BaseType strips modifiers from the given ClickHouse type, if possible. func BaseType(chType string) string { extType, intType := divideTypeToExtAndInt(chType) if extType == "Nullable" { @@ -148,7 +148,7 @@ var chTypeWithModifierRe *regexp.Regexp = regexp.MustCompile(`(\w*)\((.*)\)`) // ToYtType converts the given ClickHouse type to YT type and a requiredness flag. // -// XXX: support all types from system.data_type_families, such as Decimal, Tuple, Array, Nested, Array(Array(...)) +// XXX: support all types from system.data_type_families, such as Decimal, Tuple, Array, Nested, Array(Array(...)). func ToYtType(chType string) (ytType string, required bool) { var result schema.Type switch BaseType(chType) { diff --git a/pkg/providers/clickhouse/conn/connection.go b/pkg/providers/clickhouse/conn/connection.go index 6e70ac72..5e1c3c3c 100644 --- a/pkg/providers/clickhouse/conn/connection.go +++ b/pkg/providers/clickhouse/conn/connection.go @@ -25,10 +25,8 @@ func ResolveShards(config model.ChSinkParams, transfer *dp_model.Transfer) error } config.SetShards(shards) - } else { - if len(config.Shards()) == 0 { - return abstract.NewFatalError(xerrors.New("shards not set for an on-premises ClickHouse")) - } + } else if len(config.Shards()) == 0 { + return abstract.NewFatalError(xerrors.New("shards not set for an on-premises ClickHouse")) } return nil } diff --git a/pkg/providers/clickhouse/errors/check_distributed.go b/pkg/providers/clickhouse/errors/check_distributed.go index 64adff66..3590640e 100644 --- a/pkg/providers/clickhouse/errors/check_distributed.go +++ b/pkg/providers/clickhouse/errors/check_distributed.go @@ -10,9 +10,9 @@ import ( ) const ( - // ClickhouseDDLTimeout is default value of distributed_ddl_task_timeout CH setting, used as fallback + // ClickhouseDDLTimeout is default value of distributed_ddl_task_timeout CH setting, used as fallback. ClickhouseDDLTimeout = 180 - // DDLTimeoutCorrection is added to DDL query timeout to have a chance to catch CH error, instead of ContextDeadlineExceeded + // DDLTimeoutCorrection is added to DDL query timeout to have a chance to catch CH error, instead of ContextDeadlineExceeded. DDLTimeoutCorrection = 2 ClickhouseReadTimeout = 30 * time.Second ) @@ -35,7 +35,7 @@ type ErrDistributedDDLTimeout struct { wrapped error } -// Unwrap implements xerrors.Uwrapper +// Unwrap implements xerrors.Uwrapper. func (e *ErrDistributedDDLTimeout) Unwrap() error { return e.wrapped } @@ -46,7 +46,7 @@ func (e *ErrDistributedDDLTimeout) Error() string { var distrTaskRe = regexp.MustCompile(`(/clickhouse/task_queue/ddl/[\w-]+)`) -// AsDistributedDDLTimeout checks whether err is clickhouse DDL timeout error and finds DDL task path +// AsDistributedDDLTimeout checks whether err is clickhouse DDL timeout error and finds DDL task path. func AsDistributedDDLTimeout(err error) *ErrDistributedDDLTimeout { chError := new(clickhouse.Exception) // Distributed DDL Timeout has code 159 (TIMEOUT_EXCEEDED) and diff --git a/pkg/providers/clickhouse/format/csv_event.go b/pkg/providers/clickhouse/format/csv_event.go index 5134ffdf..fe5e2127 100644 --- a/pkg/providers/clickhouse/format/csv_event.go +++ b/pkg/providers/clickhouse/format/csv_event.go @@ -65,7 +65,7 @@ func (e *CSVEvent) ToOldChangeItem() (*abstract.ChangeItem, error) { ColumnNames: e.colNames, ColumnValues: row, TableSchema: e.cols, - OldKeys: *new(abstract.OldKeysType), + OldKeys: abstract.OldKeysType{}, TxID: "", Query: "", Size: abstract.EmptyEventSize(), diff --git a/pkg/providers/clickhouse/format/json_compact_event.go b/pkg/providers/clickhouse/format/json_compact_event.go index afe53268..6eccad0c 100644 --- a/pkg/providers/clickhouse/format/json_compact_event.go +++ b/pkg/providers/clickhouse/format/json_compact_event.go @@ -44,7 +44,7 @@ func (e *JSONCompactEvent) ToOldChangeItem() (*abstract.ChangeItem, error) { ColumnNames: e.colNames, ColumnValues: row, TableSchema: e.cols, - OldKeys: *new(abstract.OldKeysType), + OldKeys: abstract.OldKeysType{}, TxID: "", Query: "", Size: abstract.EmptyEventSize(), diff --git a/pkg/providers/clickhouse/httpclient/http_client_impl.go b/pkg/providers/clickhouse/httpclient/http_client_impl.go index 672611ba..7cf6c3b4 100644 --- a/pkg/providers/clickhouse/httpclient/http_client_impl.go +++ b/pkg/providers/clickhouse/httpclient/http_client_impl.go @@ -122,7 +122,7 @@ func (c *httpClientImpl) Exec(ctx context.Context, lgr log.Logger, host string, } // error string ex.: "Code: 170. DB::Exception: Requested cluster 'test' not found. (BAD_GET) (version 22.3.12.19 (official build))" -// may contain code name or not. Punctuation may vary among CH versions +// may contain code name or not. Punctuation may vary among CH versions. var ( httpExcBaseRe = regexp.MustCompile(`^.*Code: (?P\d+)\W+(?P.*)`) httpExcVersionRe = regexp.MustCompile(`(?P.*)\s+\(version.*`) diff --git a/pkg/providers/clickhouse/httpclient/http_client_mock.go b/pkg/providers/clickhouse/httpclient/http_client_mock.go index 8bd56fda..b5908600 100644 --- a/pkg/providers/clickhouse/httpclient/http_client_mock.go +++ b/pkg/providers/clickhouse/httpclient/http_client_mock.go @@ -13,8 +13,8 @@ import ( io "io" reflect "reflect" - log "go.ytsaurus.tech/library/go/core/log" gomock "go.uber.org/mock/gomock" + log "go.ytsaurus.tech/library/go/core/log" ) // MockHTTPClient is a mock of HTTPClient interface. diff --git a/pkg/providers/clickhouse/httpuploader/marshal.go b/pkg/providers/clickhouse/httpuploader/marshal.go index 0b167f2e..9d50803e 100644 --- a/pkg/providers/clickhouse/httpuploader/marshal.go +++ b/pkg/providers/clickhouse/httpuploader/marshal.go @@ -67,7 +67,7 @@ func marshalTime(colType *columntypes.TypeDescription, v time.Time, buf *bytes.B case colType.IsDateTime64: fullTS := v.UnixNano() if colType.DateTime64Precision() > 0 && colType.DateTime64Precision() < 9 { - fullTS = fullTS / int64(math.Pow(10, float64(9-colType.DateTime64Precision()))) + fullTS /= int64(math.Pow(10, float64(9-colType.DateTime64Precision()))) } fmt.Fprintf(buf, "%d", fullTS) case colType.IsDate: @@ -120,7 +120,7 @@ func MarshalCItoJSON(row abstract.ChangeItem, rules *MarshallingRules, buf *byte break } buf.WriteString(`"`) - if bytes.ContainsRune([]byte(v), rune('"')) || bytes.ContainsRune([]byte(v), rune('\\')) { + if strings.ContainsRune(v, rune('"')) || strings.ContainsRune(v, rune('\\')) { // We have a " symbol in value, so we must quote string before submit it // we want to preserve bytes as is since, clickhouse can accept them and store properly // that's why we use this custom quote func @@ -204,7 +204,7 @@ func MarshalCItoJSON(row abstract.ChangeItem, rules *MarshallingRules, buf *byte } case []byte: if columntypes.LegacyIsDecimal(colSchema.OriginalType) || colType.IsDecimal { - buf.WriteString(string(v)) + buf.Write(v) break } if colType.IsArray { @@ -256,7 +256,7 @@ func isNilValue(v interface{}) bool { return vv.Kind() == reflect.Pointer && vv.IsNil() } -// questionableQuoter is like strconv.Quote, but do not try to escape non-utf8 chars +// questionableQuoter is like strconv.Quote, but do not try to escape non-utf8 chars. func questionableQuoter(v string) string { return strings.ReplaceAll(strings.ReplaceAll(v, `\`, `\\`), `"`, `\"`) } diff --git a/pkg/providers/clickhouse/httpuploader/uploader.go b/pkg/providers/clickhouse/httpuploader/uploader.go index f48ac20a..2584484d 100644 --- a/pkg/providers/clickhouse/httpuploader/uploader.go +++ b/pkg/providers/clickhouse/httpuploader/uploader.go @@ -48,7 +48,7 @@ type UploadStats struct { UploadStartTime time.Time } -// Insert row buffers should be pooled by tables as different tables may have different row size +// Insert row buffers should be pooled by tables as different tables may have different row size. func getPoolForTable(table string) *sync.Pool { tableBufMu.Lock() defer tableBufMu.Unlock() diff --git a/pkg/providers/clickhouse/model/model_ch_destination.go b/pkg/providers/clickhouse/model/model_ch_destination.go index e2a635f6..9fc8cc23 100644 --- a/pkg/providers/clickhouse/model/model_ch_destination.go +++ b/pkg/providers/clickhouse/model/model_ch_destination.go @@ -30,7 +30,7 @@ var ( _ model.Describable = (*ChDestination)(nil) ) -// ChDestination - see description of fields in sink_params.go +// ChDestination - see description of fields in sink_params.go. type ChDestination struct { // ChSinkServerParams MdbClusterID string `json:"Cluster"` @@ -213,14 +213,14 @@ func (d *ChDestination) shallUseJSON(transfer *model.Transfer) bool { return model.IsAppendOnlySource(transfer.Src) } -// ToSinkParams converts the model into sink properties object, which contains extra information which depends on transfer type +// ToSinkParams converts the model into sink properties object, which contains extra information which depends on transfer type. func (d *ChDestination) ToSinkParams(transfer *model.Transfer) ChDestinationWrapper { wrapper := newChDestinationWrapper(*d) wrapper.useJSON = d.shallUseJSON(transfer) return *wrapper } -// ToReplicationFromPGSinkParams converts the model into sink properties object that would be constructed for a replication from PostgreSQL +// ToReplicationFromPGSinkParams converts the model into sink properties object that would be constructed for a replication from PostgreSQL. func (d *ChDestination) ToReplicationFromPGSinkParams() ChDestinationWrapper { return *newChDestinationWrapper(*d) } @@ -235,7 +235,7 @@ func (d *ChDestination) FillDependentFields(transfer *model.Transfer) { } } -// ChDestinationWrapper implements ChSinkParams +// ChDestinationWrapper implements ChSinkParams. type ChDestinationWrapper struct { Model *ChDestination host string // host is here, bcs it needed only in SinkServer/SinkTable @@ -248,7 +248,7 @@ func (d ChDestinationWrapper) InsertSettings() InsertParams { return d.Model.InsertParams } -// newChDestinationWrapper copies the model provided to it in order to be able to modify the fields in it +// newChDestinationWrapper copies the model provided to it in order to be able to modify the fields in it. func newChDestinationWrapper(model ChDestination) *ChDestinationWrapper { return &ChDestinationWrapper{ Model: &model, @@ -423,7 +423,7 @@ func (d ChDestinationWrapper) MakeChildShardParams(altHosts []string) ChSinkShar } // SetShards -// we can set model variables, bcs we make copy of ChDestination in NewChDestinationV1 +// we can set model variables, bcs we make copy of ChDestination in NewChDestinationV1. func (d ChDestinationWrapper) SetShards(shards map[string][]string) { d.Model.ShardsList = []ClickHouseShard{} for shardName, hosts := range shards { diff --git a/pkg/providers/clickhouse/model/model_sink_params.go b/pkg/providers/clickhouse/model/model_sink_params.go index f26c6f3a..c7188b18 100644 --- a/pkg/providers/clickhouse/model/model_sink_params.go +++ b/pkg/providers/clickhouse/model/model_sink_params.go @@ -11,7 +11,7 @@ const ( // BufferTriggingSizeDefault is a recommended default value for bufferer trigging size // Default value assume that we have 4 thread writer in 3gb box (default runtime box) // so each thread would consume at most 256 * 2 (one time for source one time for target) mb + some constant memory - // in total it would eat 512 * 4 = 2gb, which is less than 3gb + // in total it would eat 512 * 4 = 2gb, which is less than 3gb. BufferTriggingSizeDefault uint64 = 256 * humanize.MiByte ) diff --git a/pkg/providers/clickhouse/provider.go b/pkg/providers/clickhouse/provider.go index 742395e8..2923afc4 100644 --- a/pkg/providers/clickhouse/provider.go +++ b/pkg/providers/clickhouse/provider.go @@ -38,7 +38,7 @@ func init() { const ProviderType = abstract.ProviderType("ch") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Snapshot = (*Provider)(nil) _ providers.Abstract2Provider = (*Provider)(nil) diff --git a/pkg/providers/clickhouse/schema/ddl.go b/pkg/providers/clickhouse/schema/ddl.go index a898a3c4..186a6c19 100644 --- a/pkg/providers/clickhouse/schema/ddl.go +++ b/pkg/providers/clickhouse/schema/ddl.go @@ -9,9 +9,9 @@ import ( const matViewEngine = "MaterializedView" const ( - // ClickhouseDDLTimeout is default value of distributed_ddl_task_timeout CH setting, used as fallback + // ClickhouseDDLTimeout is default value of distributed_ddl_task_timeout CH setting, used as fallback. ClickhouseDDLTimeout = 180 - // DDLTimeoutCorrection is added to DDL query timeout to have a chance to catch CH error, instead of ContextDeadlineExceeded + // DDLTimeoutCorrection is added to DDL query timeout to have a chance to catch CH error, instead of ContextDeadlineExceeded. DDLTimeoutCorrection = 2 ClickhouseReadTimeout = 30 * time.Second ) diff --git a/pkg/providers/clickhouse/schema/describe.go b/pkg/providers/clickhouse/schema/describe.go index 9091c9f5..9b2db3e4 100644 --- a/pkg/providers/clickhouse/schema/describe.go +++ b/pkg/providers/clickhouse/schema/describe.go @@ -91,7 +91,7 @@ func DescribeTable(db *sql.DB, database, table string, knownPrimaryKeys []string // clickhouse will return unordered columns, this need to set stable column names order sort.Strings(colNames) schema := make([]abstract.ColSchema, 0) - //add primary keys first + // add primary keys first for _, pkeyColname := range knownPrimaryKeys { colPrimary[pkeyColname] = true coltype, ok := colTypes[pkeyColname] @@ -116,7 +116,7 @@ func DescribeTable(db *sql.DB, database, table string, knownPrimaryKeys []string }) } - //add other fields + // add other fields for _, colname := range colNames { if colname == "__data_transfer_commit_time" || colname == "__data_transfer_delete_time" { continue diff --git a/pkg/providers/clickhouse/schema/engine.go b/pkg/providers/clickhouse/schema/engine.go index 248578b3..5fdae098 100644 --- a/pkg/providers/clickhouse/schema/engine.go +++ b/pkg/providers/clickhouse/schema/engine.go @@ -49,7 +49,7 @@ const ( ReplicatedVersionedCollapsingMergeTree = EngineType("ReplicatedVersionedCollapsingMergeTree") ReplicatedGraphiteMergeTree = EngineType("ReplicatedGraphiteMergeTree") - // SharedMergeTree family is available on ch.inc exclusively + // SharedMergeTree family is available on ch.inc exclusively. SharedMergeTree = EngineType("SharedMergeTree") SharedReplacingMergeTree = EngineType("SharedReplacingMergeTree") SharedSummingMergeTree = EngineType("SharedSummingMergeTree") @@ -165,32 +165,6 @@ func ParseMergeTreeFamilyEngine(sql string) (*MergeTreeFamilyEngine, string, err return GetEngine(engineStr) } -type substr struct { - startIdx int - endIdx int -} - -func grepEngineFull(sql string) (substr, error) { - startIdx := strings.Index(sql, "ENGINE = ") - if startIdx == -1 { - return substr{}, xerrors.Errorf("Cannot find engine definition in '%v'", sql) - } - startIdx += len("ENGINE = ") - - endIdx := -1 - if endIdx = TryFindNextStatement(sql, startIdx); endIdx == -1 { - if endIdx = strings.Index(sql[startIdx:], ")"); endIdx == -1 { - if endIdx = strings.Index(sql[startIdx:], " "); endIdx == -1 { - return substr{}, xerrors.Errorf("Cannot find engine definition in '%v'", sql) - } - } else { - endIdx += 1 // need include closing parenthesis - } - endIdx += startIdx - } - return substr{startIdx: startIdx, endIdx: endIdx}, nil -} - func TryFindNextStatement(sql string, from int) int { possibleStatements := []string{ " ORDER BY", diff --git a/pkg/providers/clickhouse/storage.go b/pkg/providers/clickhouse/storage.go index 0c567cf4..4b073194 100644 --- a/pkg/providers/clickhouse/storage.go +++ b/pkg/providers/clickhouse/storage.go @@ -222,7 +222,7 @@ func (s *Storage) LoadTable(ctx context.Context, table abstract.TableDescription } }() - //get tables schema []abstract.ColSchema (as in ListTable) + // get tables schema []abstract.ColSchema (as in ListTable) tableAllColumns, tableFilteredColumns, err := s.getTableSchema(table.ID(), s.IsHomo) if err != nil { return xerrors.Errorf("unable to discover table schema in storage: %w", err) @@ -237,7 +237,7 @@ func (s *Storage) LoadTable(ctx context.Context, table abstract.TableDescription chunkSize := s.inferDesiredChunkSize(table) - //build read query with filter and offset + // build read query with filter and offset readQ := buildSelectQuery(&table, tableAllColumns.Columns(), s.IsHomo, deletable, "") s.logger.Info("built select query", log.Any("query", readQ)) @@ -357,7 +357,7 @@ func (s *Storage) inferDesiredChunkSize(table abstract.TableDescription) uint64 return uint64(minSize) } calculatedSize := int(float64(s.bufSize) / (float64(bytes) / float64(rows))) - chunkSize := calculatedSize + var chunkSize int if bytes > 0 && calculatedSize < minSize { chunkSize = minSize } else { @@ -374,7 +374,7 @@ func (s *Storage) inferDesiredChunkSize(table abstract.TableDescription) uint64 } // in fact - it's estimate, and it's enough to be estimate -// works only for 20+ clickhouse version +// works only for 20+ clickhouse version. func (s *Storage) getTableSize(tableID abstract.TableID) (rows uint64, bytes uint64, err error) { if s.version.Major < 20 { return 0, 0, nil @@ -453,7 +453,7 @@ func (t *table) ToTableID() abstract.TableID { } func (s *Storage) BuildTableQuery(table abstract.TableDescription) (*abstract.TableSchema, string, string, error) { - //get tables schema []abstract.ColSchema (as in ListTable) + // get tables schema []abstract.ColSchema (as in ListTable) tableAllColumns, readCols, err := s.getTableSchema(table.ID(), s.IsHomo) if err != nil { return nil, "", "", xerrors.Errorf("unable to discover table schema in storage: %w", err) @@ -462,7 +462,7 @@ func (s *Storage) BuildTableQuery(table abstract.TableDescription) (*abstract.Ta if err != nil { return nil, "", "", xerrors.Errorf("failed to determine deletable table %s: %w", table.Fqtn(), err) } - //build read query with filter and offset + // build read query with filter and offset readQ := buildSelectQuery(&table, tableAllColumns.Columns(), s.IsHomo, deletable, "") s.logger.Info("built select query", log.Any("query", readQ)) countQ := buildCountQuery(&table, deletable, "") diff --git a/pkg/providers/clickhouse/toast.go b/pkg/providers/clickhouse/toast.go index 44135ba9..3527d9f9 100644 --- a/pkg/providers/clickhouse/toast.go +++ b/pkg/providers/clickhouse/toast.go @@ -69,7 +69,7 @@ func buildPkeysDistinct(cols []abstract.ColSchema) string { return "(" + strings.Join(q, ",") + ")" } -// primaryKeyValuesFlattened returns a list of (old) primary key values for all items in the given list +// primaryKeyValuesFlattened returns a list of (old) primary key values for all items in the given list. func primaryKeyValuesFlattened(items []abstract.ChangeItem, colSchemas []abstract.ColSchema) ([]interface{}, error) { colSchemaByName := make(map[string]*abstract.ColSchema) for i := range colSchemas { @@ -139,7 +139,7 @@ func getColumns(cols []abstract.ColSchema, changeItems []abstract.ChangeItem) ([ return result, nil } -// convertToastedToNormal converts toasted items (if any are present) to their complete versions using values ofbtained from the target CH +// convertToastedToNormal converts toasted items (if any are present) to their complete versions using values ofbtained from the target CH. func convertToastedToNormal(t *sinkTable, items []abstract.ChangeItem) ([]abstract.ChangeItem, error) { toastedItems, err := getToastedChangeItems(t, items) if err != nil { diff --git a/pkg/providers/coralogix/api.go b/pkg/providers/coralogix/api.go index 190820c2..2dc0d45b 100644 --- a/pkg/providers/coralogix/api.go +++ b/pkg/providers/coralogix/api.go @@ -17,7 +17,7 @@ import ( type Severity int -// 1 – Debug, 2 – Verbose, 3 – Info, 4 – Warn, 5 – Error, 6 – Critical +// 1 – Debug, 2 – Verbose, 3 – Info, 4 – Warn, 5 – Error, 6 – Critical. const ( Debug = Severity(1) Verbose = Severity(2) @@ -50,7 +50,7 @@ func SubmitLogs(data []HTTPLogItem, domain, token string) error { } body := bytes.NewReader(payloadBytes) - req, err := http.NewRequest("POST", fmt.Sprintf("https://ingress.%s/logs/v1/singles", domain), body) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("https://ingress.%s/logs/v1/singles", domain), body) if err != nil { return xerrors.Errorf("unable to make request: %w", err) } diff --git a/pkg/providers/coralogix/provider.go b/pkg/providers/coralogix/provider.go index a83c6fa0..0564bb11 100644 --- a/pkg/providers/coralogix/provider.go +++ b/pkg/providers/coralogix/provider.go @@ -26,7 +26,7 @@ func destinationModelFactory() model.Destination { const ProviderType = abstract.ProviderType("coralogix") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) ) diff --git a/pkg/providers/datadog/provider.go b/pkg/providers/datadog/provider.go index 7c2e910d..0ee9a4fe 100644 --- a/pkg/providers/datadog/provider.go +++ b/pkg/providers/datadog/provider.go @@ -28,7 +28,7 @@ func destinationModelFactory() model.Destination { const ProviderType = abstract.ProviderType("datadog") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) ) diff --git a/pkg/providers/delta/action/action.go b/pkg/providers/delta/action/action.go index 85f274d4..85ec6b69 100644 --- a/pkg/providers/delta/action/action.go +++ b/pkg/providers/delta/action/action.go @@ -46,21 +46,22 @@ type Single struct { } func (s *Single) Unwrap() Container { - if s.Add != nil { + switch { + case s.Add != nil: return s.Add - } else if s.Remove != nil { + case s.Remove != nil: return s.Remove - } else if s.MetaData != nil { + case s.MetaData != nil: return s.MetaData - } else if s.Txn != nil { + case s.Txn != nil: return s.Txn - } else if s.Protocol != nil { + case s.Protocol != nil: return s.Protocol - } else if s.Cdc != nil { + case s.Cdc != nil: return s.Cdc - } else if s.CommitInfo != nil { + case s.CommitInfo != nil: return s.CommitInfo - } else { + default: return nil } } diff --git a/pkg/providers/delta/golden_storage_test.go b/pkg/providers/delta/golden_storage_test.go index 9e119a8c..049c398f 100644 --- a/pkg/providers/delta/golden_storage_test.go +++ b/pkg/providers/delta/golden_storage_test.go @@ -23,7 +23,7 @@ import ( ) // badGoldenTest it's a set of cases that doomed to fail -// that was designed that way +// that was designed that way. var badGoldenTest = set.New( // todo: special character, s3 client fail them "deltatbl-special-chars-in-partition-column", @@ -130,7 +130,7 @@ func uploadOne(t *testing.T, cfg *DeltaSource, fname string, uploader *s3manager require.NoError(t, err) } -// this only works for local use +// this only works for local use. func TestGoldenDataSet(t *testing.T) { if os.Getenv("S3MDS_PORT") != "" { t.Skip() // only works with premade s3 bucket for now diff --git a/pkg/providers/delta/model_source.go b/pkg/providers/delta/model_source.go index 30901507..45f03a1e 100644 --- a/pkg/providers/delta/model_source.go +++ b/pkg/providers/delta/model_source.go @@ -6,7 +6,7 @@ import ( s3_provider "github.com/doublecloud/transfer/pkg/providers/s3" ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ model.Source = (*DeltaSource)(nil) ) diff --git a/pkg/providers/delta/protocol/checkpoint.go b/pkg/providers/delta/protocol/checkpoint.go index effbcb17..a6dd7628 100644 --- a/pkg/providers/delta/protocol/checkpoint.go +++ b/pkg/providers/delta/protocol/checkpoint.go @@ -37,12 +37,12 @@ func (t *CheckpointInstance) Compare(other CheckpointInstance) int { } } -// IsEarlierThan compare based just on version and amount of parts in checkpoint +// IsEarlierThan compare based just on version and amount of parts in checkpoint. func (t *CheckpointInstance) IsEarlierThan(other CheckpointInstance) bool { return t.IsNotLaterThan(other) || (t.Version == other.Version && t.NumParts < other.NumParts) } -// IsNotLaterThan compare based just on version +// IsNotLaterThan compare based just on version. func (t *CheckpointInstance) IsNotLaterThan(other CheckpointInstance) bool { if other.Compare(MaxInstance) == 0 { return true diff --git a/pkg/providers/delta/protocol/protocol_golden_test.go b/pkg/providers/delta/protocol/protocol_golden_test.go index 47efed5a..4153af73 100644 --- a/pkg/providers/delta/protocol/protocol_golden_test.go +++ b/pkg/providers/delta/protocol/protocol_golden_test.go @@ -12,7 +12,7 @@ import ( ) // badGoldenTest it's a set of cases that doomed to fail -// that was designed that way +// that was designed that way. var badGoldenTest = set.New( "versions-not-contiguous", "deltalog-state-reconstruction-without-protocol", diff --git a/pkg/providers/delta/protocol/replayer.go b/pkg/providers/delta/protocol/replayer.go index e3759169..08ab95c0 100644 --- a/pkg/providers/delta/protocol/replayer.go +++ b/pkg/providers/delta/protocol/replayer.go @@ -125,7 +125,7 @@ type replayTuple struct { type MemoryOptimizedLogReplay struct { files []string logStore store.Store - //timezone time.Location + // timezone time.Location checkpointReader CheckpointReader } diff --git a/pkg/providers/delta/protocol/snapshot.go b/pkg/providers/delta/protocol/snapshot.go index a6bea579..6c8119a6 100644 --- a/pkg/providers/delta/protocol/snapshot.go +++ b/pkg/providers/delta/protocol/snapshot.go @@ -152,38 +152,16 @@ func (s *Snapshot) Metadata() (*action.Metadata, error) { return s.metadata, nil } -// Version returns the version of this Snapshot +// Version returns the version of this Snapshot. func (s *Snapshot) Version() int64 { return s.version } -// CommitTS returns the time of commit for this Snapshot +// CommitTS returns the time of commit for this Snapshot. func (s *Snapshot) CommitTS() time.Time { return time.Unix(0, s.commitTS*int64(time.Millisecond)).UTC() } -func (s *Snapshot) tombstones() ([]*action.RemoveFile, error) { - return iter.ToSlice(s.state.tombstones) -} - -func (s *Snapshot) setTransactions() []*action.SetTransaction { - return s.state.setTransactions -} - -func (s *Snapshot) transactions() map[string]int64 { - // appID to version - trxs := s.setTransactions() - res := make(map[string]int64, len(trxs)) - for _, trx := range trxs { - res[trx.AppID] = int64(trx.Version) - } - return res -} - -func (s *Snapshot) numOfFiles() (int64, error) { - return s.state.numOfFiles, nil -} - func (s *Snapshot) files() []string { var res []string for _, f := range s.segment.Deltas { diff --git a/pkg/providers/delta/protocol/snapshot_reader.go b/pkg/providers/delta/protocol/snapshot_reader.go index 3212ba5e..7a1e227c 100644 --- a/pkg/providers/delta/protocol/snapshot_reader.go +++ b/pkg/providers/delta/protocol/snapshot_reader.go @@ -216,7 +216,7 @@ func (r *SnapshotReader) logSegmentForVersion(startCheckpoint int64, versionToLo } // emptySegment means there is no starting checkpoint found. This means that we should definitely have version 0, or the -// last checkpoint we thought should exist (the `_last_checkpoint` file) no longer exists +// last checkpoint we thought should exist (the `_last_checkpoint` file) no longer exists. func (r *SnapshotReader) emptySegment(startCheckpoint int64, deltas []*store.FileMeta) (*LogSegment, error) { if startCheckpoint > 0 { return nil, xerrors.Errorf("missing file part: %v", startCheckpoint) diff --git a/pkg/providers/delta/protocol/table_config.go b/pkg/providers/delta/protocol/table_config.go index 9332b57d..3a553f97 100644 --- a/pkg/providers/delta/protocol/table_config.go +++ b/pkg/providers/delta/protocol/table_config.go @@ -23,9 +23,7 @@ var ( DeltaConfigCheckpointInterval = &TableConfig[int]{ Key: "checkpointInterval", DefaultValue: "10", - FromString: func(s string) (int, error) { - return strconv.Atoi(s) - }, + FromString: strconv.Atoi, } EnableExpiredLogCleanupProp = &TableConfig[bool]{ Key: "enableExpiredLogCleanup", @@ -75,7 +73,7 @@ var timeMultiplexer = map[string]int{ // The string value of this config has to have the following format: interval . // Where is either week, day, hour, second, millisecond, microsecond or nanosecond. -// If it's missing in metadata then the `self.default` is used +// If it's missing in metadata then the `self.default` is used. func parseDuration(s string) (time.Duration, error) { fields := strings.Fields(strings.ToLower(s)) if len(fields) != 3 { diff --git a/pkg/providers/delta/provider.go b/pkg/providers/delta/provider.go index 9420b899..ad9f6285 100644 --- a/pkg/providers/delta/provider.go +++ b/pkg/providers/delta/provider.go @@ -6,7 +6,6 @@ import ( "github.com/doublecloud/transfer/library/go/core/metrics" "github.com/doublecloud/transfer/library/go/core/xerrors" "github.com/doublecloud/transfer/pkg/abstract" - "github.com/doublecloud/transfer/pkg/abstract/coordinator" "github.com/doublecloud/transfer/pkg/abstract/model" "github.com/doublecloud/transfer/pkg/providers" "go.ytsaurus.tech/library/go/core/log" @@ -24,7 +23,7 @@ func init() { abstract.RegisterProviderName(ProviderType, "Delta Lake") } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Snapshot = (*Provider)(nil) ) @@ -32,7 +31,6 @@ var ( type Provider struct { logger log.Logger registry metrics.Registry - cp coordinator.Coordinator transfer *model.Transfer } diff --git a/pkg/providers/delta/storage.go b/pkg/providers/delta/storage.go index cf0d8262..7f705e1d 100644 --- a/pkg/providers/delta/storage.go +++ b/pkg/providers/delta/storage.go @@ -23,13 +23,13 @@ import ( "go.ytsaurus.tech/yt/go/schema" ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ abstract.Storage = (*Storage)(nil) ) // defaultReadBatchSize is magic number by in-leskin -// we need to push rather small chunks so our bufferer can buffer effectively +// we need to push rather small chunks so our bufferer can buffer effectively. const defaultReadBatchSize = 128 type Storage struct { diff --git a/pkg/providers/delta/storage_sharding.go b/pkg/providers/delta/storage_sharding.go index 10da0777..5cc531f4 100644 --- a/pkg/providers/delta/storage_sharding.go +++ b/pkg/providers/delta/storage_sharding.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cast" ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ abstract.ShardingStorage = (*Storage)(nil) _ abstract.ShardingContextStorage = (*Storage)(nil) diff --git a/pkg/providers/delta/storage_snapshotable.go b/pkg/providers/delta/storage_snapshotable.go index 83b2bf26..b0169e6a 100644 --- a/pkg/providers/delta/storage_snapshotable.go +++ b/pkg/providers/delta/storage_snapshotable.go @@ -8,7 +8,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ abstract.SnapshotableStorage = (*Storage)(nil) ) diff --git a/pkg/providers/elastic/client.go b/pkg/providers/elastic/client.go index f0ad9b6f..a185b74b 100644 --- a/pkg/providers/elastic/client.go +++ b/pkg/providers/elastic/client.go @@ -2,7 +2,8 @@ package elastic import ( "fmt" - "io/ioutil" + "io" + "net" "reflect" "unsafe" @@ -29,7 +30,7 @@ func openSearchResolveHosts(clusterID string) ([]string, error) { result := make([]string, 0) for _, currHost := range hosts { if currHost.Type == "OPENSEARCH" { - result = append(result, fmt.Sprintf("https://%s:%d", currHost.Name, 9200)) + result = append(result, fmt.Sprintf("https://%s", net.JoinHostPort(currHost.Name, "9200"))) } } return result, nil @@ -43,7 +44,7 @@ func elasticSearchResolveHosts(clusterID string) ([]string, error) { result := make([]string, 0) for _, currHost := range hosts { if currHost.Type == "DATA_NODE" { - result = append(result, fmt.Sprintf("https://%s:%d", currHost.Name, 9200)) + result = append(result, fmt.Sprintf("https://%s", net.JoinHostPort(currHost.Name, "9200"))) } } return result, nil @@ -78,7 +79,7 @@ func ConfigFromDestination(logger log.Logger, cfg *ElasticSearchDestination, ser } if cfg.ClusterID == "" { - var protocol = "http" + protocol := "http" if cfg.SSLEnabled { protocol = "https" } @@ -103,7 +104,7 @@ func ConfigFromDestination(logger log.Logger, cfg *ElasticSearchDestination, ser } // setProductCheckSuccess -// cures client from working-only-with-elastic +// cures client from working-only-with-elastic. func setProductCheckSuccess(client *elasticsearch.Client) error { value := reflect.ValueOf(&client) elem := value.Elem() @@ -125,7 +126,7 @@ func getResponseBody(res *esapi.Response, err error) ([]byte, error) { } defer res.Body.Close() - body, err := ioutil.ReadAll(res.Body) + body, err := io.ReadAll(res.Body) if err != nil { return nil, xerrors.Errorf("failed to read response body: %w", err) } diff --git a/pkg/providers/elastic/dump_index.go b/pkg/providers/elastic/dump_index.go index 26b4ad57..beeca4eb 100644 --- a/pkg/providers/elastic/dump_index.go +++ b/pkg/providers/elastic/dump_index.go @@ -28,7 +28,7 @@ type IsElasticLikeDestination interface { ToElasticSearchDestination() (*ElasticSearchDestination, ServerType) } -// sourceHomoElasticSearch returns a non-nil object only for homogenous OpenSearch / ElasticSearch transfers +// sourceHomoElasticSearch returns a non-nil object only for homogenous OpenSearch / ElasticSearch transfers. func srcDstHomoElasticSearch(transfer *model.Transfer) (*ElasticSearchSource, ServerType) { src, srcIsElasticLike := transfer.Src.(IsElasticLikeSource) _, dstIsElasticLike := transfer.Dst.(IsElasticLikeDestination) diff --git a/pkg/providers/elastic/provider.go b/pkg/providers/elastic/provider.go index f9c205f4..2c8be0c1 100644 --- a/pkg/providers/elastic/provider.go +++ b/pkg/providers/elastic/provider.go @@ -35,7 +35,7 @@ func destinationModelFactory() model.Destination { const ProviderType = abstract.ProviderType("elasticsearch") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) _ providers.Snapshot = (*Provider)(nil) diff --git a/pkg/providers/elastic/schema.go b/pkg/providers/elastic/schema.go index 91279748..33778a5f 100644 --- a/pkg/providers/elastic/schema.go +++ b/pkg/providers/elastic/schema.go @@ -51,7 +51,7 @@ func (s *Storage) getSchemaFromElasticMapping(mappings mappingProperties, isHomo var originalType string if field.Type == "" && field.Properties != nil { - ok := false + var ok bool // object type schemaType, ok = rules[objectType] if !ok { @@ -60,7 +60,7 @@ func (s *Storage) getSchemaFromElasticMapping(mappings mappingProperties, isHomo originalType = objectType } if field.Type != "" { - ok := false + var ok bool if field.Type == aliasType && field.Path != "" { actualType, err := getOriginalTypeFromAliasField(mappings, field.Path) if err != nil { diff --git a/pkg/providers/elastic/sharding_storage.go b/pkg/providers/elastic/sharding_storage.go index 57826e8b..974d840a 100644 --- a/pkg/providers/elastic/sharding_storage.go +++ b/pkg/providers/elastic/sharding_storage.go @@ -33,7 +33,7 @@ func UnmarshalFilter(marshalledFilter string) (ShardingFilter, error) { } func filterFromTable(table abstract.TableDescription) (ShardingFilter, error) { - filter := ShardingFilter(emptyFilter) + filter := emptyFilter if table.Filter != "" { var err error diff --git a/pkg/providers/elastic/sink.go b/pkg/providers/elastic/sink.go index afc629da..c2215d74 100644 --- a/pkg/providers/elastic/sink.go +++ b/pkg/providers/elastic/sink.go @@ -101,13 +101,13 @@ func (s *Sink) applyIndexDump(item abstract.ChangeItem) error { if err != nil { return xerrors.Errorf("unable to check if index %q exists: %w", indexName, err) } - if response.StatusCode == 200 { + if response.StatusCode == http.StatusOK { s.existsIndexesMutex.Lock() defer s.existsIndexesMutex.Unlock() s.existsIndexes.Add(tableID) return nil } - if response.StatusCode != 404 { + if response.StatusCode != http.StatusNotFound { return xerrors.Errorf("wrong status code when checking index %q: %s", indexName, response.String()) } @@ -162,7 +162,7 @@ func makeIndexBodyFromChangeItem(changeItem abstract.ChangeItem) ([]byte, error) } // json.Marshal converts []byte to base64 form. -// bytesToStringInMapValues should fix it +// bytesToStringInMapValues should fix it. func bytesToStringInMapValues(itemMap map[string]interface{}) { if itemMap == nil { return @@ -215,10 +215,10 @@ func sanitizeKeysInMap(in map[string]interface{}) []map[string]interface{} { func sanitizeMapKey(in string) string { runes := []rune(in) - var outStringLen = 0 + outStringLen := 0 - var startCopyStr = 0 - var isEmptyCopyStr = true + startCopyStr := 0 + isEmptyCopyStr := true for i := 0; i <= len(runes); i++ { if i == len(runes) || runes[i] == '.' { if !isEmptyCopyStr { @@ -300,7 +300,7 @@ func (s *Sink) pushBatch(changeItems []abstract.ChangeItem) error { if len(changeItems) == 0 { return nil } - var indexResult = make(chan error) + indexResult := make(chan error) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { diff --git a/pkg/providers/elastic/storage.go b/pkg/providers/elastic/storage.go index bc696884..dcfeff67 100644 --- a/pkg/providers/elastic/storage.go +++ b/pkg/providers/elastic/storage.go @@ -182,7 +182,7 @@ func (s *Storage) TableList(includeTableFilter abstract.IncludeTableList) (abstr tableID := abstract.TableID{Namespace: "", Name: index} tables[tableID] = abstract.TableInfo{ - EtaRow: uint64(etaRow), + EtaRow: etaRow, IsView: false, Schema: abstract.NewTableSchema(schema.Columns), } diff --git a/pkg/providers/error_codes.go b/pkg/providers/error_codes.go index e4aedcd3..5083693d 100644 --- a/pkg/providers/error_codes.go +++ b/pkg/providers/error_codes.go @@ -7,13 +7,13 @@ var ( UnknownCluster = coded.Register("generic", "unknown_cluster") InvalidCredential = coded.Register("generic", "invalid_credentials") - // MissingData means that user asked for a table / topic / object wich is not exists on a source side + // MissingData means that user asked for a table / topic / object wich is not exists on a source side. MissingData = coded.Register("generic", "missing_data") - // DataOutOfRange means data type is correct but the value is outside the supported range + // DataOutOfRange means data type is correct but the value is outside the supported range. DataOutOfRange = coded.Register("data", "out_of_range") - // UnsupportedConversion means the source data type cannot be converted into the target data type + // UnsupportedConversion means the source data type cannot be converted into the target data type. UnsupportedConversion = coded.Register("data", "unsupported_type_conversion") - // DataValueError means there is something wrong with the value itself (i.e value []byte("foo") cannot be put into Decimal field) + // DataValueError means there is something wrong with the value itself (i.e value []byte("foo") cannot be put into Decimal field). DataValueError = coded.Register("data", "value_error") ) diff --git a/pkg/providers/eventhub/provider.go b/pkg/providers/eventhub/provider.go index 1657bf73..1085b724 100644 --- a/pkg/providers/eventhub/provider.go +++ b/pkg/providers/eventhub/provider.go @@ -22,7 +22,7 @@ func init() { providers.Register(ProviderType, New) } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Replication = (*Provider)(nil) ) diff --git a/pkg/providers/greenplum/connection.go b/pkg/providers/greenplum/connection.go index cd399008..a9745102 100644 --- a/pkg/providers/greenplum/connection.go +++ b/pkg/providers/greenplum/connection.go @@ -51,7 +51,7 @@ func Segment(index int) GPSegPointer { } // openPGStorage is a specification of a constructor of PostgreSQL storage for Greenplum. -// May modify the passed storage parameters +// May modify the passed storage parameters. func openPGStorage(ctx context.Context, config *postgres.PgStorageParams) (*postgres.Storage, error) { // this creates a TCP connection to the segment! var errs util.Errors @@ -111,7 +111,7 @@ func (s *Storage) getPgStorageParams(role GPRole) *postgres.PgStorageParams { return result } -// openPGStorageForAnyInPair connects to the current primary of the given high-availability pair AND checks it can execute SQL +// openPGStorageForAnyInPair connects to the current primary of the given high-availability pair AND checks it can execute SQL. func (s *Storage) openPGStorageForAnyInPair(ctx context.Context, sp GPSegPointer) (*postgres.Storage, error) { cfg := s.getPgStorageParams(sp.role) hap := s.config.Connection.OnPremises.SegByID(sp.seg) @@ -148,7 +148,7 @@ func (s *Storage) openPGStorageForAnyInPair(ctx context.Context, sp GPSegPointer return nil, xerrors.Errorf("failed to connect to any host in a highly-availabile pair:\t\t(primary): %v\t\t(mirror): %v", errs[0], errs[1]) } -// checkConnection checks whether the connection in `pgs` is valid (working) +// checkConnection checks whether the connection in `pgs` is valid (working). func checkConnection(ctx context.Context, pgs *postgres.Storage, expectedSP GPSegPointer) error { conn, err := pgs.Conn.Acquire(ctx) if err != nil { @@ -218,7 +218,7 @@ func segmentsFromGP(ctx context.Context, cpgs *postgres.Storage) ([]*GpHAP, erro return result, nil } -// isGPMirrorErr checks if the given `err` is due to a connection to a Greenplum instance in recovery mode +// isGPMirrorErr checks if the given `err` is due to a connection to a Greenplum instance in recovery mode. func isGPMirrorErr(err error, instanceNameForLog string) bool { var pgErr *pgconn.PgError if xerrors.As(err, &pgErr) { diff --git a/pkg/providers/greenplum/ddl_operations.go b/pkg/providers/greenplum/ddl_operations.go index 1af9f8cd..ad5424c4 100644 --- a/pkg/providers/greenplum/ddl_operations.go +++ b/pkg/providers/greenplum/ddl_operations.go @@ -140,7 +140,7 @@ func copyTmpTableToTarget(ctx context.Context, schemaCI *abstract.ChangeItem, co return nil } -// InsertQueryColumns returns a set of columns (fields, not values) for an INSERT query. Auto-generated columns are removed from the result +// InsertQueryColumns returns a set of columns (fields, not values) for an INSERT query. Auto-generated columns are removed from the result. func InsertQueryColumns(ci *abstract.ChangeItem) []string { result := make([]string, 0) for i := range ci.TableSchema.Columns() { @@ -154,12 +154,12 @@ func InsertQueryColumns(ci *abstract.ChangeItem) []string { return result } -// InsertFromSelectQuery returns a `INSERT INTO ... SELECT FROM` SQL query +// InsertFromSelectQuery returns a `INSERT INTO ... SELECT FROM` SQL query. func InsertFromSelectQuery(tableDst string, tableSrc string, columnNames []string) string { return fmt.Sprintf(`INSERT INTO %[1]s(%[2]s) SELECT %[2]s FROM %[3]s`, tableDst, strings.Join(columnNames, ", "), tableSrc) } -// DropTableQuery returns a `DROP TABLE IF EXISTS` SQL query. So the resulting query is "ensuring", not "imperative" +// DropTableQuery returns a `DROP TABLE IF EXISTS` SQL query. So the resulting query is "ensuring", not "imperative". func DropTableQuery(tableFQTN string) string { return fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tableFQTN) } diff --git a/pkg/providers/greenplum/flavour.go b/pkg/providers/greenplum/flavour.go index d173b5c2..5df39f1b 100644 --- a/pkg/providers/greenplum/flavour.go +++ b/pkg/providers/greenplum/flavour.go @@ -21,7 +21,7 @@ func NewGreenplumFlavourImpl(coordinatorOnlyMode bool, pgClassFilter func(bool, } } -// NewGreenplumFlavour constructs a flavour for PostgreSQL schema extractor +// NewGreenplumFlavour constructs a flavour for PostgreSQL schema extractor. func NewGreenplumFlavour(coordinatorOnlyMode bool) *GreenplumFlavour { return NewGreenplumFlavourImpl(coordinatorOnlyMode, pgClassFilter, pgClassRelsOnlyFilter) } diff --git a/pkg/providers/greenplum/gptx.go b/pkg/providers/greenplum/gptx.go index 13733aca..c9ecdcc5 100644 --- a/pkg/providers/greenplum/gptx.go +++ b/pkg/providers/greenplum/gptx.go @@ -13,7 +13,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// gpTx is a transaction with a connection +// gpTx is a transaction with a connection. type gpTx struct { tx pgx.Tx txMutex sync.Mutex @@ -60,7 +60,7 @@ func (s *gpTx) withConnection(f func(conn *pgx.Conn) error) error { return f(s.tx.Conn()) } -// CloseRollback ROLLBACKs the transaction +// CloseRollback ROLLBACKs the transaction. func (s *gpTx) CloseRollback(ctx context.Context) error { if s.closed { return nil @@ -76,7 +76,7 @@ func (s *gpTx) CloseRollback(ctx context.Context) error { return nil } -// CloseCommit first tries to COMMIT transaction. If an error is encountered, the transaction is ROLLBACKed +// CloseCommit first tries to COMMIT transaction. If an error is encountered, the transaction is ROLLBACKed. func (s *gpTx) CloseCommit(ctx context.Context) error { if s.closed { return nil diff --git a/pkg/providers/greenplum/model_gp_source.go b/pkg/providers/greenplum/model_gp_source.go index b84c8ee0..358661f5 100644 --- a/pkg/providers/greenplum/model_gp_source.go +++ b/pkg/providers/greenplum/model_gp_source.go @@ -39,9 +39,6 @@ type GpSourceAdvancedProps struct { // EnforceConsistency enables *enforcement* of consistent snapshot. When it is not set, the user is responsible for snapshot consistency EnforceConsistency bool - batchLimitRows int // deprecated: is not used anymore - batchLimitSize model.BytesSize // deprecated: is not used anymore - ServiceSchema string // AllowCoordinatorTxFailure disables coordinator TX monitoring (liveness monitor) and enables the transfer to finish snapshot successfully even if the coordinator TX fails @@ -161,7 +158,7 @@ func (s *GpCluster) SegByID(id int) *GpHAP { return s.Segments[id] } -// GpHAP stands for "Greenplum Highly Available host Pair" +// GpHAP stands for "Greenplum Highly Available host Pair". type GpHAP struct { Primary *GpHP Mirror *GpHP @@ -204,7 +201,7 @@ func GpHAPFromGreenplumUIHAPair(hap greenplumHAPair) *GpHAP { return pair } -// GpHP stands for "Greenplum Host/Port" +// GpHP stands for "Greenplum Host/Port". type GpHP struct { Host string Port int @@ -217,7 +214,7 @@ func NewGpHP(host string, port int) *GpHP { } } -// NewGpHpWithMDBReplacement replaces domain names for Cloud Preprod & Prod and returns a new host-port pair +// NewGpHpWithMDBReplacement replaces domain names for Cloud Preprod & Prod and returns a new host-port pair. func NewGpHpWithMDBReplacement(host string, port int) *GpHP { if mdbPreprodDomainRe.MatchString(host) { host = mdbPreprodDomainRe.ReplaceAllLiteralString(host, mdbServiceDomainExternalCloud) diff --git a/pkg/providers/greenplum/mutexed_postgreses.go b/pkg/providers/greenplum/mutexed_postgreses.go index b0ecf6d1..10cfaccc 100644 --- a/pkg/providers/greenplum/mutexed_postgreses.go +++ b/pkg/providers/greenplum/mutexed_postgreses.go @@ -40,7 +40,7 @@ func (s *mutexedPostgreses) Close() { } } -// PGStorage returns a live PG storage or an error +// PGStorage returns a live PG storage or an error. func (s *Storage) PGStorage(ctx context.Context, sp GPSegPointer) (*postgres.Storage, error) { s.postgreses.mutex.Lock() defer s.postgreses.mutex.Unlock() @@ -124,7 +124,7 @@ func (s *Storage) ensureCompleteClusterData(ctx context.Context) error { return nil } -// TotalSegments returns the actual total number of segments in Greenplum cluster. Never returns `0` +// TotalSegments returns the actual total number of segments in Greenplum cluster. Never returns `0`. func (s *Storage) TotalSegments(ctx context.Context) (int, error) { s.postgreses.mutex.Lock() defer s.postgreses.mutex.Unlock() diff --git a/pkg/providers/greenplum/progress.go b/pkg/providers/greenplum/progress.go index 6250a0a4..f9513cf8 100644 --- a/pkg/providers/greenplum/progress.go +++ b/pkg/providers/greenplum/progress.go @@ -4,7 +4,7 @@ import "github.com/doublecloud/transfer/pkg/abstract" const EtaRowPartialProgress = 1 << 20 -// ComposePartialProgressFn allows to transform progress by part into total progress by multiple parts +// ComposePartialProgressFn allows to transform progress by part into total progress by multiple parts. func ComposePartialProgressFn(base abstract.LoadProgress, completedParts uint, totalParts uint, totalEta uint64) abstract.LoadProgress { return func(current uint64, progress uint64, total uint64) { inPartProgress := (float64(progress) / float64(total)) diff --git a/pkg/providers/greenplum/progress_test.go b/pkg/providers/greenplum/progress_test.go index 2ca2b77b..63fc1cfd 100644 --- a/pkg/providers/greenplum/progress_test.go +++ b/pkg/providers/greenplum/progress_test.go @@ -39,9 +39,9 @@ const ( ) const ( - // TotalTest is the number of rows in the current partition + // TotalTest is the number of rows in the current partition. TotalTest uint64 = 12 - // EtaRowTest is the total number of rows (in all partitions) + // EtaRowTest is the total number of rows (in all partitions). EtaRowTest uint64 = 120 ) diff --git a/pkg/providers/greenplum/provider.go b/pkg/providers/greenplum/provider.go index 06bfe41e..bf373750 100644 --- a/pkg/providers/greenplum/provider.go +++ b/pkg/providers/greenplum/provider.go @@ -58,7 +58,7 @@ const ( ProviderType = abstract.ProviderType("gp") ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Snapshot = (*Provider)(nil) _ providers.Sinker = (*Provider)(nil) diff --git a/pkg/providers/greenplum/segpointerpool.go b/pkg/providers/greenplum/segpointerpool.go index c107898b..2626dfec 100644 --- a/pkg/providers/greenplum/segpointerpool.go +++ b/pkg/providers/greenplum/segpointerpool.go @@ -4,7 +4,7 @@ import ( "math/rand" ) -// SegPointerPool is a set of Greenplum storage segment pointers with additional functions +// SegPointerPool is a set of Greenplum storage segment pointers with additional functions. type SegPointerPool struct { // pool is a set of segments this sink uses to INSERT data to. Is initialized at the first push of a row changeitem pool []GPSegPointer @@ -12,7 +12,7 @@ type SegPointerPool struct { nextRoundRobinIndex int } -// NewRandomSegPointerPool constructs a pool of the given size, the first element of which is chosen randomly from a ring consisting of the given total number of segments +// NewRandomSegPointerPool constructs a pool of the given size, the first element of which is chosen randomly from a ring consisting of the given total number of segments. func NewRandomSegPointerPool(totalSegments int, size int) *SegPointerPool { result := &SegPointerPool{ pool: make([]GPSegPointer, size), diff --git a/pkg/providers/greenplum/sink.go b/pkg/providers/greenplum/sink.go index 117cec53..9059851d 100644 --- a/pkg/providers/greenplum/sink.go +++ b/pkg/providers/greenplum/sink.go @@ -149,7 +149,7 @@ func (s *Sink) processRowChangeItem(ctx context.Context, changeItem *abstract.Ch } // setTemporaryTableForChangeItem sets the temporary table as a target for the given ChangeItem. -// If an error is returned, ChangeItem is left unchanged +// If an error is returned, ChangeItem is left unchanged. func setTemporaryTableForChangeItem(changeItem *abstract.ChangeItem) { changeItem.Schema, changeItem.Table = temporaryTable(changeItem.Schema, changeItem.Table) } @@ -252,7 +252,7 @@ func (s *Sink) pushChangeItemsToSegment(ctx context.Context, seg GPSegPointer, c return nil } -// processCleanupChangeItem flushes ChangeItems and pushes the given one to coordinator +// processCleanupChangeItem flushes ChangeItems and pushes the given one to coordinator. func (s *Sink) processCleanupChangeItem(ctx context.Context, changeItem *abstract.ChangeItem) error { if err := s.flushRowChangeItems(ctx); err != nil { return xerrors.Errorf("failed to flush rows: %w", err) diff --git a/pkg/providers/greenplum/sink_test.go b/pkg/providers/greenplum/sink_test.go index f10f0d68..a1f2357d 100644 --- a/pkg/providers/greenplum/sink_test.go +++ b/pkg/providers/greenplum/sink_test.go @@ -49,7 +49,7 @@ func makeTestChangeItem(t *testing.T, colNames []string, colValues []interface{} } // TestGpParallel checks that sink pushes values to segments asynchronously when destination has more than 12 segments -// The value of required segments is due to the SegPoolShare field of sink +// The value of required segments is due to the SegPoolShare field of sink. func TestGpParallel(t *testing.T) { fakeSinks := new(fakePgSinks) fakeSinks.totalSegmentsCached = 13 diff --git a/pkg/providers/greenplum/storage.go b/pkg/providers/greenplum/storage.go index 2e18e429..58943417 100644 --- a/pkg/providers/greenplum/storage.go +++ b/pkg/providers/greenplum/storage.go @@ -400,7 +400,7 @@ func (s *Storage) SetShardingContext(shardedState []byte) error { } // Named BeginGPSnapshot to NOT match abstract.SnapshotableStorage; -// BeginGPSnapshot starts a Greenplum cluster-global transaction; +// BeginGPSnapshot starts a Greenplum cluster-global transaction;. func (s *Storage) BeginGPSnapshot(ctx context.Context, tables []abstract.TableDescription) error { if err := s.ensureCoordinatorTx(ctx); err != nil { return xerrors.Errorf("failed to start a transaction on Greenplum %s: %w", Coordinator().String(), err) @@ -452,7 +452,7 @@ func (s *Storage) ensureCoordinatorTx(ctx context.Context) error { } // Named EndGPSnapshot to NOT match abstract.SnapshotableStorage; -// EndGPSnapshot ceases a Greenplum cluster-global transaction; +// EndGPSnapshot ceases a Greenplum cluster-global transaction;. func (s *Storage) EndGPSnapshot(ctx context.Context) error { s.livenessMonitor.Close() diff --git a/pkg/providers/kafka/model_connection.go b/pkg/providers/kafka/model_connection.go index 39563dcc..b3cd5638 100644 --- a/pkg/providers/kafka/model_connection.go +++ b/pkg/providers/kafka/model_connection.go @@ -79,7 +79,7 @@ func (o *KafkaConnectionOptions) TLSConfig() (*tls.Config, error) { }, nil } -// BrokersHostnames returns a list of brokers' hostnames +// BrokersHostnames returns a list of brokers' hostnames. func (o *KafkaConnectionOptions) BrokersHostnames() ([]string, error) { result := make([]string, len(o.Brokers)) for i, b := range o.Brokers { diff --git a/pkg/providers/kafka/model_source.go b/pkg/providers/kafka/model_source.go index a4dacdad..bf0d5b78 100644 --- a/pkg/providers/kafka/model_source.go +++ b/pkg/providers/kafka/model_source.go @@ -10,21 +10,21 @@ import ( const DefaultAuth = "admin" type KafkaSource struct { - Connection *KafkaConnectionOptions - Auth *KafkaAuth - Topic string - GroupTopics []string - Transformer *model.DataTransformOptions + Connection *KafkaConnectionOptions `json:"connection"` + Auth *KafkaAuth `json:"auth"` + Topic string `json:"topic"` + GroupTopics []string `json:"group_topics"` + Transformer *model.DataTransformOptions `json:"transformer"` - BufferSize model.BytesSize // it's not some real buffer size - see comments to waitLimits() method in kafka-source + BufferSize model.BytesSize `json:"buffer_size"` // it's not some real buffer size - see comments to waitLimits() method in kafka-source - SecurityGroupIDs []string + SecurityGroupIDs []string `json:"security_group_ids"` - ParserConfig map[string]interface{} - IsHomo bool // enabled kafka mirror protocol which can work only with kafka target - SynchronizeIsNeeded bool // true, if we need to send synchronize events on releasing partitions + ParserConfig map[string]interface{} `json:"parser_config"` + IsHomo bool `json:"is_homo"` // enabled kafka mirror protocol which can work only with kafka target + SynchronizeIsNeeded bool `json:"synchronize_is_needed"` // true, if we need to send synchronize events on releasing partitions - OffsetPolicy OffsetPolicy // specify from what topic part start message consumption + OffsetPolicy OffsetPolicy `json:"offset_policy"` // specify from what topic part start message consumption } type OffsetPolicy string diff --git a/pkg/providers/kafka/provider.go b/pkg/providers/kafka/provider.go index 8f2f5a0e..9bf4fc6e 100644 --- a/pkg/providers/kafka/provider.go +++ b/pkg/providers/kafka/provider.go @@ -34,7 +34,7 @@ func init() { const ProviderType = abstract.ProviderType("kafka") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Replication = (*Provider)(nil) _ providers.Sniffer = (*Provider)(nil) diff --git a/pkg/providers/kafka/reader.go b/pkg/providers/kafka/reader.go index 05abf4fc..3d8b4949 100644 --- a/pkg/providers/kafka/reader.go +++ b/pkg/providers/kafka/reader.go @@ -22,7 +22,7 @@ func (r *franzReader) CommitMessages(ctx context.Context, msgs ...kgo.Record) er return r.client.CommitRecords(ctx, forCommit...) } -// FetchMessage doesn't return pointer to struct, because franz-go has no guarantees about the returning values +// FetchMessage doesn't return pointer to struct, because franz-go has no guarantees about the returning values. func (r *franzReader) FetchMessage(ctx context.Context) (kgo.Record, error) { fetcher := r.client.PollRecords(ctx, 1) err := fetcher.Err() diff --git a/pkg/providers/kafka/source.go b/pkg/providers/kafka/source.go index 54774c47..49b5284a 100644 --- a/pkg/providers/kafka/source.go +++ b/pkg/providers/kafka/source.go @@ -25,9 +25,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -var ( - noDataErr = xerrors.NewSentinel("no data") -) +var noDataErr = xerrors.NewSentinel("no data") type reader interface { CommitMessages(ctx context.Context, msgs ...kgo.Record) error @@ -231,7 +229,6 @@ func (p *Source) Fetch() ([]abstract.ChangeItem, error) { res = data } if p.executor != nil { - res = nil // reset result transformed, err := p.executor.Do(data) if err != nil { return nil, xerrors.Errorf("failed to execute a batch of changeitems: %w", err) diff --git a/pkg/providers/kafka/writer/writer_mock.go b/pkg/providers/kafka/writer/writer_mock.go index 4aea7302..14fe4b94 100644 --- a/pkg/providers/kafka/writer/writer_mock.go +++ b/pkg/providers/kafka/writer/writer_mock.go @@ -13,11 +13,11 @@ import ( tls "crypto/tls" reflect "reflect" - log "go.ytsaurus.tech/library/go/core/log" queue "github.com/doublecloud/transfer/pkg/serializer/queue" kafka "github.com/segmentio/kafka-go" sasl "github.com/segmentio/kafka-go/sasl" gomock "go.uber.org/mock/gomock" + log "go.ytsaurus.tech/library/go/core/log" ) // MockAbstractWriter is a mock of AbstractWriter interface. diff --git a/pkg/providers/kinesis/consumer/group.go b/pkg/providers/kinesis/consumer/group.go index aa084383..af9d24ef 100644 --- a/pkg/providers/kinesis/consumer/group.go +++ b/pkg/providers/kinesis/consumer/group.go @@ -6,7 +6,7 @@ import ( "github.com/aws/aws-sdk-go/service/kinesis" ) -// Group interface used to manage which shard to process +// Group interface used to manage which shard to process. type Group interface { Start(ctx context.Context, shardc chan *kinesis.Shard) GetCheckpoint(streamName, shardID string) (string, error) diff --git a/pkg/providers/kinesis/consumer/group_all.go b/pkg/providers/kinesis/consumer/group_all.go index cf59c0e1..a3ae44d9 100644 --- a/pkg/providers/kinesis/consumer/group_all.go +++ b/pkg/providers/kinesis/consumer/group_all.go @@ -13,7 +13,7 @@ import ( ) // NewAllGroup returns an intitialized AllGroup for consuming -// all shards on a stream +// all shards on a stream. func NewAllGroup(ksis kinesisiface.KinesisAPI, store Store, streamName string, logger log.Logger) *AllGroup { return &AllGroup{ Store: store, @@ -87,7 +87,7 @@ func (g *AllGroup) findNewShards(shardc chan *kinesis.Shard) { } } -// listShards pulls a list of shard IDs from the kinesis api +// listShards pulls a list of shard IDs from the kinesis api. func listShards(ksis kinesisiface.KinesisAPI, streamName string) ([]*kinesis.Shard, error) { var ss []*kinesis.Shard var listShardsInput = &kinesis.ListShardsInput{ diff --git a/pkg/providers/kinesis/consumer/options.go b/pkg/providers/kinesis/consumer/options.go index 7a68b732..0b7993f5 100644 --- a/pkg/providers/kinesis/consumer/options.go +++ b/pkg/providers/kinesis/consumer/options.go @@ -6,45 +6,45 @@ import ( "github.com/aws/aws-sdk-go/service/kinesis/kinesisiface" ) -// Option is used to override defaults when creating a new Consumer +// Option is used to override defaults when creating a new Consumer. type Option func(*Consumer) -// WithGroup overrides the default storage +// WithGroup overrides the default storage. func WithGroup(group Group) Option { return func(c *Consumer) { c.group = group } } -// WithStore overrides the default storage +// WithStore overrides the default storage. func WithStore(store Store) Option { return func(c *Consumer) { c.store = store } } -// WithClient overrides the default client +// WithClient overrides the default client. func WithClient(client kinesisiface.KinesisAPI) Option { return func(c *Consumer) { c.client = client } } -// WithShardIteratorType overrides the starting point for the consumer +// WithShardIteratorType overrides the starting point for the consumer. func WithShardIteratorType(t string) Option { return func(c *Consumer) { c.initialShardIteratorType = t } } -// WithTimestamp overrides the starting point for the consumer +// WithTimestamp overrides the starting point for the consumer. func WithTimestamp(t time.Time) Option { return func(c *Consumer) { c.initialTimestamp = &t } } -// WithScanInterval overrides the scan interval for the consumer +// WithScanInterval overrides the scan interval for the consumer. func WithScanInterval(d time.Duration) Option { return func(c *Consumer) { c.scanInterval = d @@ -53,7 +53,7 @@ func WithScanInterval(d time.Duration) Option { // WithMaxRecords overrides the maximum number of records to be // returned in a single GetRecords call for the consumer (specify a -// value of up to 10,000) +// value of up to 10,000). func WithMaxRecords(n int64) Option { return func(c *Consumer) { c.maxRecords = n diff --git a/pkg/providers/kinesis/consumer/store.go b/pkg/providers/kinesis/consumer/store.go index 4b4a9d40..b95fbd2f 100644 --- a/pkg/providers/kinesis/consumer/store.go +++ b/pkg/providers/kinesis/consumer/store.go @@ -1,12 +1,12 @@ package consumer -// Store interface used to persist scan progress +// Store interface used to persist scan progress. type Store interface { GetCheckpoint(streamName, shardID string) (string, error) SetCheckpoint(streamName, shardID, sequenceNumber string) error } -// noopStore implements the storage interface with discard +// noopStore implements the storage interface with discard. type noopStore struct{} func (n noopStore) GetCheckpoint(string, string) (string, error) { return "", nil } diff --git a/pkg/providers/kinesis/source.go b/pkg/providers/kinesis/source.go index b4f7f686..8e0a2798 100644 --- a/pkg/providers/kinesis/source.go +++ b/pkg/providers/kinesis/source.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "hash/fnv" - "math/big" "strconv" "strings" "sync" @@ -84,7 +83,7 @@ func (s *Source) addInflight(size int) { func (s *Source) reduceInflight(size int) { s.inflightMutex.Lock() defer s.inflightMutex.Unlock() - s.inflightBytes = s.inflightBytes - size + s.inflightBytes -= size } func (s *Source) Run(sink abstract.AsyncSink) error { @@ -216,21 +215,6 @@ const ( SequenceMask = (1 << 4) - 1 ) -// parseSeqNo try to extract lsn from seq-no -// the sequenceNumber in Kinesis streams is only guaranteed to be unique within each shard (partition key is what determines the shard). -// but we know that for certain version (186 bit length) we can extract sequence mask -// this code is extracted from here: https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/SequenceNumberValidator.java#L39 -func parseSeqNo(id string) int64 { - bigint, ok := new(big.Int).SetString(id, 10) - if !ok { - return hash(id) - } - if bigint.BitLen() != ExpectedBitLength { - return hash(id) - } - return bigint.Rsh(bigint, SequenceMask).Int64() -} - func hash(id string) int64 { algorithm := fnv.New64a() _, _ = algorithm.Write([]byte(id)) @@ -321,5 +305,4 @@ func NewSource( consumer: c, lastError: nil, }, nil - } diff --git a/pkg/providers/mongo/batcher.go b/pkg/providers/mongo/batcher.go index 4f41e720..b41060fa 100644 --- a/pkg/providers/mongo/batcher.go +++ b/pkg/providers/mongo/batcher.go @@ -10,7 +10,6 @@ import ( "github.com/doublecloud/transfer/library/go/core/xerrors" "github.com/doublecloud/transfer/pkg/abstract" "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readconcern" "go.ytsaurus.tech/library/go/core/log" @@ -40,10 +39,10 @@ type sizedFullDocument struct { size int } -// FullDocumentExtractor to get default instance of this type use MakeDefaultFullDocumentExtractor +// FullDocumentExtractor to get default instance of this type use MakeDefaultFullDocumentExtractor. type FullDocumentExtractor func(ctx context.Context, ns Namespace, keyList bson.A) ([]sizedFullDocument, error) -// MakeDefaultFullDocumentExtractor Constructs default document extractor for batcher that uses Mongo connection +// MakeDefaultFullDocumentExtractor Constructs default document extractor for batcher that uses Mongo connection. func MakeDefaultFullDocumentExtractor(client *MongoClientWrapper) FullDocumentExtractor { return func(ctx context.Context, ns Namespace, keyList bson.A) ([]sizedFullDocument, error) { filter := bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: keyList}}}} @@ -82,7 +81,7 @@ func MakeDefaultFullDocumentExtractor(client *MongoClientWrapper) FullDocumentEx // it uses namespace, key, size of key and timestamp in order to form // batch of keys, then it uploads documents directly from source // database and produce new change events that dispatch into specified -// in NewKeyBatcher's fullDocumentPusher argument +// in NewKeyBatcher's fullDocumentPusher argument. type keyBatcher struct { logger log.Logger fullDocumentExtractor FullDocumentExtractor @@ -106,7 +105,7 @@ type keyBatcher struct { cancel func() } -// Close do not forget to defer Close() this resource +// Close do not forget to defer Close() this resource. func (f *keyBatcher) Close() error { f.closeOnce.Do(func() { f.cancel() @@ -140,7 +139,6 @@ func (f *keyBatcher) PushKeyChangeEvent(keyEvent *keyChangeEvent) error { defer f.batchMutex.Unlock() return f.flush() // f.batchMutex taken }() - if err != nil { return xerrors.Errorf("Couldn't flush batch: %w", err) } @@ -191,7 +189,7 @@ func (f *keyBatcher) periodicFlusher() { // putInBatch responsible for adding key in namespace for update. // do not use it directly as client -// byteSize needed to track request BSON size +// byteSize needed to track request BSON size. func (f *keyBatcher) putInBatch(chEvent *keyChangeEvent) error { event := chEvent.keyEvent byteSize := uint64(chEvent.size) @@ -219,7 +217,7 @@ func (f *keyBatcher) putInBatch(chEvent *keyChangeEvent) error { } if oplogEvent, exists := f.batch[kd]; exists { - if primitive.CompareTimestamp(oplogEvent.keyEvent.ClusterTime, event.ClusterTime) <= 0 { + if oplogEvent.keyEvent.ClusterTime.Compare(event.ClusterTime) <= 0 { // refresh key time to newer time and register collapse oplogEvent.keyEvent.OperationType = event.OperationType oplogEvent.keyEvent.ClusterTime = event.ClusterTime @@ -317,7 +315,7 @@ func (f *keyBatcher) pushUnorderedChangeEvents(changeEventSlice []*changeEvent) sort.Slice(changeEventSlice, func(i, j int) bool { tsi := changeEventSlice[i].event.ClusterTime tsj := changeEventSlice[j].event.ClusterTime - return primitive.CompareTimestamp(tsi, tsj) > 0 + return tsi.Compare(tsj) > 0 }) // serially provide events to pusher for _, ce := range changeEventSlice { @@ -359,7 +357,7 @@ func (f *keyBatcher) processBatch(batchAndMetainfo keyBatch) error { for _, sizedFullDoc := range sizedFullDocuments { fullDocument := sizedFullDoc.document // build extracted fullDocument descriptor - fullDocumentKD := makeKeyDescriptor(ns, fullDocument.Map()["_id"]) + fullDocumentKD := makeKeyDescriptor(ns, fullDocument.Map()["_id"]) //nolint:staticcheck // extract timestamp lastOplogEvent, ok := batchAndMetainfo.batch[fullDocumentKD] if !ok { @@ -375,7 +373,7 @@ func (f *keyBatcher) processBatch(batchAndMetainfo keyBatch) error { } // NewKeyBatcher -// nil parameters are default parameters +// nil parameters are default parameters. func NewKeyBatcher( ctx context.Context, logger log.Logger, diff --git a/pkg/providers/mongo/bson.go b/pkg/providers/mongo/bson.go index 19a74013..ad65297e 100644 --- a/pkg/providers/mongo/bson.go +++ b/pkg/providers/mongo/bson.go @@ -100,7 +100,7 @@ func (d *DExtension) Value(isHomo, preventJSONRepack bool) DValue { func (d *DExtension) Map() bson.M { if d.modified { - d.cache = d.D.Map() + d.cache = d.D.Map() //nolint:staticcheck d.modified = false } return d.cache @@ -125,7 +125,7 @@ func GetValueByPath(doc any, path string) (any, bool) { val := doc for _, key := range pathTokens { var elem any - ok := false + var ok bool switch v := val.(type) { case bson.M: elem, ok = getKeyM(v, key) diff --git a/pkg/providers/mongo/change_stream_watcher.go b/pkg/providers/mongo/change_stream_watcher.go index 6a996f8d..335708e2 100644 --- a/pkg/providers/mongo/change_stream_watcher.go +++ b/pkg/providers/mongo/change_stream_watcher.go @@ -38,7 +38,7 @@ type puChangeEvent struct { type changeEventPusher func(ctx context.Context, event *changeEvent) error // ChangeStreamWatcher produces changeEvents -// encapsulates method with which full documents of mongo collections are retrieved +// encapsulates method with which full documents of mongo collections are retrieved. type ChangeStreamWatcher interface { // Watch is one-shot method. After return all allocated resources should be freed with calling Close Watch(context.Context, changeEventPusher) error diff --git a/pkg/providers/mongo/client.go b/pkg/providers/mongo/client.go index 80186266..10dc511f 100644 --- a/pkg/providers/mongo/client.go +++ b/pkg/providers/mongo/client.go @@ -5,7 +5,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" + "os" "strings" "time" @@ -49,7 +49,7 @@ func (w *MongoClientWrapper) Close(ctx context.Context) error { var tmMongoConnectID atomic.Int64 // Connect function should be one and only one valid method of creation mongo client -// this method logs into 'lgr' about connection options +// this method logs into 'lgr' about connection options. func Connect(ctx context.Context, opts MongoConnectionOptions, lgr log.Logger) (*MongoClientWrapper, error) { if lgr == nil { lgr = logger.Log @@ -142,17 +142,12 @@ func Connect(ctx context.Context, opts MongoConnectionOptions, lgr log.Logger) ( } func newClient(ctx context.Context, connOpts *options.ClientOptions, lgr log.Logger) (*mongo.Client, error) { - client, err := mongo.NewClient(connOpts) + client, err := mongo.Connect(ctx, connOpts) isWithPassword := bool(connOpts != nil && connOpts.Auth != nil && len(connOpts.Auth.Password) > 0) if err != nil && isWithPassword && strings.Contains(err.Error(), connOpts.Auth.Password) { err = xerrors.New(strings.ReplaceAll(err.Error(), connOpts.Auth.Password, "")) - } - if err != nil { - return nil, xerrors.Errorf("unable to create mongo client: %w", err) - } - if err := client.Connect(ctx); err != nil { return nil, xerrors.Errorf("unable to connect mongo client: %w", err) } @@ -180,7 +175,7 @@ func getClusterInfo(endpoint MongoConnectionOptions) (hosts []string, sharded bo return nil, false, xerrors.Errorf("unable to resolve mongodb hosts: %w", err) } // step of choosing port, shard-ness and type of mongo instances to connect - port := 0 + var port int shardResolver, err := provider.ShardResolver(dbaas.ProviderTypeMongodb, endpoint.ClusterID) if err != nil { return nil, false, xerrors.Errorf("unable to init shard resolver: %w", err) @@ -221,6 +216,7 @@ func getClusterInfo(endpoint MongoConnectionOptions) (hosts []string, sharded bo } return hosts, sharded, nil } + func DriverConnectionSrvOptions(mongoConnectionOptions *MongoConnectionOptions) (*options.ClientOptions, error) { if len(mongoConnectionOptions.Hosts) != 1 { return nil, xerrors.Errorf("Cannot be empty or more than hosts in srv connection") @@ -256,6 +252,7 @@ func DriverConnectionSrvOptions(mongoConnectionOptions *MongoConnectionOptions) } return clientOptions, nil } + func DriverConnectionOptions(mongoConnectionOptions *MongoConnectionOptions) (*options.ClientOptions, error) { opts := options.ClientOptions{} hosts, sharded, err := getClusterInfo(*mongoConnectionOptions) @@ -307,7 +304,7 @@ func newTLSConfig(caCert TrustedCACertificate) (*tls.Config, error) { case CACertificatePEMFilePaths: caFilePaths := downcasted for _, cert := range caFilePaths { - pemFileContent, err := ioutil.ReadFile(string(cert)) + pemFileContent, err := os.ReadFile(string(cert)) if err != nil { return nil, xerrors.Errorf("Cannot read file %s: %w", cert, err) } diff --git a/pkg/providers/mongo/convert.go b/pkg/providers/mongo/convert.go index 7ed7fe28..36b017fd 100644 --- a/pkg/providers/mongo/convert.go +++ b/pkg/providers/mongo/convert.go @@ -8,7 +8,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) -// ValueInMongoFormat extracts the column value at the given index and converts it to MongoDB format +// ValueInMongoFormat extracts the column value at the given index and converts it to MongoDB format. func ValueInMongoFormat(item *abstract.ChangeItem, index int) (interface{}, error) { if item == nil { return nil, abstract.NewFatalError(xerrors.New("impossible to extract value from a nil item")) diff --git a/pkg/providers/mongo/database_document_key_watcher.go b/pkg/providers/mongo/database_document_key_watcher.go index 493a959f..b6cf1287 100644 --- a/pkg/providers/mongo/database_document_key_watcher.go +++ b/pkg/providers/mongo/database_document_key_watcher.go @@ -15,7 +15,7 @@ import ( // databaseDocumentKeyWatcher subscribes for changes in database // it tries to get only document id's to load changes -// using this ID's it loads documents directly +// using this ID's it loads documents directly. type databaseDocumentKeyWatcher struct { logger log.Logger databaseName string @@ -35,7 +35,7 @@ func (f *databaseDocumentKeyWatcher) Close(ctx context.Context) { } // Watch assumed to run in single thread -// you should call it exactly once +// you should call it exactly once. func (f *databaseDocumentKeyWatcher) Watch(ctx context.Context, changeEventPusher changeEventPusher) error { ctx, cancel := context.WithCancel(ctx) diff --git a/pkg/providers/mongo/database_full_document_watcher.go b/pkg/providers/mongo/database_full_document_watcher.go index 01617e7a..d22b06f8 100644 --- a/pkg/providers/mongo/database_full_document_watcher.go +++ b/pkg/providers/mongo/database_full_document_watcher.go @@ -16,7 +16,7 @@ import ( ) // namespaceOnlyWatcher subscribes for changes in database -// this is a default watcher (old behaviour) +// this is a default watcher (old behaviour). type databaseFullDocumentWatcher struct { logger log.Logger database *mongo.Database diff --git a/pkg/providers/mongo/fallback_dvalue_json_repack.go b/pkg/providers/mongo/fallback_dvalue_json_repack.go index 4dfb2ec0..4880f3cb 100644 --- a/pkg/providers/mongo/fallback_dvalue_json_repack.go +++ b/pkg/providers/mongo/fallback_dvalue_json_repack.go @@ -32,7 +32,7 @@ func init() { } val, ok := ci.ColumnValues[i].(DValue) if !ok { - return ci, xerrors.Errorf("expected column value of type %T, actual type: %T", *new(DValue), val) + return ci, xerrors.Errorf("expected column value of type %T, actual type: %T", DValue{}, val) } fallbackApplied = true newVal, err := val.RepackValue() diff --git a/pkg/providers/mongo/local_oplog_rs_watcher.go b/pkg/providers/mongo/local_oplog_rs_watcher.go index 916813f6..5a0fca50 100644 --- a/pkg/providers/mongo/local_oplog_rs_watcher.go +++ b/pkg/providers/mongo/local_oplog_rs_watcher.go @@ -153,8 +153,8 @@ func (w localOplogRsWatcher) openOplogCursor(ctx context.Context, ts primitive.T func (w localOplogRsWatcher) watchBatchFrom(ctx context.Context, ts primitive.Timestamp, batcher *keyBatcher) (until primitive.Timestamp, eventsRead int, err error) { w.logger.Infof("Begin watching batch from time %v", FromMongoTimestamp(ts)) // check that timestamp is still in oplog - from, to, err := GetLocalOplogInterval(ctx, w.client) - if primitive.CompareTimestamp(ts, from) < 0 { + from, to, _ := GetLocalOplogInterval(ctx, w.client) + if ts.Compare(from) < 0 { return until, eventsRead, xerrors.Errorf("local.oplog.rs watcher is out of timeline. Requested timestamp %v, actual interval: [%v, %v]", FromMongoTimestamp(ts), FromMongoTimestamp(from), diff --git a/pkg/providers/mongo/model_mongo_connection_options.go b/pkg/providers/mongo/model_mongo_connection_options.go index f7a3ddd1..a75e6e24 100644 --- a/pkg/providers/mongo/model_mongo_connection_options.go +++ b/pkg/providers/mongo/model_mongo_connection_options.go @@ -15,7 +15,7 @@ type MongoConnectionOptions struct { SRVMode bool } -// IsDocDB check if we connect to amazon doc DB +// IsDocDB check if we connect to amazon doc DB. func (o MongoConnectionOptions) IsDocDB() bool { for _, h := range o.Hosts { if strings.Contains(h, "docdb.amazonaws.com") { diff --git a/pkg/providers/mongo/model_mongo_source.go b/pkg/providers/mongo/model_mongo_source.go index b5ed4595..bafd144b 100644 --- a/pkg/providers/mongo/model_mongo_source.go +++ b/pkg/providers/mongo/model_mongo_source.go @@ -15,12 +15,12 @@ import ( ) const ( - // TODO(@kry127) tune constant, or make more deterministic approach of measuring pipeline BSON size + // TODO(@kry127) tune constant, or make more deterministic approach of measuring pipeline BSON size. DefaultKeySizeThreshold = 4 * 1024 * 1024 // soft upper border for batch size in bytes (if one key is bigger, it'll fit) DefaultBatchFlushInterval = 5 * time.Second // batch is guaranteed to be flushed every five seconds, but it may flush more frequently DefaultBatchSizeLimit = 500 // limit of amount of events inside one batch - // desired part size of collection + // desired part size of collection. TablePartByteSize = 1024 * 1024 * 1024 DataTransferSystemDatabase = "__data_transfer" @@ -182,7 +182,7 @@ var ( ) // BuildPipeline returns mongo pipeline that should be able -// to filter the oplog from unwanted changes in database 'forDatabaseName' +// to filter the oplog from unwanted changes in database 'forDatabaseName'. func (f MongoCollectionFilter) BuildPipeline(forDatabaseName string) (mongo.Pipeline, error) { excluded := map[string]struct{}{} included := map[string]struct{}{} diff --git a/pkg/providers/mongo/mongo_recipe.go b/pkg/providers/mongo/mongo_recipe.go index 0a37cc19..fc0cc628 100644 --- a/pkg/providers/mongo/mongo_recipe.go +++ b/pkg/providers/mongo/mongo_recipe.go @@ -131,7 +131,7 @@ func GetIntFromEnv(varName string) int { return val } -// PostgresContainer represents the postgres container type used in the module +// PostgresContainer represents the postgres container type used in the module. type MongoContainer struct { testcontainers.Container user string diff --git a/pkg/providers/mongo/namespace_only_watcher.go b/pkg/providers/mongo/namespace_only_watcher.go index 5dc84418..d878591a 100644 --- a/pkg/providers/mongo/namespace_only_watcher.go +++ b/pkg/providers/mongo/namespace_only_watcher.go @@ -14,14 +14,14 @@ import ( // oneshotNamespaceRetriever subscribes for namespace changes ONLY in database // it is used when oplog on database is no longer operational (does not goes further, i.e. BSONObjTooLarge) // For example, use when server-side mongo cannot pack change keyEvent into single BSON because it is too large -// Assumed, that it reads the same failed keyEvent from previously failed change stream +// Assumed, that it reads the same failed keyEvent from previously failed change stream. type oneshotNamespaceRetriever struct { logger log.Logger database *mongo.Database changeStream *mongo.ChangeStream } -// TODO can be used for skipping bad documents +// TODO can be used for skipping bad documents. func (f *oneshotNamespaceRetriever) GetResumeToken() bson.Raw { return f.changeStream.ResumeToken() } @@ -55,7 +55,7 @@ func (f *oneshotNamespaceRetriever) GetNamespace(ctx context.Context) (*Namespac } // NewOneshotNamespaceRetriever creates namespace-only watcher -// recovery point (resume token) should be specified precisely, or current time will be used +// recovery point (resume token) should be specified precisely, or current time will be used. func NewOneshotNamespaceRetriever(s *mongoSource, resumeToken bson.Raw, dbName string) (*oneshotNamespaceRetriever, error) { rollbacks := util.Rollbacks{} defer rollbacks.Do() diff --git a/pkg/providers/mongo/oplog_v2_parser.go b/pkg/providers/mongo/oplog_v2_parser.go index ce1801e8..6f87692b 100644 --- a/pkg/providers/mongo/oplog_v2_parser.go +++ b/pkg/providers/mongo/oplog_v2_parser.go @@ -9,7 +9,7 @@ import ( const OplogProtocolVersion = 2 -// protocol is actual for oplog v2, mongo v3.6.23 +// protocol is actual for oplog v2, mongo v3.6.23. type oplogRsChangeEventV2 struct { Timestamp primitive.Timestamp `bson:"ts"` Hash int64 `bson:"h"` @@ -66,7 +66,7 @@ func newNoopChangeEvent(clusterTime primitive.Timestamp) *KeyChangeEvent { } // Note: this function DOES NOT extract full document for one reason: -// update commands do not contain full documents, but mongo commands to modify existing document +// update commands do not contain full documents, but mongo commands to modify existing document. func (e oplogRsChangeEventV2) toMongoKeyChangeEvent(log log.Logger) (*KeyChangeEvent, error) { if e.Version != OplogProtocolVersion { return nil, xerrors.Errorf("version %d of oplog protocol is not supported. Oplog keyEvent: %v", e.Version, e) diff --git a/pkg/providers/mongo/provider.go b/pkg/providers/mongo/provider.go index 78e9f6d6..68b4c6f5 100644 --- a/pkg/providers/mongo/provider.go +++ b/pkg/providers/mongo/provider.go @@ -44,17 +44,9 @@ const ( ClusterTimeCollName = "__dt_cluster_time" ) -func isSystemTable(tableName string) bool { - switch tableName { - case SystemDatabase, ClusterTimeCollName: - return true - } - return false -} - const ProviderType = abstract.ProviderType("mongo") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) _ providers.Replication = (*Provider)(nil) diff --git a/pkg/providers/mongo/sharding_storage.go b/pkg/providers/mongo/sharding_storage.go index 0b0841aa..2d6979fc 100644 --- a/pkg/providers/mongo/sharding_storage.go +++ b/pkg/providers/mongo/sharding_storage.go @@ -18,10 +18,12 @@ const ( maxDelimiters = 31 ) -type ShardingFilter bson.D -type delimiter interface { - isDelimiter() -} +type ( + ShardingFilter bson.D + delimiter interface { + isDelimiter() + } +) func (*regularDelimiter) isDelimiter() {} func (*infDelimiter) isDelimiter() {} @@ -30,9 +32,9 @@ type regularDelimiter struct { ID interface{} } -func newDelimiter(ID interface{}) *regularDelimiter { +func newDelimiter(id interface{}) *regularDelimiter { return ®ularDelimiter{ - ID: ID, + ID: id, } } @@ -86,7 +88,7 @@ func filterFromTable(table abstract.TableDescription) (ShardingFilter, error) { return filter, nil } -// getRepresentativeFromEveryTypeBracket acquires representative from every type bracket +// getRepresentativeFromEveryTypeBracket acquires representative from every type bracket. func getRepresentativeFromEveryTypeBracket(ctx context.Context, collection *mongo.Collection, isDocDB bool) ([]delimiter, error) { identifiers := []delimiter{} // user primitive.JavaScript instances and etc. instead of $type query like this: @@ -110,9 +112,9 @@ func getRepresentativeFromEveryTypeBracket(ctx context.Context, collection *mong I: 0, }, // primitive.Regex{Pattern: "asdf", Options: "i"}, // TODO(@kry127) how to turn on? - //int32(3), // the same type bracket as 1.27 - //int64(293120039813945800), // the same type bracket as 1.27 - //primitive.NewDecimal128(1, 1), // the same type bracket as 1.27 + // int32(3), // the same type bracket as 1.27 + // int64(293120039813945800), // the same type bracket as 1.27 + // primitive.NewDecimal128(1, 1), // the same type bracket as 1.27 } if !isDocDB { objects = append(objects, @@ -142,7 +144,7 @@ func getRepresentativeFromEveryTypeBracket(ctx context.Context, collection *mong return identifiers, nil } -// getRandomIdentifiers returns desired amount of random document identifiers +// getRandomIdentifiers returns desired amount of random document identifiers. func getRandomIdentifiers(ctx context.Context, collection *mongo.Collection, amount uint64) ([]delimiter, error) { // db.coll.aggregate([{ $sample: { size: 3 } }, { $sort : { _id : 1 }}]) // if delimiter count is larger than collection, all documents will be returned in order @@ -172,7 +174,7 @@ func getRandomIdentifiers(ctx context.Context, collection *mongo.Collection, amo } // getDelimiters acquires delimiters that has representative in every type bracket -// and has desired amount of parts if it is possible +// and has desired amount of parts if it is possible. func getDelimiters(ctx context.Context, collection *mongo.Collection, amountOfDelimiters uint64, isDocDB bool) ([]delimiter, error) { typeBracketDelimiters, err := getRepresentativeFromEveryTypeBracket(ctx, collection, isDocDB) if err != nil { diff --git a/pkg/providers/mongo/source.go b/pkg/providers/mongo/source.go index 28e0de92..5d3b5279 100644 --- a/pkg/providers/mongo/source.go +++ b/pkg/providers/mongo/source.go @@ -195,7 +195,7 @@ func (s *mongoSource) pingParallelizationUnits(changeStreams map[Parallelization } // used for extra processing error -// it is possible to implement here fallback mechanics on different behaviour (ChangeStreamWatcher) when certain errors occur +// it is possible to implement here fallback mechanics on different behaviour (ChangeStreamWatcher) when certain errors occur. func (s *mongoSource) watcherFallback(watcher ChangeStreamWatcher, pu ParallelizationUnit, reason error) (ChangeStreamWatcher, error) { // always close watcher, it will not be reused on fallback defer func() { @@ -600,7 +600,7 @@ func (s *mongoSource) getCollectionFilter(client *MongoClientWrapper) (*MongoCol return &result, nil } -// openChangeStreams is mongo version dependent function +// openChangeStreams is mongo version dependent function. func (s *mongoSource) openChangeStreams() (map[ParallelizationUnit]ChangeStreamWatcher, error) { mongoVersion, err := GetVersion(s.ctx, s.client, s.config.AuthSource) if err != nil { diff --git a/pkg/providers/mongo/storage.go b/pkg/providers/mongo/storage.go index 57245c9e..e2403f49 100644 --- a/pkg/providers/mongo/storage.go +++ b/pkg/providers/mongo/storage.go @@ -89,7 +89,7 @@ func (s *Storage) LoadTable(ctx context.Context, table abstract.TableDescription } coll := s.Client.Database(table.Schema).Collection(table.Name) - tableRowsCount, err := s.Client.Database(table.Schema).Collection(table.Name).EstimatedDocumentCount(ctx) + _, err := s.Client.Database(table.Schema).Collection(table.Name).EstimatedDocumentCount(ctx) if err != nil { logger.Log.Warn("Cannot estimate total count", log.Error(err)) } @@ -100,7 +100,6 @@ func (s *Storage) LoadTable(ctx context.Context, table abstract.TableDescription if table.Offset > 0 { findOptions.SetSkip(int64(table.Offset)) - tableRowsCount -= int64(table.Offset) } filter, err := filterFromTable(table) @@ -111,7 +110,6 @@ func (s *Storage) LoadTable(ctx context.Context, table abstract.TableDescription var cursor *mongo.Cursor if err := backoff.Retry(func() error { cursor, err = coll.Find(ctx, filter, findOptions) - if err != nil { return xerrors.Errorf("Cannot create document cursor: %w", err) } diff --git a/pkg/providers/mongo/time.go b/pkg/providers/mongo/time.go index 08b831a1..acbfcbe1 100644 --- a/pkg/providers/mongo/time.go +++ b/pkg/providers/mongo/time.go @@ -51,7 +51,7 @@ func GetLocalOplogInterval(ctx context.Context, client *MongoClientWrapper) (fro return fromTimestampHolder.Timestamp, toTimestampHolder.Timestamp, nil } -// SyncClusterTime is mongo version dependent code +// SyncClusterTime is mongo version dependent code. func SyncClusterTime(ctx context.Context, src *MongoSource, defaultCACertPaths []string) error { client, err := Connect(ctx, src.ConnectionOptions(defaultCACertPaths), nil) if err != nil { diff --git a/pkg/providers/mongo/version.go b/pkg/providers/mongo/version.go index ef125c31..141da5ca 100644 --- a/pkg/providers/mongo/version.go +++ b/pkg/providers/mongo/version.go @@ -21,7 +21,7 @@ var ( MongoVersion4_0 = must(semver.ParseTolerant("4.0")) ) -// GetVersion tries to get version from database authSource. If authSource is empty string, default will be used +// GetVersion tries to get version from database authSource. If authSource is empty string, default will be used. func GetVersion(ctx context.Context, client *MongoClientWrapper, authSource string) (*semver.Version, error) { if authSource == "" { authSource = DefaultAuthSource diff --git a/pkg/providers/mysql/canal.go b/pkg/providers/mysql/canal.go index 6327ed90..854f35d8 100644 --- a/pkg/providers/mysql/canal.go +++ b/pkg/providers/mysql/canal.go @@ -32,7 +32,7 @@ var globalExclude = []*regexp.Regexp{ } // Canal can sync your MySQL data into everywhere, like Elasticsearch, Redis, etc... -// MySQL must open row format for binlog +// MySQL must open row format for binlog. type Canal struct { cfg *Config @@ -58,7 +58,7 @@ type Canal struct { cancel context.CancelFunc } -// canal will retry fetching unknown table's meta after UnknownTableRetryPeriod +// canal will retry fetching unknown table's meta after UnknownTableRetryPeriod. var ( UnknownTableRetryPeriod = time.Second * time.Duration(10) ErrExcludedTable = xerrors.New("table is excluded") @@ -364,7 +364,7 @@ func (c *Canal) loadTableConstraints(db, table string) (map[string][]string, err return LoadTableConstraints(dbConn, abstract.TableID{Namespace: db, Name: table}) } -// ClearTableCache clear table cache +// ClearTableCache clear table cache. func (c *Canal) ClearTableCache(db []byte, table []byte) { key := fmt.Sprintf("%s.%s", db, table) c.tableLock.Lock() @@ -375,7 +375,7 @@ func (c *Canal) ClearTableCache(db []byte, table []byte) { c.tableLock.Unlock() } -// CheckBinlogRowImage checks MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB +// CheckBinlogRowImage checks MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB. func (c *Canal) CheckBinlogRowImage(image string) error { // need to check MySQL binlog row image? full, minimal or noblob? // now only log @@ -452,7 +452,7 @@ func (c *Canal) prepareSyncer() error { return nil } -// Execute a SQL +// Execute a SQL. func (c *Canal) Execute(cmd string, args ...interface{}) (*mysql.Result, error) { c.connLock.Lock() defer c.connLock.Unlock() diff --git a/pkg/providers/mysql/config.go b/pkg/providers/mysql/config.go index fc0cc64a..fc0cae2c 100644 --- a/pkg/providers/mysql/config.go +++ b/pkg/providers/mysql/config.go @@ -5,7 +5,7 @@ import ( "time" ) -// TODO: remove +// TODO: remove. type Config struct { Addr string User string diff --git a/pkg/providers/mysql/mysql_connection_params.go b/pkg/providers/mysql/mysql_connection_params.go index e69c9073..185c288d 100644 --- a/pkg/providers/mysql/mysql_connection_params.go +++ b/pkg/providers/mysql/mysql_connection_params.go @@ -140,7 +140,7 @@ func (params *ConnectionParams) ResolveLocation(locationStr string) error { func resolveConnection(connectionID string, database string) (*connection.ConnectionMySQL, error) { connCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - //DP agent token here + // DP agent token here conn, err := connection.Resolver().ResolveConnection(connCtx, connectionID, ProviderType) if err != nil { return nil, err diff --git a/pkg/providers/mysql/mysqlrecipe/container.go b/pkg/providers/mysql/mysqlrecipe/container.go index 37755efc..dc053973 100644 --- a/pkg/providers/mysql/mysqlrecipe/container.go +++ b/pkg/providers/mysql/mysqlrecipe/container.go @@ -23,7 +23,7 @@ const ( TargetDB = "target" ) -// MySQLContainer represents the MySQL container type used in the module +// MySQLContainer represents the MySQL container type used in the module. type MySQLContainer struct { testcontainers.Container username string @@ -61,7 +61,7 @@ func PrepareContainer(ctx context.Context) { } } -// Run creates an instance of the MySQL container type +// Run creates an instance of the MySQL container type. func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustomizer) (*MySQLContainer, error) { tz, _ := time.Now().Zone() @@ -95,7 +95,7 @@ CREATE USER '%[3]s'@'%%' IDENTIFIED BY '%[4]s'; GRANT ALL PRIVILEGES ON *.* TO '%[3]s'@'%%'; SET GLOBAL time_zone = "%[5]s"; `, SourceDB, TargetDB, defaultUser, defaultPassword, tz) - if _, err := f.Write([]byte(initSQL)); err != nil { + if _, err := f.WriteString(initSQL); err != nil { return nil, xerrors.Errorf("unable to write init script: %w", err) } diff --git a/pkg/providers/mysql/provider.go b/pkg/providers/mysql/provider.go index efaea4b3..321ad9b4 100644 --- a/pkg/providers/mysql/provider.go +++ b/pkg/providers/mysql/provider.go @@ -62,7 +62,7 @@ func isSystemTable(tableName string) bool { const ProviderType = abstract.ProviderType("mysql") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Snapshot = (*Provider)(nil) _ providers.Replication = (*Provider)(nil) diff --git a/pkg/providers/mysql/queries_builder.go b/pkg/providers/mysql/queries_builder.go index 72d6fd30..e50d4947 100644 --- a/pkg/providers/mysql/queries_builder.go +++ b/pkg/providers/mysql/queries_builder.go @@ -419,7 +419,7 @@ func columnNameToIndex(tableSchema *[]abstract.ColSchema) map[string]int { // changeItem.Kind passed to function input - should be "delete" // // If there are present primary key - condition includes only primaryKey: "(`pKey0`=pVal0 AND `pKey1`=pVal1, ...)" -// If there are no primary key - condition includes all keys&values in changeItem: "(`key0`=val0 AND `key1`=val1, ...)" +// If there are no primary key - condition includes all keys&values in changeItem: "(`key0`=val0 AND `key1`=val1, ...)". func buildPartOfQueryDeleteCondition(tableSchema *[]abstract.ColSchema, changeItem *abstract.ChangeItem, colNameToIndexInTableSchema *map[string]int) string { currentConditions := make([]string, len(changeItem.OldKeys.KeyNames)) for i, column := range changeItem.OldKeys.KeyNames { @@ -479,7 +479,7 @@ func buildQueryUpdate(table abstract.TableID, tableSchema *[]abstract.ColSchema, return "", false } -// When we are here, guaranteed len(tableSchema)==len(ColumnNames) +// When we are here, guaranteed len(tableSchema)==len(ColumnNames). func buildPartOfQueryInsert(tableSchema *[]abstract.ColSchema, changeItem *abstract.ChangeItem, colNameToIndexInTableSchema *map[string]int) string { vals := make([]string, len(changeItem.ColumnNames)) for i, n := range changeItem.ColumnNames { diff --git a/pkg/providers/mysql/rows.go b/pkg/providers/mysql/rows.go index 8cee9837..8237901c 100644 --- a/pkg/providers/mysql/rows.go +++ b/pkg/providers/mysql/rows.go @@ -175,7 +175,7 @@ func tryRestoreStringAndBytesTypes(event *RowsEvent, columnIndex int, columnYtTy func alterUTF16Charset(charsetName string) string { result := strings.ReplaceAll(charsetName, "utf16", "utf-16") - //by default mysql utf16 charset is big endian but in golang.org/x/net/html that we use here it is processed as little endian + // by default mysql utf16 charset is big endian but in golang.org/x/net/html that we use here it is processed as little endian if !strings.HasSuffix(strings.ToLower(charsetName), "le") { result += "be" } diff --git a/pkg/providers/mysql/sink.go b/pkg/providers/mysql/sink.go index 1ab20fcb..ee9c9d08 100644 --- a/pkg/providers/mysql/sink.go +++ b/pkg/providers/mysql/sink.go @@ -76,7 +76,7 @@ func generatedBounds(queries []string, maxSize int) []abstract.TxBound { left = i + 1 size = 0 } else { - size = size + len(q) + size += len(q) } } if len(queries) != left { diff --git a/pkg/providers/mysql/sink_test.go b/pkg/providers/mysql/sink_test.go index 9fc38f2a..c31e2c93 100644 --- a/pkg/providers/mysql/sink_test.go +++ b/pkg/providers/mysql/sink_test.go @@ -438,7 +438,7 @@ func Test_generatedBounds(t *testing.T) { }, bounds) } -// checking that if the table cannot be created, a fatal error is returned +// checking that if the table cannot be created, a fatal error is returned. func Test_create_table(t *testing.T) { db, mock, err := sqlmock.New() require.NoError(t, err) diff --git a/pkg/providers/mysql/storage.go b/pkg/providers/mysql/storage.go index 48511eda..8bcf2aab 100644 --- a/pkg/providers/mysql/storage.go +++ b/pkg/providers/mysql/storage.go @@ -216,7 +216,6 @@ func (s *Storage) Position(ctx context.Context) (*abstract.LogPosition, error) { if xerrors.As(err, ¬Master) || xerrors.As(err, notMaster) { isSlave = true logger.Log.Info(notMaster.Error()) - err = nil } else { return nil, xerrors.Errorf("unable to get binlog position: %w", err) } diff --git a/pkg/providers/mysql/sync.go b/pkg/providers/mysql/sync.go index d23c3921..4eb2a2ed 100644 --- a/pkg/providers/mysql/sync.go +++ b/pkg/providers/mysql/sync.go @@ -43,8 +43,7 @@ func (c *Canal) runSyncBinlog() error { return xerrors.Errorf("failed to start syncer: %w", err) } - savePos := false - force := false + var savePos, force bool // The name of the binlog file received in the fake rotate event. // It must be preserved until the new position is saved. @@ -349,7 +348,7 @@ func (c *Canal) GetMasterPos() (mysql.Position, error) { } func (c *Canal) GetMasterGTIDSet() (mysql.GTIDSet, error) { - query := "" + var query string switch c.cfg.Flavor { case mysql.MariaDBFlavor: query = "SELECT @@GLOBAL.gtid_current_pos" diff --git a/pkg/providers/mysql/tasks.go b/pkg/providers/mysql/tasks.go index 899582ca..21585e8e 100644 --- a/pkg/providers/mysql/tasks.go +++ b/pkg/providers/mysql/tasks.go @@ -121,7 +121,7 @@ func isFailOnDecimal(transfer *model.Transfer) bool { return true } -// See TM-4581 +// See TM-4581. func checkRestrictedColumnTypes(transfer *model.Transfer, tables abstract.TableMap) error { if !isFailOnDecimal(transfer) { return nil diff --git a/pkg/providers/mysql/unmarshaller/replication/hetero.go b/pkg/providers/mysql/unmarshaller/replication/hetero.go index 21bc8698..6e33888f 100644 --- a/pkg/providers/mysql/unmarshaller/replication/hetero.go +++ b/pkg/providers/mysql/unmarshaller/replication/hetero.go @@ -123,7 +123,7 @@ func makeCastUint64ToBytes(originalType string) func(value any) (any, error) { } } -// typeLengthModifier returns -1 if there is no length modifier for the type +// typeLengthModifier returns -1 if there is no length modifier for the type. func typeLengthModifier(originalType string) (int, error) { bracesOpenIdx := strings.Index(originalType, "(") if bracesOpenIdx < 0 { diff --git a/pkg/providers/mysql/unmarshaller/types/temporal.go b/pkg/providers/mysql/unmarshaller/types/temporal.go index ccdcfbe7..f5ae3481 100644 --- a/pkg/providers/mysql/unmarshaller/types/temporal.go +++ b/pkg/providers/mysql/unmarshaller/types/temporal.go @@ -11,7 +11,7 @@ import ( "github.com/doublecloud/transfer/pkg/providers/postgres/sqltimestamp" ) -// Temporal enables to scan any temporal value with a date from []byte +// Temporal enables to scan any temporal value with a date from []byte. type Temporal struct { location *time.Location isNull bool diff --git a/pkg/providers/mysql/utils.go b/pkg/providers/mysql/utils.go index b4a04f05..f9b1792c 100644 --- a/pkg/providers/mysql/utils.go +++ b/pkg/providers/mysql/utils.go @@ -187,7 +187,7 @@ func pushCreateTable(ctx context.Context, tx Queryable, table abstract.TableID, return applyDDLs([]ddlValue{ddlItem}, pusher) } -// fileOffset for next file we add this number to LSN +// fileOffset for next file we add this number to LSN. const fileOffset = 1_000_000_000_000 func CalculateLSN(file string, pos uint32) uint64 { diff --git a/pkg/providers/opensearch/provider.go b/pkg/providers/opensearch/provider.go index 23237f40..864af6d8 100644 --- a/pkg/providers/opensearch/provider.go +++ b/pkg/providers/opensearch/provider.go @@ -35,7 +35,7 @@ func destinationModelFactory() model.Destination { const ProviderType = abstract.ProviderType("opensearch") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) _ providers.Snapshot = (*Provider)(nil) diff --git a/pkg/providers/postgres/change_processor.go b/pkg/providers/postgres/change_processor.go index 0f940e1e..416980d5 100644 --- a/pkg/providers/postgres/change_processor.go +++ b/pkg/providers/postgres/change_processor.go @@ -371,11 +371,9 @@ func expectedAnyCastReplication(value any, oid pgtype.OID, colSchema *abstract.C unmarshalled, successful, unmarshalErr := tryUnmarshalComplexType(v, oid, connInfo) if unmarshalErr != nil { logger.Log.Warn("failed to unpack complex types", log.Error(unmarshalErr)) - } else { - if successful { - result, err = unmarshalled, nil - break - } + } else if successful { + result, err = unmarshalled, nil + break } } switch ClearOriginalType(colSchema.OriginalType) { diff --git a/pkg/providers/postgres/changeitems_fetcher.go b/pkg/providers/postgres/changeitems_fetcher.go index b3f28691..f4c18872 100644 --- a/pkg/providers/postgres/changeitems_fetcher.go +++ b/pkg/providers/postgres/changeitems_fetcher.go @@ -15,7 +15,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// ChangeItemsFetcher consolidates multiple objects (of which most important is pgx.Rows) into a single one, providing bufferized fetching and parsing of rows from a postgres source +// ChangeItemsFetcher consolidates multiple objects (of which most important is pgx.Rows) into a single one, providing bufferized fetching and parsing of rows from a postgres source. type ChangeItemsFetcher struct { rows pgx.Rows connInfo *pgtype.ConnInfo @@ -57,7 +57,7 @@ func (f *ChangeItemsFetcher) WithLimitCount(limitCount int) *ChangeItemsFetcher return f } -// WithLimitBytes accepts 0 to disable limit +// WithLimitBytes accepts 0 to disable limit. func (f *ChangeItemsFetcher) WithLimitBytes(limitBytes model.BytesSize) *ChangeItemsFetcher { f.limitBytes = limitBytes return f diff --git a/pkg/providers/postgres/client.go b/pkg/providers/postgres/client.go index 21a2e16e..f8ad177c 100644 --- a/pkg/providers/postgres/client.go +++ b/pkg/providers/postgres/client.go @@ -23,7 +23,7 @@ import ( const SelectCurrentLsnDelay = `select pg_wal_lsn_diff(pg_last_wal_replay_lsn(), $1);` -// replica specific stuff +// replica specific stuff. func getHostPreferablyReplica(lgr log.Logger, conn *connection.ConnectionPG, slotID string) (*connection.Host, error) { if conn.ClusterID == "" { replHost, err := getReplicaOnPrem(conn) @@ -122,8 +122,8 @@ func detectHostsByRoles(lgr log.Logger, conn *connection.ConnectionPG) (master * aliveSyncReplicas = make([]*connection.Host, 0) if master = conn.MasterHost(); master != nil { - //roles are already detected in connection - this is managed connection for managed pg cluster - //however replica type is not always available - seems it is filled only when async is present + // roles are already detected in connection - this is managed connection for managed pg cluster + // however replica type is not always available - seems it is filled only when async is present for _, host := range conn.Hosts { if host.Role != connection.Replica { continue @@ -136,7 +136,7 @@ func detectHostsByRoles(lgr log.Logger, conn *connection.ConnectionPG) (master * } } } else { - //roles are unknown - this is manual connection or on-premise pg installation + // roles are unknown - this is manual connection or on-premise pg installation var clusterHosts []dbaas.ClusterHost clusterHosts, err = dbaas.ResolveClusterHosts(dbaas.ProviderTypePostgresql, conn.ClusterID) lgr.Info("Resolved cluster to hosts via dbaas", log.String("cluster", conn.ClusterID), log.Any("hosts", clusterHosts)) @@ -164,7 +164,7 @@ func detectHostsByRoles(lgr log.Logger, conn *connection.ConnectionPG) (master * } func getHostPortFromMDBHostname(hostname string, connPG *connection.ConnectionPG) (string, uint16, error) { - //get from name if available, otherwise getting from connection param (user input) + // get from name if available, otherwise getting from connection param (user input) portFromConfig := connPG.GetPort(hostname) if portFromConfig == 0 { logger.Log.Info("No port present in config, using default mdb port") @@ -246,11 +246,11 @@ func isReplicaUpToDate(connParams *ConnectionParams, lsn string) bool { } func getMasterConnectionParams(lgr log.Logger, connParams *connection.ConnectionPG) (*ConnectionParams, error) { - //master was resolved in connection + // master was resolved in connection if connParams.MasterHost() != nil { return toConnParams(connParams.MasterHost(), connParams), nil } - //find suitable host manually + // find suitable host manually masterHost, masterPort, err := resolveMasterHostImpl(connParams) if err != nil { return nil, xerrors.Errorf("unable to resolve master host, err: %w", err) @@ -270,7 +270,7 @@ func getMasterConnectionParams(lgr log.Logger, connParams *connection.Connection func resolveMasterHostImpl(conn *connection.ConnectionPG) (string, uint16, error) { if master := conn.MasterHost(); master != nil { - //it is resolved connection for cluster + // it is resolved connection for cluster return master.Name, uint16(master.Port), nil } @@ -333,7 +333,7 @@ func makeConnConfig(connParams *ConnectionParams, connString string, tryHostCAce } func getTLSConfig(host string, hasTLS bool, tlsFile string, cluster string, tryHostCAcertificates bool) (*tls.Config, error) { - //what changes for pg source/dst (but not storage!!) : use custom cert for mdb if present + // what changes for pg source/dst (but not storage!!) : use custom cert for mdb if present if hasTLS { rootCertPool := x509.NewCertPool() if len(tlsFile) > 0 { @@ -498,7 +498,7 @@ func NewPgConnPool(connConfig *pgx.ConnConfig, lgr log.Logger, dataTypesOptions return pool, nil } -// pgStatementTimeout returns the GUC `statement_timeout` or a default timeout if the former is zero +// pgStatementTimeout returns the GUC `statement_timeout` or a default timeout if the former is zero. func pgStatementTimeout(ctx context.Context, conn *pgx.Conn) (time.Duration, error) { sql := "select setting from pg_settings where name = 'statement_timeout'" var rawResult string @@ -518,7 +518,7 @@ func pgStatementTimeout(ctx context.Context, conn *pgx.Conn) (time.Duration, err } } -// NewPgConnPoolConfig creates a connection pool. It provides built-in timeouts to limit the duration of connection attempts +// NewPgConnPoolConfig creates a connection pool. It provides built-in timeouts to limit the duration of connection attempts. func NewPgConnPoolConfig(ctx context.Context, poolConfig *pgxpool.Config) (*pgxpool.Pool, error) { basicCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() @@ -647,7 +647,7 @@ func makeConnectionFromSink(dst PgSinkParams) *connection.ConnectionPG { func resolveConnection(connectionID string, database string) (*connection.ConnectionPG, error) { connCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - //DP agent token here + // DP agent token here conn, err := connection.Resolver().ResolveConnection(connCtx, connectionID, ProviderType) if err != nil { return nil, err diff --git a/pkg/providers/postgres/client_test.go b/pkg/providers/postgres/client_test.go index f815ac33..7c5d953b 100644 --- a/pkg/providers/postgres/client_test.go +++ b/pkg/providers/postgres/client_test.go @@ -98,7 +98,7 @@ func waitMaster(cfg *PgStorageParams, hosts []string, duration time.Duration) (s } // checkTwoPostgresIsRunning -// returns 'true' if needed to check one more time +// returns 'true' if needed to check one more time. func checkTwoPostgresIsRunning(t *testing.T) bool { cmd := `sudo docker ps | grep '\bUp\b.*\bpostgres_\d\b' | wc -l | awk '{print $1}'` out, err := exec.Command("bash", "-c", cmd).Output() @@ -121,7 +121,7 @@ func checkTwoPostgresIsRunning(t *testing.T) bool { } // checkExactlyOnePostgresIsPrimary -// returns 'true' if there are two primary +// returns 'true' if there are two primary. func checkExactlyOnePostgresIsPrimary(t *testing.T) bool { cmd1 := `sudo docker exec postgres_1 psql -t -c "select case when pg_is_in_recovery() then 'secondary' else 'primary' end as host_status;" "dbname=habrdb user=habrpguser password=pgpwd4habr" | awk '{print $1}' | grep -v "^$"` out1Arr, err := exec.Command("bash", "-c", cmd1).Output() diff --git a/pkg/providers/postgres/composite.go b/pkg/providers/postgres/composite.go index 46eebb0a..f15b65af 100644 --- a/pkg/providers/postgres/composite.go +++ b/pkg/providers/postgres/composite.go @@ -61,7 +61,7 @@ func (c *composite) FullName() TypeFullName { func (c *composite) DependsOn() []pgtype.OID { var fieldTypes []pgtype.OID for _, f := range c.fields { - fieldTypes = append(fieldTypes, pgtype.OID(f.oid)) + fieldTypes = append(fieldTypes, f.oid) } return fieldTypes } @@ -116,9 +116,9 @@ func selectCompositeTypes(ctx context.Context, conn *pgx.Conn, complexTypes map[ return xerrors.Errorf("scan: %w", err) } - typ, ok := complexTypes[pgtype.OID(oid)].(*composite) + typ, ok := complexTypes[oid].(*composite) if !ok { - complexTypes[pgtype.OID(oid)] = &composite{ + complexTypes[oid] = &composite{ oid: oid, namespace: namespace.String, name: name.String, diff --git a/pkg/providers/postgres/conn.go b/pkg/providers/postgres/conn.go index 91a29e50..02e951f9 100644 --- a/pkg/providers/postgres/conn.go +++ b/pkg/providers/postgres/conn.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgx/v4" ) -// go-sumtype:decl DataTypesOption +// go-sumtype:decl DataTypesOption. type DataTypesOption interface { isDataTypeOption() } diff --git a/pkg/providers/postgres/date.go b/pkg/providers/postgres/date.go index 428babda..dee4e7cb 100644 --- a/pkg/providers/postgres/date.go +++ b/pkg/providers/postgres/date.go @@ -16,10 +16,10 @@ var _ TextDecoderAndValuerWithHomo = (*Date)(nil) // NewDate constructs a DATE representation which supports BC years // -// TODO: remove this when https://st.yandex-team.ru/TM-5127 is done +// TODO: remove this when https://st.yandex-team.ru/TM-5127 is done. func NewDate() *Date { return &Date{ - Date: *new(pgtype.Date), + Date: pgtype.Date{}, } } diff --git a/pkg/providers/postgres/incremental_storage.go b/pkg/providers/postgres/incremental_storage.go index 80f137b4..66c75530 100644 --- a/pkg/providers/postgres/incremental_storage.go +++ b/pkg/providers/postgres/incremental_storage.go @@ -12,7 +12,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// ensure that Storage is indeed incremental +// ensure that Storage is indeed incremental. var _ abstract.IncrementalStorage = new(Storage) var repeatableReadReadOnlyTxOptions pgx.TxOptions = pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly, DeferrableMode: pgx.NotDeferrable} diff --git a/pkg/providers/postgres/model_pg_destination.go b/pkg/providers/postgres/model_pg_destination.go index 98b1d117..cae10729 100644 --- a/pkg/providers/postgres/model_pg_destination.go +++ b/pkg/providers/postgres/model_pg_destination.go @@ -61,7 +61,7 @@ func (d *PgDestination) FillDependentFields(transfer *dp_model.Transfer) { } } -// AllHosts - function to move from legacy 'Host' into modern 'Hosts' +// AllHosts - function to move from legacy 'Host' into modern 'Hosts'. func (d *PgDestination) AllHosts() []string { return utils.HandleHostAndHosts(d.Host, d.Hosts) } diff --git a/pkg/providers/postgres/model_pg_source.go b/pkg/providers/postgres/model_pg_source.go index cb4b2162..aecea3ec 100644 --- a/pkg/providers/postgres/model_pg_source.go +++ b/pkg/providers/postgres/model_pg_source.go @@ -173,7 +173,7 @@ func (s *PgSource) FillDependentFields(transfer *model.Transfer) { } } -// AllHosts - function to move from legacy 'Host' into modern 'Hosts' +// AllHosts - function to move from legacy 'Host' into modern 'Hosts'. func (s *PgSource) AllHosts() []string { return utils.HandleHostAndHosts(s.Host, s.Hosts) } diff --git a/pkg/providers/postgres/model_pg_storage_params.go b/pkg/providers/postgres/model_pg_storage_params.go index a74b2590..33a7c8a0 100644 --- a/pkg/providers/postgres/model_pg_storage_params.go +++ b/pkg/providers/postgres/model_pg_storage_params.go @@ -58,7 +58,7 @@ func (p *PgStorageParams) TLSConfigTemplate() (*tls.Config, error) { return nil, nil } -// tlsStatusString returns a string describing the TLS connection status of the params +// tlsStatusString returns a string describing the TLS connection status of the params. func (p *PgStorageParams) secureConnectionStatusString() string { if len(p.TLSFile) > 0 { return "secure, user-provided CA certificate" diff --git a/pkg/providers/postgres/pg_dump.go b/pkg/providers/postgres/pg_dump.go index bd8275d9..b21bae27 100644 --- a/pkg/providers/postgres/pg_dump.go +++ b/pkg/providers/postgres/pg_dump.go @@ -134,7 +134,6 @@ func formatFqtn(in string) (string, error) { } func PostgresDumpConnString(src *PgSource) (string, model.SecretString, error) { - config, err := GetConnParamsFromSrc(logger.Log, src) if err != nil { return "", "", err @@ -205,7 +204,7 @@ func pgDumpSchemaArgs(src *PgSource, seqsIncluded []abstract.TableID, seqsExclud return args, nil } -// dumpSequenceValues produces SEQUENCE SET pg_dump events which transmit the current state of all sequences in the given list from the source database +// dumpSequenceValues produces SEQUENCE SET pg_dump events which transmit the current state of all sequences in the given list from the source database. func dumpSequenceValues(ctx context.Context, conn *pgx.Conn, sequences []abstract.TableID) ([]*pgDumpItem, error) { result := make([]*pgDumpItem, 0) for _, seq := range sequences { @@ -225,7 +224,7 @@ func dumpSequenceValues(ctx context.Context, conn *pgx.Conn, sequences []abstrac return result, nil } -// sourceInPgPg returns a non-nil object only for homogenous PG-PG transfers +// sourceInPgPg returns a non-nil object only for homogenous PG-PG transfers. func sourceInPgPg(transfer *model.Transfer) *PgSource { var src *PgSource var srcIsPG bool @@ -238,7 +237,7 @@ func sourceInPgPg(transfer *model.Transfer) *PgSource { return src } -// ExtractPgDumpSchema returns the dump ONLY for homogenous PG-PG transfers. It also logs its actions +// ExtractPgDumpSchema returns the dump ONLY for homogenous PG-PG transfers. It also logs its actions. func ExtractPgDumpSchema(transfer *model.Transfer) ([]*pgDumpItem, error) { src := sourceInPgPg(transfer) if src == nil { @@ -254,7 +253,7 @@ func ExtractPgDumpSchema(transfer *model.Transfer) ([]*pgDumpItem, error) { return pgdump, nil } -// ApplyPgDumpPreSteps takes the given dump and applies pre-steps defined in transfer source ONLY for homogenous PG-PG transfers. It also logs its actions +// ApplyPgDumpPreSteps takes the given dump and applies pre-steps defined in transfer source ONLY for homogenous PG-PG transfers. It also logs its actions. func ApplyPgDumpPreSteps(pgdump []*pgDumpItem, transfer *model.Transfer, registry metrics.Registry) error { if len(pgdump) == 0 { return nil @@ -271,7 +270,7 @@ func ApplyPgDumpPreSteps(pgdump []*pgDumpItem, transfer *model.Transfer, registr return nil } -// ApplyPgDumpPostSteps takes the given dump and applies post-steps defined in transfer source ONLY for homogenous PG-PG transfers. It also logs its actions +// ApplyPgDumpPostSteps takes the given dump and applies post-steps defined in transfer source ONLY for homogenous PG-PG transfers. It also logs its actions. func ApplyPgDumpPostSteps(pgdump []*pgDumpItem, transfer *model.Transfer, registry metrics.Registry) error { if len(pgdump) == 0 { return nil @@ -318,7 +317,7 @@ func determineExcludedTypes(allTypes []*pgDumpItem, allowedTypes []*pgDumpItem) return excludedTypes } -// loadPgDumpSchema actually loads the schema from PostgreSQL source using a storage constructed in-place +// loadPgDumpSchema actually loads the schema from PostgreSQL source using a storage constructed in-place. func loadPgDumpSchema(ctx context.Context, src *PgSource, transfer *model.Transfer) ([]*pgDumpItem, error) { storage, err := NewStorage(src.ToStorageParams(transfer)) if err != nil { @@ -400,7 +399,7 @@ func loadPgDumpSchema(ctx context.Context, src *PgSource, transfer *model.Transf return result, nil } -// listAllSequences constructs a pg Storage in-place and obtains all (accessible) SEQUENCEs +// listAllSequences constructs a pg Storage in-place and obtains all (accessible) SEQUENCEs. func listAllSequences(ctx context.Context, src *PgSource, conn *pgx.Conn) (SequenceMap, error) { if !src.PreSteps.Sequence && !src.PostSteps.Sequence { return make(SequenceMap), nil @@ -409,7 +408,7 @@ func listAllSequences(ctx context.Context, src *PgSource, conn *pgx.Conn) (Seque return ListSequencesWithDependants(ctx, conn, src.KeeperSchema) } -// filterSequences separates the given sequences into included and excluded ones by applying the given filter +// filterSequences separates the given sequences into included and excluded ones by applying the given filter. func filterSequences(sequences SequenceMap, filter abstract.Includeable) (included []abstract.TableID, excluded []abstract.TableID) { for _, sequenceInfo := range sequences { sequenceIncluded := false @@ -467,7 +466,7 @@ func dumpCollations(collations []*pgDumpItem, tablesSchemas *set.Set[string]) [] return result } -// parse and validate types in cast +// parse and validate types in cast. func isAllowedCast(createCastSQL string, excludedTypes *set.Set[string], tablesSchemas *set.Set[string]) bool { cleanedStatement := strings.TrimPrefix(createCastSQL, "\n--\nCREATE CAST (") parts := splitSQLBySeparator(cleanedStatement, " AS ") @@ -527,11 +526,11 @@ func dumpDefinedItems(connString string, connPass model.SecretString, src *PgSou return result, nil } -// strings.Split without considering the separator inside the quotes -func splitSQLBySeparator(SQL string, sep string) []string { +// strings.Split without considering the separator inside the quotes. +func splitSQLBySeparator(sql string, sep string) []string { result := make([]string, 0) - parts := strings.Split(SQL, sep) + parts := strings.Split(sql, sep) cur := "" for _, i := range parts { @@ -558,7 +557,7 @@ func isArgMode(argPart string) bool { return argPart == "IN" || argPart == "OUT" || argPart == "INOUT" || argPart == "VARIADIC" } -// parse args in ... FUNCTION ([argMode1] [argName1] arg1, [argMode2] [argName2] arg2, ..., [argMode] [argName] arg) ... argMode and argName is optional +// parse args in ... FUNCTION ([argMode1] [argName1] arg1, [argMode2] [argName2] arg2, ..., [argMode] [argName] arg) ... argMode and argName is optional. func extractFunctionArgsTypes(functionBody string) ([]string, bool) { nameWithoutCloseBracket := splitSQLBySeparator(functionBody, ")") argsParts := splitSQLBySeparator(nameWithoutCloseBracket[0], "(") @@ -592,7 +591,7 @@ func extractFunctionArgsTypes(functionBody string) ([]string, bool) { return result, true } -// function is allowed if all types of args and the returned type are allowed +// function is allowed if all types of args and the returned type are allowed. func isAllowedFunction(function *pgDumpItem, excludedTypes *set.Set[string]) bool { argsTypes, isOk := extractFunctionArgsTypes(function.Body) if !isOk { @@ -650,8 +649,8 @@ func dumpCasts(definedCasts []*pgDumpItem, src *PgSource, excludedTypes *set.Set return result } -func filterDump(dump []*pgDumpItem, DBTables []string) []*pgDumpItem { - if len(DBTables) == 0 { +func filterDump(dump []*pgDumpItem, dbTables []string) []*pgDumpItem { + if len(dbTables) == 0 { return dump } result := make([]*pgDumpItem, 0, len(dump)) @@ -664,7 +663,7 @@ func filterDump(dump []*pgDumpItem, DBTables []string) []*pgDumpItem { splitSQL := splitSQLBySeparator(catSQL, " ATTACH") parentTable := splitSQL[0] - if !slices.Contains(DBTables, parentTable) { + if !slices.Contains(dbTables, parentTable) { continue } case "INDEX": @@ -718,7 +717,7 @@ func execPgDump(pgDump []string, connString string, password model.SecretString, // TM-1650: permission error should be fatal err = abstract.NewFatalError(err) } - return nil, xerrors.Errorf("failed to execute pg_dump. STDERR:\n%s\nerror: %w", string(truncate(string(stderrBytes), 2000)), err) + return nil, xerrors.Errorf("failed to execute pg_dump. STDERR:\n%s\nerror: %w", truncate(string(stderrBytes), 2000), err) } pgDumpOut := parsePgDumpOut(&stdout) logPgDumpOut(pgDumpOut) diff --git a/pkg/providers/postgres/pgrecipe/postgres_recipe.go b/pkg/providers/postgres/pgrecipe/postgres_recipe.go index f9040b81..2419160f 100644 --- a/pkg/providers/postgres/pgrecipe/postgres_recipe.go +++ b/pkg/providers/postgres/pgrecipe/postgres_recipe.go @@ -145,7 +145,7 @@ func WithoutPgDump() RecipeOption { type RecipeOption func(pg *RecipeParams) -// no need to recreate same postgres recipe prefix over and over again +// no need to recreate same postgres recipe prefix over and over again. var postgresContainers = map[string]bool{} func ManagedConnection(opts ...RecipeOption) *connection.ConnectionPG { @@ -302,7 +302,7 @@ max_replication_slots = 64 # max number of replication slots (change requi // PgDump dumps all tables in the PostgreSQL database specified in the connString. // Use the specified pgDumpExecutable for dumping schemas, and the given // psqlExecutable for dumping table data. -// You can specify tableLikePattern to filter tables in the dump, for example, 'public.my%table' +// You can specify tableLikePattern to filter tables in the dump, for example, 'public.my%table'. func PgDump(t *testing.T, pgDumpExecutable []string, psqlExecutable []string, connString string, tableLikePattern string) string { pgDumpArgs := append(pgDumpExecutable, "--no-publications", "--no-subscriptions", "--format=plain", "--no-owner", "--schema-only", connString) pgDumpCommand := exec.Command(pgDumpArgs[0], pgDumpArgs[1:]...) diff --git a/pkg/providers/postgres/provider.go b/pkg/providers/postgres/provider.go index 4277534c..c79df82e 100644 --- a/pkg/providers/postgres/provider.go +++ b/pkg/providers/postgres/provider.go @@ -57,7 +57,7 @@ const ( const ProviderType = abstract.ProviderType("pg") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sampleable = (*Provider)(nil) _ providers.Snapshot = (*Provider)(nil) diff --git a/pkg/providers/postgres/publisher.go b/pkg/providers/postgres/publisher.go index 302371f8..0df1dfad 100644 --- a/pkg/providers/postgres/publisher.go +++ b/pkg/providers/postgres/publisher.go @@ -144,7 +144,7 @@ func filterTablesList(config *PgSource) ([]abstract.TableID, error) { } // wal2jsonTableFromTableID formats TableID so that it can be passed as a parameter to wal2json. -// The specification is at https://github.com/eulerto/wal2json/blob/821147b21cb3672d8c67f708440ff4732e320e0e/README.md#parameters, `filter-tables` +// The specification is at https://github.com/eulerto/wal2json/blob/821147b21cb3672d8c67f708440ff4732e320e0e/README.md#parameters, `filter-tables`. func wal2jsonTableFromTableID(tableID abstract.TableID) string { parts := make([]string, 0, 2) if len(tableID.Namespace) > 0 { @@ -174,7 +174,7 @@ func wal2jsonEscape(identifier string) string { return builder.String() } -// wal2jsonTableList converts the given list of FQTNs into wal2json format +// wal2jsonTableList converts the given list of FQTNs into wal2json format. func wal2jsonTableList(tableIDs []abstract.TableID) string { parts := make([]string, 0, len(tableIDs)) for _, tableID := range tableIDs { @@ -321,7 +321,7 @@ WHERE slot_name = '%v' AND active_pid IS NOT NULL;`, config.SlotID) rb.Cancel() return res, nil } - //todo this should be resolved too!! + // todo this should be resolved too!! if config.ClusterID != "" { return nil, xerrors.Errorf("unable to init replication connection: %w", err) } diff --git a/pkg/providers/postgres/publisher_polling.go b/pkg/providers/postgres/publisher_polling.go index ff6e59c2..fdb4cf2d 100644 --- a/pkg/providers/postgres/publisher_polling.go +++ b/pkg/providers/postgres/publisher_polling.go @@ -252,7 +252,7 @@ func (p *poller) Stop() { } func (p *poller) Run(sink abstract.AsyncSink) error { - slotTroubleCh := p.slotMonitor.StartSlotMonitoring(int64(p.config.SlotByteLagLimit)) + slotTroubleCh := p.slotMonitor.StartSlotMonitoring(p.config.SlotByteLagLimit) go p.monitorStuckQueries() defer p.wg.Done() ticker := time.NewTicker(500 * time.Millisecond) diff --git a/pkg/providers/postgres/publisher_replication.go b/pkg/providers/postgres/publisher_replication.go index 6df8a8d6..d0f5b10b 100644 --- a/pkg/providers/postgres/publisher_replication.go +++ b/pkg/providers/postgres/publisher_replication.go @@ -52,7 +52,7 @@ type replication struct { objects *model.DataObjects sequencer *sequencer2.Sequencer parseQ *parsequeue.ParseQueue[[]abstract.ChangeItem] - objectsMap map[abstract.TableID]bool //tables to include in transfer + objectsMap map[abstract.TableID]bool // tables to include in transfer skippedTables map[abstract.TableID]bool } @@ -67,7 +67,7 @@ const BufferLimit = 16 * humanize.MiByte func (p *replication) Run(sink abstract.AsyncSink) error { var err error - //level of parallelism combined with hardcoded buffer size in receiver(16mb) prevent OOM in parsequeue + // level of parallelism combined with hardcoded buffer size in receiver(16mb) prevent OOM in parsequeue p.parseQ = parsequeue.New(p.logger, 10, sink, p.WithIncludeFilter, p.ack) if err = p.reloadSchema(); err != nil { @@ -83,7 +83,7 @@ func (p *replication) Run(sink abstract.AsyncSink) error { return xerrors.Errorf("unable to build transfer data-objects: %w", err) } - slotTroubleCh := p.slotMonitor.StartSlotMonitoring(int64(p.config.SlotByteLagLimit)) + slotTroubleCh := p.slotMonitor.StartSlotMonitoring(p.config.SlotByteLagLimit) p.wg.Add(2) go p.receiver(slotTroubleCh) @@ -225,7 +225,7 @@ func (p *replication) reloadSchema() error { const FakeParentPKeyStatusMessageCategory string = "fake_primary_key_parent" -// tableMapToDBSchemaForTables converts one type of schema to another ONLY for tables (dropping VIEWs) +// tableMapToDBSchemaForTables converts one type of schema to another ONLY for tables (dropping VIEWs). func tableMapToDBSchemaForTables(tableMap abstract.TableMap) abstract.DBSchema { result := make(abstract.DBSchema) for id, info := range tableMap { diff --git a/pkg/providers/postgres/schema.go b/pkg/providers/postgres/schema.go index f0af026a..aaa8ab0a 100644 --- a/pkg/providers/postgres/schema.go +++ b/pkg/providers/postgres/schema.go @@ -87,7 +87,7 @@ func (e *SchemaExtractor) WithLogger(logger log.Logger) *SchemaExtractor { return e } -// LoadSchema returns a settings-customized schema(s) of table(s) in PostgreSQL +// LoadSchema returns a settings-customized schema(s) of table(s) in PostgreSQL. func (e *SchemaExtractor) LoadSchema(ctx context.Context, conn *pgx.Conn, specificTable *abstract.TableID) (abstract.DBSchema, error) { tableColumns, err := e.tableToColumnsMapping(ctx, conn, specificTable) if err != nil { @@ -126,7 +126,7 @@ func (e *SchemaExtractor) LoadSchema(ctx context.Context, conn *pgx.Conn, specif return result, nil } -// listSchemaQuery returns a SQL query without placeholders when the given table is `nil`, or with two placeholders otherwise +// listSchemaQuery returns a SQL query without placeholders when the given table is `nil`, or with two placeholders otherwise. func (e *SchemaExtractor) listSchemaQuery(specificTable *abstract.TableID) string { return e.flavour.ListSchemaQuery(e.excludeViews, specificTable != nil, e.forbiddenSchemas, e.forbiddenTables) } @@ -229,7 +229,7 @@ func (e *SchemaExtractor) tableToColumnsMapping(ctx context.Context, conn *pgx.C return result, nil } -// listPKeysQuery returns a SQL query without placeholders when the given table is `nil`, or with two placeholders otherwise +// listPKeysQuery returns a SQL query without placeholders when the given table is `nil`, or with two placeholders otherwise. func (e *SchemaExtractor) listPKeysQuery(specificTable *abstract.TableID) string { // See documentation on PostgreSQL service relations and views used in this query: // https://www.postgresql.org/docs/9.4/catalog-pg-class.html @@ -360,7 +360,7 @@ func (e *SchemaExtractor) tableToPKColumnsMapping(ctx context.Context, conn *pgx return result, nil } -// replicaIdentityFullListTablesQuery returns a SQL query without placeholders when the given table is `nil`, or with two placeholders otherwise +// replicaIdentityFullListTablesQuery returns a SQL query without placeholders when the given table is `nil`, or with two placeholders otherwise. func (e *SchemaExtractor) replicaIdentityFullListTablesQuery(specificTable *abstract.TableID) string { // See documentation on PostgreSQL service relations and views used in this query: // https://www.postgresql.org/docs/9.4/catalog-pg-class.html @@ -433,7 +433,7 @@ func (s tableIDWithInfos) String() string { return strings.Join(result, ", ") } -// TablesList returns a list of basic information pieces about all tables in the given schema +// TablesList returns a list of basic information pieces about all tables in the given schema. func (e *SchemaExtractor) TablesList(ctx context.Context, conn *pgx.Conn) ([]tableIDWithInfo, time.Time, error) { var ts time.Time query := e.listTablesQuery() diff --git a/pkg/providers/postgres/sequence.go b/pkg/providers/postgres/sequence.go index dbd25bb2..b45201a0 100644 --- a/pkg/providers/postgres/sequence.go +++ b/pkg/providers/postgres/sequence.go @@ -64,7 +64,7 @@ WHERE seq.relkind = 'S'` getSequenceDataQuery = `SELECT last_value, is_called FROM %s` ) -// SequenceInfo is a description of a PostgreSQL sequence +// SequenceInfo is a description of a PostgreSQL sequence. type SequenceInfo struct { SequenceID abstract.TableID DependentTables []abstract.TableID @@ -77,7 +77,7 @@ func newSequenceInfo(sequenceID abstract.TableID) *SequenceInfo { } } -// SequenceMap is a mapping of sequence identifiers to their descriptions +// SequenceMap is a mapping of sequence identifiers to their descriptions. type SequenceMap map[abstract.TableID]*SequenceInfo // ListSequencesWithDependants returns a mapping with sequence information. diff --git a/pkg/providers/postgres/sequencer/lsn_transaction.go b/pkg/providers/postgres/sequencer/lsn_transaction.go index 01ef7b15..7d45a1b4 100644 --- a/pkg/providers/postgres/sequencer/lsn_transaction.go +++ b/pkg/providers/postgres/sequencer/lsn_transaction.go @@ -2,7 +2,7 @@ package sequencer import "github.com/doublecloud/transfer/library/go/core/xerrors" -// lsnTransaction contains all message lsns and ensures correct order +// lsnTransaction contains all message lsns and ensures correct order. type lsnTransaction struct { lastLsn uint64 // last message lsn that was received lsns []uint64 // all the message lsns that were added, must go in increasing order @@ -10,7 +10,7 @@ type lsnTransaction struct { // appendLsn checks that we append lsns in nondecreasing order only. // same lsn may appear in a transaction when COPY TO/FROM is used -// This is required because we read messages from server synchronously +// This is required because we read messages from server synchronously. func (p *lsnTransaction) appendLsn(newLsn uint64) error { if p.lastLsn > newLsn { return xerrors.Errorf("tried to add lsns in decreasing order. Last lsn: %v, new lsn: %v", p.lastLsn, newLsn) @@ -23,7 +23,7 @@ func (p *lsnTransaction) appendLsn(newLsn uint64) error { // here we go through lsns and add to the new slice those that were not found in pushedLsns // we also check that we only try to remove lsns that were previously added -// objects in pushedLsns must go in increasing order +// objects in pushedLsns must go in increasing order. func (p *lsnTransaction) removeLsn(pushedLsns []uint64) error { if err := checkOrder(pushedLsns); err != nil { return err @@ -38,7 +38,7 @@ func (p *lsnTransaction) removeLsn(pushedLsns []uint64) error { } if lsn == pushedLsns[pointer] { - pointer++ //found lsn to remove, don't add it to new slice + pointer++ // found lsn to remove, don't add it to new slice } else { newLsns = append(newLsns, lsn) } diff --git a/pkg/providers/postgres/sequencer/progress_info.go b/pkg/providers/postgres/sequencer/progress_info.go index ac12d527..9bf3c0f6 100644 --- a/pkg/providers/postgres/sequencer/progress_info.go +++ b/pkg/providers/postgres/sequencer/progress_info.go @@ -6,7 +6,7 @@ import ( // progressInfo responsible for grouping lsns by transactions // it also ensures that only lsns of committed transactions are sent back to server in Standby update -// and all transactions preceding the one that we send in Standby are already processed +// and all transactions preceding the one that we send in Standby are already processed. type progressInfo struct { processing map[uint32]*lsnTransaction // transaction id -> all lsns transactionIDs []uint32 // all transaction ids in correct order @@ -24,7 +24,7 @@ func (p *progressInfo) add(tid uint32, lsn uint64) error { return nil } -// check that this transaction is processing +// check that this transaction is processing. func (p *progressInfo) check(tid uint32) bool { _, ok := p.processing[tid] return ok @@ -42,7 +42,7 @@ func (p *progressInfo) remove(tid uint32, lsns []uint64) error { } func (p *progressInfo) updateCommitted() uint64 { - //in case last transaction is not complete yet + // in case last transaction is not complete yet if len(p.transactionIDs) <= 1 { return p.lastCommitted } diff --git a/pkg/providers/postgres/sequencer/sequencer.go b/pkg/providers/postgres/sequencer/sequencer.go index e26042c2..9e61ff88 100644 --- a/pkg/providers/postgres/sequencer/sequencer.go +++ b/pkg/providers/postgres/sequencer/sequencer.go @@ -7,13 +7,13 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// Sequencer takes items and updates progressInfo +// Sequencer takes items and updates progressInfo. type Sequencer struct { mutex sync.Mutex progress *progressInfo } -// StartProcessing receives changes in correct transaction order because we read from server synchronously +// StartProcessing receives changes in correct transaction order because we read from server synchronously. func (s *Sequencer) StartProcessing(changes []abstract.ChangeItem) error { s.mutex.Lock() defer s.mutex.Unlock() diff --git a/pkg/providers/postgres/sink.go b/pkg/providers/postgres/sink.go index 02a10f49..77d68bd6 100644 --- a/pkg/providers/postgres/sink.go +++ b/pkg/providers/postgres/sink.go @@ -136,7 +136,7 @@ func prepareOriginalTypes(schema []abstract.ColSchema) error { return nil } -// CreateSchemaQuery returns a CREATE SCHEMA IF NOT EXISTS query for the given schema +// CreateSchemaQuery returns a CREATE SCHEMA IF NOT EXISTS query for the given schema. func createSchemaQuery(schemaName string) string { return fmt.Sprintf(`CREATE SCHEMA IF NOT EXISTS "%v"; `, schemaName) } @@ -537,13 +537,13 @@ func Represent(val interface{}, colSchema abstract.ColSchema) (string, error) { if strings.HasPrefix(colSchema.OriginalType, "pg:time") && !strings.HasPrefix(colSchema.OriginalType, "pg:timestamp") { - //by default Value of time always returns as array of bytes which can not be processed in plain insert - //however if we cast decoder to pgtype.Time while unmarshalling it will lead to errors in tests because + // by default Value of time always returns as array of bytes which can not be processed in plain insert + // however if we cast decoder to pgtype.Time while unmarshalling it will lead to errors in tests because //pgtype.Time doesn't store the precision and always uses the maximum(6) if vvv, ok := vv.([]byte); ok && vv != nil { coder := new(pgtype.Time) - //we only use binary->binary (de)serialization in homogeneous pg->pg + // we only use binary->binary (de)serialization in homogeneous pg->pg if err := coder.DecodeBinary(nil, vvv); err == nil { //nolint:descriptiveerrors return Represent(coder, colSchema) @@ -876,7 +876,7 @@ func (s *sink) buildDeleteQuery(table string, schema []abstract.ColSchema, row a } // buildQueries constructs a list of SQL queries to push the given items into PostgreSQL. -// Each query in non-nil output strictly corresponds to an item in the given items +// Each query in non-nil output strictly corresponds to an item in the given items. func (s *sink) buildQueries(table string, schema []abstract.ColSchema, items []abstract.ChangeItem) ([]string, error) { rev := abstract.MakeMapColNameToIndex(schema) @@ -901,7 +901,7 @@ func (s *sink) buildQueries(table string, schema []abstract.ColSchema, items []a return result, nil } -// buildQuery is a wrapper around buildQueries which makes sure the output contains exactly one query +// buildQuery is a wrapper around buildQueries which makes sure the output contains exactly one query. func (s *sink) buildQuery(table string, schema []abstract.ColSchema, items []abstract.ChangeItem) (string, error) { queries, err := s.buildQueries(table, schema, items) if err != nil { @@ -946,7 +946,7 @@ func (s *sink) insert(ctx context.Context, table string, schema []abstract.ColSc return xerrors.Errorf("failed to build queries to process items at sink: %w", err) } - //s.logger.Infof("Prepare query %v rows %v for table %v", len(items), format.SizeInt(len(queries)), table) + // s.logger.Infof("Prepare query %v rows %v for table %v", len(items), format.SizeInt(len(queries)), table) execStart := time.Now() processedQueries := 0 for processedQueries < len(queries) { diff --git a/pkg/providers/postgres/sqltimestamp/parse.go b/pkg/providers/postgres/sqltimestamp/parse.go index 233cc21e..69b4f677 100644 --- a/pkg/providers/postgres/sqltimestamp/parse.go +++ b/pkg/providers/postgres/sqltimestamp/parse.go @@ -14,7 +14,7 @@ var InfiniteTimestampError = xerrors.NewSentinel("infinity is unparseable by the // Parse converts a text representation of an PostgreSQL-formatted date, timestamp, or timestamptz into time.Time. // In contrast with pgtype parsers, this parser supports BC years. // -// TODO: remove this when https://st.yandex-team.ru/TM-5127 is done +// TODO: remove this when https://st.yandex-team.ru/TM-5127 is done. func Parse(timestamp string) (time.Time, error) { switch timestamp { case "": diff --git a/pkg/providers/postgres/storage.go b/pkg/providers/postgres/storage.go index 004f4a61..10d87a9e 100644 --- a/pkg/providers/postgres/storage.go +++ b/pkg/providers/postgres/storage.go @@ -276,7 +276,7 @@ func (s *Storage) Ping() error { return nil } -// TableList in PostgreSQL returns a table map with schema +// TableList in PostgreSQL returns a table map with schema. func (s *Storage) TableList(filter abstract.IncludeTableList) (abstract.TableMap, error) { ctx := context.TODO() @@ -486,7 +486,7 @@ func (s *Storage) TableExists(table abstract.TableID) (bool, error) { return exists, nil } -// RowCount queries the storage and returns the exact number of rows in the given table +// RowCount queries the storage and returns the exact number of rows in the given table. func RowCount(ctx context.Context, conn *pgx.Conn, table *abstract.TableDescription) (uint64, error) { query := exactCountQuery(table) var count uint64 @@ -999,7 +999,7 @@ func ParseLsn(lsn string) uint64 { return outputLsn } -// go-sumtype:decl StorageOpt +// go-sumtype:decl StorageOpt. type StorageOpt interface { isStorageOpt() } @@ -1029,7 +1029,7 @@ func (s *Storage) RunSlotMonitor(ctx context.Context, serverSource interface{}, } } -// Named BeginPGSnapshot to NOT match abstract.SnapshotableStorage +// Named BeginPGSnapshot to NOT match abstract.SnapshotableStorage. func (s *Storage) BeginPGSnapshot(ctx context.Context) error { rb := util.Rollbacks{} defer rb.Do() @@ -1094,7 +1094,7 @@ func isAurora(ctx context.Context, conn pgxtype.Querier, lgr log.Logger) bool { return true } -// Named EndPGSnapshot to NOT match abstract.SnapshotableStorage +// Named EndPGSnapshot to NOT match abstract.SnapshotableStorage. func (s *Storage) EndPGSnapshot(ctx context.Context) error { if s.snapshotConnection == nil || s.snapshotTransaction == nil { return nil diff --git a/pkg/providers/postgres/timestamp.go b/pkg/providers/postgres/timestamp.go index 7b34312f..6397e307 100644 --- a/pkg/providers/postgres/timestamp.go +++ b/pkg/providers/postgres/timestamp.go @@ -10,7 +10,7 @@ import ( "github.com/jackc/pgtype" ) -// TimeZoneParameterStatusKey is the identifier of the PostgreSQL connection property containing time zone +// TimeZoneParameterStatusKey is the identifier of the PostgreSQL connection property containing time zone. const TimeZoneParameterStatusKey string = "TimeZone" type Timestamp struct { @@ -23,7 +23,7 @@ var _ TextDecoderAndValuerWithHomo = (*Timestamp)(nil) // NewTimestamp constructs a TIMESTAMP WITHOUT TIME ZONE representation which supports BC years // -// TODO: this type must become significantly simpler after https://st.yandex-team.ru/TM-5127 is done +// TODO: this type must become significantly simpler after https://st.yandex-team.ru/TM-5127 is done. func NewTimestamp(tz *time.Location) *Timestamp { return &Timestamp{ Timestamp: *(new(pgtype.Timestamp)), @@ -75,7 +75,7 @@ func actualYear(t time.Time) int { return result } -// MinusToBC checks if the given string starts with a minus and if so, trims it and adds a "BC" suffix +// MinusToBC checks if the given string starts with a minus and if so, trims it and adds a "BC" suffix. func MinusToBC(v string) string { if strings.HasPrefix(v, "-") { return strings.TrimPrefix(v, "-") + " BC" diff --git a/pkg/providers/postgres/timestamptz.go b/pkg/providers/postgres/timestamptz.go index 993be3e3..560990c7 100644 --- a/pkg/providers/postgres/timestamptz.go +++ b/pkg/providers/postgres/timestamptz.go @@ -16,10 +16,10 @@ var _ TextDecoderAndValuerWithHomo = (*Timestamptz)(nil) // NewTimestamptz constructs a TIMESTAMP WITH TIME ZONE representation which supports BC years // -// TODO: remove this when https://st.yandex-team.ru/TM-5127 is done +// TODO: remove this when https://st.yandex-team.ru/TM-5127 is done. func NewTimestamptz() *Timestamptz { return &Timestamptz{ - Timestamptz: *new(pgtype.Timestamptz), + Timestamptz: pgtype.Timestamptz{}, } } diff --git a/pkg/providers/postgres/txutils.go b/pkg/providers/postgres/txutils.go index 93259b42..bf873c0c 100644 --- a/pkg/providers/postgres/txutils.go +++ b/pkg/providers/postgres/txutils.go @@ -24,7 +24,7 @@ func BeginTx(ctx context.Context, conn *pgx.Conn, options pgx.TxOptions, lgr log return tx, &rollbacks, nil } -// RollbackFuncForPgxTx returns a function which ROLLBACKs the given `pgx.Tx` and if an error happens, logs it with the given logger at WARNING level +// RollbackFuncForPgxTx returns a function which ROLLBACKs the given `pgx.Tx` and if an error happens, logs it with the given logger at WARNING level. func RollbackFuncForPgxTx(ctx context.Context, tx pgx.Tx, lgr log.Logger) func() { return func() { if err := tx.Rollback(ctx); err != nil { diff --git a/pkg/providers/postgres/unmarshaller.go b/pkg/providers/postgres/unmarshaller.go index 187b6e33..d6c63bae 100644 --- a/pkg/providers/postgres/unmarshaller.go +++ b/pkg/providers/postgres/unmarshaller.go @@ -58,7 +58,7 @@ func NewUnmarshaller(castData *UnmarshallerData, connInfo *pgtype.ConnInfo, sche return result, nil } -// Cast consumes raw SELECT output and produces a valid ChangeItem.Value +// Cast consumes raw SELECT output and produces a valid ChangeItem.Value. func (c *Unmarshaller) Cast(input []byte) (any, error) { if c.decoder == nil { if err := c.reconstructDecoder(); err != nil { @@ -87,7 +87,7 @@ func (c *Unmarshaller) Cast(input []byte) (any, error) { return nil, abstract.NewFatalError(xerrors.Errorf("unknown decoder format code in cast: %d", c.fieldDesc.Format)) } - var result any = nil + var result any if c.castData.isHomo { result = unmarshalFieldHomo(c.decoder, c.schema, c.connInfo) c.decoder = nil diff --git a/pkg/providers/postgres/wal2json_parser_test.go b/pkg/providers/postgres/wal2json_parser_test.go index 69c84a55..382cbf07 100644 --- a/pkg/providers/postgres/wal2json_parser_test.go +++ b/pkg/providers/postgres/wal2json_parser_test.go @@ -12,7 +12,7 @@ import ( ) var ( - // Test input {{{ + // Test input {{{. changesets = [][]byte{[]byte(`{ "xid": 47931104, "nextlsn": "6E/85004178", @@ -111,7 +111,7 @@ var ( ] }`)} // }}} - // Test canonical data {{{ + // Test canonical data {{{. multipleChangesetsCanonical = []Wal2JSONItem{ { ID: 47931104, diff --git a/pkg/providers/provider.go b/pkg/providers/provider.go index 9709c20f..c2c572f2 100644 --- a/pkg/providers/provider.go +++ b/pkg/providers/provider.go @@ -16,45 +16,45 @@ type Provider interface { } // Snapshot add to provider `abstract.Storage` factory to provider. -// this means that provider can read historycal snapshots of data +// this means that provider can read historycal snapshots of data. type Snapshot interface { Provider Storage() (abstract.Storage, error) } // Replication add to provider `abstract.Source` factory to provider. -// this means that provider can do data replication +// this means that provider can do data replication. type Replication interface { Provider Source() (abstract.Source, error) } // Abstract2Provider add `base.DataProvider` factory to provider. -// this means that provider can do abstract2 data provider +// this means that provider can do abstract2 data provider. type Abstract2Provider interface { Provider DataProvider() (base.DataProvider, error) } -// Abstract2Sinker add abstract2 writer factory to provider +// Abstract2Sinker add abstract2 writer factory to provider. type Abstract2Sinker interface { Provider Target(...abstract.SinkOption) (base.EventTarget, error) } -// Sinker add generic writer factory to provider +// Sinker add generic writer factory to provider. type Sinker interface { Provider Sink(config middlewares.Config) (abstract.Sinker, error) } -// SnapshotSinker optional separate writer for snapshots. Will always called for snapshots with all control events +// SnapshotSinker optional separate writer for snapshots. Will always called for snapshots with all control events. type SnapshotSinker interface { Provider SnapshotSink(config middlewares.Config) (abstract.Sinker, error) } -// AsyncSinker add ability to setup async-sink instead of sync-sink for provider +// AsyncSinker add ability to setup async-sink instead of sync-sink for provider. type AsyncSinker interface { Provider AsyncSink(middleware abstract.Middleware) (abstract.AsyncSink, error) diff --git a/pkg/providers/provider_tasks.go b/pkg/providers/provider_tasks.go index 3a82ade4..795a2d9c 100644 --- a/pkg/providers/provider_tasks.go +++ b/pkg/providers/provider_tasks.go @@ -50,7 +50,7 @@ type Tester interface { Test(ctx context.Context) *abstract.TestResult } -// Peeker is a thing that allow to sniff data replication sample +// Peeker is a thing that allow to sniff data replication sample. type Sniffer interface { Provider Sniffer(ctx context.Context) (abstract.Fetchable, error) diff --git a/pkg/providers/s3/model_source.go b/pkg/providers/s3/model_source.go index a3dfc9a0..ca5e77f6 100644 --- a/pkg/providers/s3/model_source.go +++ b/pkg/providers/s3/model_source.go @@ -19,7 +19,7 @@ var _ model.Source = (*S3Source)(nil) const ( // defaultReadBatchSize is magic number by in-leskin, impacts how many rows we push each times - // we need to push rather small chunks so our bufferer can buffer effectively + // we need to push rather small chunks so our bufferer can buffer effectively. defaultReadBatchSize = 128 // defaultBlockSize impacts how many bytes we read fon each request from S3 bucket // its also used in replication as a mem limit to how many inflight bytes we can have. diff --git a/pkg/providers/s3/provider/provider.go b/pkg/providers/s3/provider/provider.go index 5adc36fb..bd6020b8 100644 --- a/pkg/providers/s3/provider/provider.go +++ b/pkg/providers/s3/provider/provider.go @@ -22,7 +22,7 @@ func init() { providers.Register(s3.ProviderType, New) } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) _ providers.Snapshot = (*Provider)(nil) diff --git a/pkg/providers/s3/pusher/pusher_state.go b/pkg/providers/s3/pusher/pusher_state.go index b19ec221..8aa46f4e 100644 --- a/pkg/providers/s3/pusher/pusher_state.go +++ b/pkg/providers/s3/pusher/pusher_state.go @@ -87,7 +87,7 @@ func (s *PusherState) removeOffset(toRemove any, offsets []any) []any { return remaining } -// DeleteDone delete's a processed files form state if the read process is completed +// DeleteDone delete's a processed files form state if the read process is completed. func (s *PusherState) deleteDone(filePath string) { // to be called on commit of state to, to keep map as small as possible progress, ok := s.PushProgress[filePath] @@ -137,5 +137,5 @@ func (s *PusherState) addInflight(size int64) { func (s *PusherState) reduceInflight(size int64) { s.mu.Lock() defer s.mu.Unlock() - s.inflightBytes = s.inflightBytes - size + s.inflightBytes -= size } diff --git a/pkg/providers/s3/reader/estimator_test.go b/pkg/providers/s3/reader/estimator_test.go index e3c21f2b..1cf93098 100644 --- a/pkg/providers/s3/reader/estimator_test.go +++ b/pkg/providers/s3/reader/estimator_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -// Reader function to return dummy S3Reader with specified sizes +// Reader function to return dummy S3Reader with specified sizes. func dummyReaderF(sizes map[string]int64) readerF { return func(ctx context.Context, filePath string) (*S3Reader, error) { size, exists := sizes[filePath] diff --git a/pkg/providers/s3/reader/reader_csv.go b/pkg/providers/s3/reader/reader_csv.go index d0a0cb9b..56ad94a5 100644 --- a/pkg/providers/s3/reader/reader_csv.go +++ b/pkg/providers/s3/reader/reader_csv.go @@ -158,7 +158,7 @@ func (r *CSVReader) Read(ctx context.Context, filePath string, pusher chunk_push return xerrors.Errorf("failed to read fom S3 file: %w", err) } - currentOffset := int64(0) + var currentOffset int64 for { buff, err := r.ParseCSVRows(csvReader, filePath, s3Reader.LastModified(), &lineCounter) if err != nil { diff --git a/pkg/providers/s3/reader/reader_json_line.go b/pkg/providers/s3/reader/reader_json_line.go index ce892cfe..27addd4c 100644 --- a/pkg/providers/s3/reader/reader_json_line.go +++ b/pkg/providers/s3/reader/reader_json_line.go @@ -415,7 +415,7 @@ func readAllLines(content []byte) ([]string, int, error) { for index, line := range scannedLines { if index == len(scannedLines)-1 { // check if last line is complete - if err := fastjson.Validate(string(line)); err != nil { + if err := fastjson.Validate(line); err != nil { break } } diff --git a/pkg/providers/s3/reader/s3_reader.go b/pkg/providers/s3/reader/s3_reader.go index 78508ead..ce728109 100644 --- a/pkg/providers/s3/reader/s3_reader.go +++ b/pkg/providers/s3/reader/s3_reader.go @@ -24,7 +24,7 @@ type S3Reader struct { reader io.ReaderAt } -// ReadAt is a proxy call to the underlying reader implementation +// ReadAt is a proxy call to the underlying reader implementation. func (r *S3Reader) ReadAt(p []byte, off int64) (int, error) { read, err := r.reader.ReadAt(p, off) if err != nil && !xerrors.Is(err, io.EOF) { @@ -33,12 +33,12 @@ func (r *S3Reader) ReadAt(p []byte, off int64) (int, error) { return read, err } -// Size is a proxy call to the underlying fetcher method +// Size is a proxy call to the underlying fetcher method. func (r *S3Reader) Size() int64 { return r.fetcher.size() } -// Size is a proxy call to the underlying fetcher method +// Size is a proxy call to the underlying fetcher method. func (r *S3Reader) LastModified() time.Time { return r.fetcher.lastModified() } diff --git a/pkg/providers/s3/sink/file_cache.go b/pkg/providers/s3/sink/file_cache.go index 29c15b72..45aa661f 100644 --- a/pkg/providers/s3/sink/file_cache.go +++ b/pkg/providers/s3/sink/file_cache.go @@ -27,7 +27,7 @@ func (f *FileCache) Add(item *abstract.ChangeItem) error { return nil } -// extra copy, but works fine with range, useful for tests +// extra copy, but works fine with range, useful for tests. func (f *FileCache) AddCopy(item abstract.ChangeItem) error { return f.Add(&item) } @@ -35,7 +35,7 @@ func (f *FileCache) AddCopy(item abstract.ChangeItem) error { // Split file cache into file cache parts. Each cache part // has items that contain in one of then given intervals // and with consecutive LSNs and size that le than maxCacheSize -// NB intervals range is expected to be sorted +// NB intervals range is expected to be sorted. func (f *FileCache) Split(intervals []ObjectRange, maxCacheSize uint64) []*FileCache { var parts = make([]*FileCache, 0) if len(intervals) == 0 { diff --git a/pkg/providers/s3/sink/object_range.go b/pkg/providers/s3/sink/object_range.go index 03434c9e..1ba4be35 100644 --- a/pkg/providers/s3/sink/object_range.go +++ b/pkg/providers/s3/sink/object_range.go @@ -5,14 +5,6 @@ type ObjectRange struct { To uint64 `json:"to"` } -func (o *ObjectRange) isEqual(object ObjectRange) bool { - return o.From == object.From && o.To == object.To -} - -func (o *ObjectRange) isSubset(object ObjectRange) bool { - return o.From >= object.From && o.To <= object.To -} - func NewObjectRange(from, to uint64) ObjectRange { return ObjectRange{ From: from, diff --git a/pkg/providers/s3/sink/testutil/fake_client.go b/pkg/providers/s3/sink/testutil/fake_client.go index 50c882e1..f594a403 100644 --- a/pkg/providers/s3/sink/testutil/fake_client.go +++ b/pkg/providers/s3/sink/testutil/fake_client.go @@ -4,7 +4,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract/coordinator" ) -// FakeClientWithTransferState is a fake controlplane client which stores sharded object transfer state +// FakeClientWithTransferState is a fake controlplane client which stores sharded object transfer state. type FakeClientWithTransferState struct { coordinator.CoordinatorNoOp state map[string]*coordinator.TransferStateData diff --git a/pkg/providers/s3/sink/uploader.go b/pkg/providers/s3/sink/uploader.go index 8835d692..bd9a2364 100644 --- a/pkg/providers/s3/sink/uploader.go +++ b/pkg/providers/s3/sink/uploader.go @@ -18,9 +18,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -var ( - FatalAWSCodes = set.New("InvalidAccessKeyId") -) +var FatalAWSCodes = set.New("InvalidAccessKeyId") type replicationUploader struct { cfg *s3_provider.S3Destination @@ -36,7 +34,7 @@ func (u *replicationUploader) Upload(name string, lsns []uint64, data []byte) er fileName = fmt.Sprintf("%v-%v_%v.%v", name, lsns[0], lsns[len(lsns)-1], strings.ToLower(string(u.cfg.OutputFormat))) } if u.cfg.OutputEncoding == s3_provider.GzipEncoding { - fileName = fileName + ".gz" + fileName += ".gz" gzWriter := gzip.NewWriter(buf) if _, err := gzWriter.Write(data); err != nil { return err diff --git a/pkg/providers/s3/source/sharded_replication_test/sharded_replication_test.go b/pkg/providers/s3/source/sharded_replication_test/sharded_replication_test.go index 0bf106a8..35e1aa59 100644 --- a/pkg/providers/s3/source/sharded_replication_test/sharded_replication_test.go +++ b/pkg/providers/s3/source/sharded_replication_test/sharded_replication_test.go @@ -89,7 +89,7 @@ func TestNativeS3PathsAreUnescaped(t *testing.T) { Region: aws.String(sqsRegion), S3ForcePathStyle: aws.Bool(src.ConnectionConfig.S3ForcePathStyle), Credentials: credentials.NewStaticCredentials( - sqsUser, string(sqsQueueName), "", + sqsUser, sqsQueueName, "", ), }) require.NoError(t, err) diff --git a/pkg/providers/s3/storage/storage_incremental.go b/pkg/providers/s3/storage/storage_incremental.go index 6444a77e..a932b2ae 100644 --- a/pkg/providers/s3/storage/storage_incremental.go +++ b/pkg/providers/s3/storage/storage_incremental.go @@ -13,7 +13,7 @@ import ( "github.com/doublecloud/transfer/pkg/providers/s3/reader" ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ abstract.IncrementalStorage = (*Storage)(nil) ) diff --git a/pkg/providers/s3/storage/storage_sharding.go b/pkg/providers/s3/storage/storage_sharding.go index e467ef15..1993d5df 100644 --- a/pkg/providers/s3/storage/storage_sharding.go +++ b/pkg/providers/s3/storage/storage_sharding.go @@ -12,7 +12,7 @@ import ( "github.com/doublecloud/transfer/pkg/providers/s3/reader" ) -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ abstract.ShardingStorage = (*Storage)(nil) ) diff --git a/pkg/providers/stdout/provider.go b/pkg/providers/stdout/provider.go index 1323b5bd..dd920017 100644 --- a/pkg/providers/stdout/provider.go +++ b/pkg/providers/stdout/provider.go @@ -36,7 +36,7 @@ func sourceModelFactory() model.Source { const ProviderTypeStdout = abstract.ProviderType("stdout") const ProviderType = abstract.ProviderType("empty") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Sinker = (*Provider)(nil) ) diff --git a/pkg/providers/ydb/decimal/parse.go b/pkg/providers/ydb/decimal/parse.go index 5dc215de..ffc6fe60 100644 --- a/pkg/providers/ydb/decimal/parse.go +++ b/pkg/providers/ydb/decimal/parse.go @@ -3,11 +3,8 @@ package decimal import ( "fmt" "math/big" - "math/bits" ) -const wordSize = bits.UintSize / 8 - var ( ten = big.NewInt(10) zero = big.NewInt(0) diff --git a/pkg/providers/ydb/logadapter/adapter.go b/pkg/providers/ydb/logadapter/adapter.go index 83301c62..55beecc5 100644 --- a/pkg/providers/ydb/logadapter/adapter.go +++ b/pkg/providers/ydb/logadapter/adapter.go @@ -7,7 +7,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -var _ ydbLog.Logger = *new(adapter) +var _ ydbLog.Logger = adapter{} type adapter struct { l log.Logger diff --git a/pkg/providers/ydb/provider.go b/pkg/providers/ydb/provider.go index 73dbc73e..31277e12 100644 --- a/pkg/providers/ydb/provider.go +++ b/pkg/providers/ydb/provider.go @@ -30,7 +30,7 @@ func init() { const ProviderType = abstract.ProviderType("ydb") -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Snapshot = (*Provider)(nil) _ providers.Replication = (*Provider)(nil) diff --git a/pkg/providers/ydb/sink.go b/pkg/providers/ydb/sink.go index c66e25db..78d8a9eb 100644 --- a/pkg/providers/ydb/sink.go +++ b/pkg/providers/ydb/sink.go @@ -136,7 +136,7 @@ const ( // based on // https://ydb.tech/ru/docs/yql/reference/types/primitive // https://ydb.tech/ru/docs/concepts/column-table#olap-data-types -// unmentioned types can't be primary keys +// unmentioned types can't be primary keys. var primaryIsAllowedFor = map[types.Type]AllowedIn{ // we cast bool to uint8 for OLAP tables types.TypeBool: BOTH, @@ -818,7 +818,6 @@ func (s *sinker) insert(tablePath ydbPath, batch []abstract.ChangeItem) error { } return nil }) - if err != nil { return xerrors.Errorf("unable to insert with legacy writer:\n %w", err) } @@ -1022,15 +1021,15 @@ func (s *sinker) ydbVal(dataType, originalType string, val interface{}) (types.V case schema.TypeBytes: switch v := val.(type) { case string: - return types.StringValue([]byte(v)), false, nil + return types.BytesValue([]byte(v)), false, nil case []uint8: - return types.StringValue(v), false, nil + return types.BytesValue(v), false, nil default: r, err := json.Marshal(val) if err != nil { return nil, false, xerrors.Errorf("unable to json marshal: %w", err) } - return types.StringValue(r), false, nil + return types.BytesValue(r), false, nil } case schema.TypeString: switch v := val.(type) { diff --git a/pkg/providers/ydb/source_tasks.go b/pkg/providers/ydb/source_tasks.go index 370cc3c5..27c5c1a8 100644 --- a/pkg/providers/ydb/source_tasks.go +++ b/pkg/providers/ydb/source_tasks.go @@ -82,7 +82,7 @@ func createChangeFeedOneTable(ctx context.Context, ydbClient *ydb.Driver, tableP } // checkChangeFeedConsumerOnline -// with this method we identify changefeed is active if our system consumer is attached to it as well +// with this method we identify changefeed is active if our system consumer is attached to it as well. func checkChangeFeedConsumerOnline(ctx context.Context, ydbClient *ydb.Driver, tablePath, transferID string) (bool, error) { topicPath := makeChangeFeedPath(tablePath, transferID) descr, err := ydbClient.Topic().Describe(ctx, topicPath) diff --git a/pkg/providers/ydb/source_test.go b/pkg/providers/ydb/source_test.go index f8f7e775..1e940b14 100644 --- a/pkg/providers/ydb/source_test.go +++ b/pkg/providers/ydb/source_test.go @@ -252,7 +252,7 @@ func checkSchemaUpdateWithMode(t *testing.T, db *ydb.Driver, transferID string, } // events with the same primary keys are ordered, -// but not ordered relative to events for records with other keys +// but not ordered relative to events for records with other keys. func checkEventsOrder(t *testing.T, events []abstract.ChangeItem, expectedVal int) { if len(events) == 0 { return diff --git a/pkg/providers/ydb/storage_sampleable.go b/pkg/providers/ydb/storage_sampleable.go index 30c6a9be..e5f77377 100644 --- a/pkg/providers/ydb/storage_sampleable.go +++ b/pkg/providers/ydb/storage_sampleable.go @@ -8,21 +8,21 @@ func (s *Storage) TableSizeInBytes(table abstract.TableID) (uint64, error) { } func (s *Storage) LoadTopBottomSample(table abstract.TableDescription, pusher abstract.Pusher) error { - //TODO implement me + // TODO implement me panic("implement me") } func (s *Storage) LoadRandomSample(table abstract.TableDescription, pusher abstract.Pusher) error { - //TODO implement me + // TODO implement me panic("implement me") } func (s *Storage) LoadSampleBySet(table abstract.TableDescription, keySet []map[string]interface{}, pusher abstract.Pusher) error { - //TODO implement me + // TODO implement me panic("implement me") } func (s *Storage) TableAccessible(table abstract.TableDescription) bool { - //TODO implement me + // TODO implement me panic("implement me") } diff --git a/pkg/providers/ydb/utils.go b/pkg/providers/ydb/utils.go index e23f50e8..0eb06e63 100644 --- a/pkg/providers/ydb/utils.go +++ b/pkg/providers/ydb/utils.go @@ -29,9 +29,9 @@ func filterYdbTableColumns(filter []YdbColumnsFilter, description options.Descri if err != nil { return nil, xerrors.Errorf("unable to compile regexp: %s: %w", filterRule.ColumnNamesRegexp, err) } - var filteredColumns = make([]options.Column, 0) + filteredColumns := make([]options.Column, 0) for _, column := range description.Columns { - var hasMatch = columnsToFilterRegExp.MatchString(column.Name) + hasMatch := columnsToFilterRegExp.MatchString(column.Name) if (filterRule.Type == YdbColumnsWhiteList && hasMatch) || (filterRule.Type == YdbColumnsBlackList && !hasMatch) { filteredColumns = append(filteredColumns, column) @@ -51,18 +51,6 @@ func filterYdbTableColumns(filter []YdbColumnsFilter, description options.Descri return description.Columns, nil } -func flatten(batch [][]abstract.ChangeItem) []abstract.ChangeItem { - sumSize := 0 - for _, currArr := range batch { - sumSize += len(currArr) - } - result := make([]abstract.ChangeItem, 0, sumSize) - for _, currArr := range batch { - result = append(result, currArr...) - } - return result -} - func tableSchema(ctx context.Context, db *ydb.Driver, database string, tableID abstract.TableID) (*abstract.TableSchema, error) { tablePath := path.Join(database, tableID.Namespace, tableID.Name) desc, err := describeTable(ctx, db, tablePath) diff --git a/pkg/providers/yt/copy/source/source.go b/pkg/providers/yt/copy/source/source.go index 332b9195..787084b1 100644 --- a/pkg/providers/yt/copy/source/source.go +++ b/pkg/providers/yt/copy/source/source.go @@ -27,7 +27,7 @@ type source struct { metrics metrics.Registry } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ base.SnapshotProvider = (*source)(nil) ) diff --git a/pkg/providers/yt/cypress.go b/pkg/providers/yt/cypress.go index 5f4c1df5..9d339b88 100644 --- a/pkg/providers/yt/cypress.go +++ b/pkg/providers/yt/cypress.go @@ -31,7 +31,7 @@ func NewNodeInfo(name string, path ypath.Path, attrs *NodeAttrs) *NodeInfo { return &NodeInfo{Name: name, Path: path, Attrs: attrs} } -// ListNodesWithAttrs returns name-sorted list of nodes with attributes based on specified arguments +// ListNodesWithAttrs returns name-sorted list of nodes with attributes based on specified arguments. func ListNodesWithAttrs(ctx context.Context, client yt.CypressClient, path ypath.Path, prefix string, recursive bool) ([]*NodeInfo, error) { var nodes []string var err error diff --git a/pkg/providers/yt/executable.go b/pkg/providers/yt/executable.go index ffd4c3cf..e42ef648 100644 --- a/pkg/providers/yt/executable.go +++ b/pkg/providers/yt/executable.go @@ -3,9 +3,7 @@ package yt import ( "context" "io" - "math/rand" "os" - "time" "github.com/doublecloud/transfer/internal/logger" "github.com/doublecloud/transfer/library/go/core/xerrors" @@ -24,7 +22,7 @@ var ( exeVersion string ) -// InitExe uploads exe and initializes related variables +// InitExe uploads exe and initializes related variables. func InitExe() { if !env.IsTest() { return @@ -78,7 +76,6 @@ func uploadExe(exePrefix, exePath string) error { } defer client.Stop() - rand.Seed(time.Now().UnixNano()) exeVersion = exePrefix + randutil.GenerateAlphanumericString(8) ExePath = DataplaneExecutablePath("", exeVersion) if _, err := client.CreateNode(context.Background(), ExePath, yt.NodeFile, &yt.CreateNodeOptions{Recursive: true}); err != nil { diff --git a/pkg/providers/yt/init/provider.go b/pkg/providers/yt/init/provider.go index 4dff0df8..03f37fc5 100644 --- a/pkg/providers/yt/init/provider.go +++ b/pkg/providers/yt/init/provider.go @@ -32,7 +32,7 @@ func init() { providers.Register(yt_provider.CopyType, New(yt_provider.CopyType)) } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ providers.Snapshot = (*Provider)(nil) _ providers.Sinker = (*Provider)(nil) diff --git a/pkg/providers/yt/lfstaging/close_gaps.go b/pkg/providers/yt/lfstaging/close_gaps.go index 0e2e3be8..b0c30c4d 100644 --- a/pkg/providers/yt/lfstaging/close_gaps.go +++ b/pkg/providers/yt/lfstaging/close_gaps.go @@ -1,28 +1,12 @@ package lfstaging import ( - "strconv" - "strings" "time" "go.ytsaurus.tech/yt/go/yt" "golang.org/x/xerrors" ) -func getTableTimestamp(node ytNode) (int64, error) { - parts := strings.Split(node.Name, "-") - if len(parts) != 2 { - return 0, xerrors.Errorf("Invalid node name '%v'", node.Name) - } - - ts, err := strconv.ParseInt(parts[0], 10, 64) - if err != nil { - return 0, xerrors.Errorf("Cannot convert value '%v' to integer: %w", parts[0], err) - } else { - return ts, nil - } -} - func closeGaps( tx yt.Tx, config *sinkConfig, diff --git a/pkg/providers/yt/lfstaging/lock.go b/pkg/providers/yt/lfstaging/lock.go deleted file mode 100644 index b4f56b77..00000000 --- a/pkg/providers/yt/lfstaging/lock.go +++ /dev/null @@ -1,40 +0,0 @@ -package lfstaging - -import ( - "context" - "time" - - "go.ytsaurus.tech/yt/go/ypath" - "go.ytsaurus.tech/yt/go/yt" - "go.ytsaurus.tech/yt/go/ytlock" - "golang.org/x/xerrors" -) - -type lock struct { - path ypath.Path - yc yt.Client -} - -func newLock(yc yt.Client, path ypath.Path) *lock { - return &lock{ - path: path, - yc: yc, - } -} - -func (l *lock) WithLock(fn func() error) error { - lock := ytlock.NewLock(l.yc, l.path) - _, err := lock.Acquire(context.Background()) - if err != nil { - time.Sleep(10 * time.Minute) - return xerrors.Errorf("Cannot acquire lock: %w", err) - } - if err := fn(); err != nil { - return err - } - err = lock.Release(context.Background()) - if err != nil { - return xerrors.Errorf("Cannot release lock: %w", err) - } - return nil -} diff --git a/pkg/providers/yt/lfstaging/rows.go b/pkg/providers/yt/lfstaging/rows.go index 6773cdb8..fdb9dd31 100644 --- a/pkg/providers/yt/lfstaging/rows.go +++ b/pkg/providers/yt/lfstaging/rows.go @@ -62,8 +62,8 @@ func intermediateRowFromChangeItem(ci abstract.ChangeItem) (intermediateRow, err TopicName: namespacedTopicName, SourceURI: "data-transfer", SourceID: "example-dt-source-id", - CommitTimestampMs: int64(message.WriteTime.UnixMilli()), - Offset: int64(message.SeqNo), + CommitTimestampMs: message.WriteTime.UnixMilli(), + Offset: message.SeqNo, Shard: int64(message.Partition), Data: message.Data, }, nil diff --git a/pkg/providers/yt/model_yt_destination.go b/pkg/providers/yt/model_yt_destination.go index 28f303be..c76f5c5d 100644 --- a/pkg/providers/yt/model_yt_destination.go +++ b/pkg/providers/yt/model_yt_destination.go @@ -184,7 +184,7 @@ func (d *YtDestinationWrapper) SetParams(jsonStr string) error { return json.Unmarshal([]byte(jsonStr), &d.Model) } -// TODO: Remove in march +// TODO: Remove in march. func (d *YtDestinationWrapper) DisableDatetimeHack() bool { return d.Model.DisableDatetimeHack } @@ -502,7 +502,7 @@ func (d *YtDestinationWrapper) SupportSharding() bool { return !(d.Model.Static && d.Rotation() != nil) } -// this is kusok govna, it here for purpose - backward compatibility and no reuse without backward compatibility +// this is kusok govna, it here for purpose - backward compatibility and no reuse without backward compatibility. func (d *YtDestinationWrapper) LegacyModel() interface{} { return d.Model } diff --git a/pkg/providers/yt/provider/dataobjects/part.go b/pkg/providers/yt/provider/dataobjects/part.go index de142515..f95d90ad 100644 --- a/pkg/providers/yt/provider/dataobjects/part.go +++ b/pkg/providers/yt/provider/dataobjects/part.go @@ -112,7 +112,7 @@ func rangeToLegacyWhere(rng ypath.Range) abstract.WhereStatement { var condPattern = "($row_index %s %d)" -// LegacyWhereToRange is now unused pair to rangeToLegacyWhere. May be needed later for incremental transfers, etc +// LegacyWhereToRange is now unused pair to rangeToLegacyWhere. May be needed later for incremental transfers, etc. func LegacyWhereToRange(where abstract.WhereStatement) (ypath.Range, error) { rng := ypath.Full() diff --git a/pkg/providers/yt/provider/snapshot.go b/pkg/providers/yt/provider/snapshot.go index a5c9f73c..fd61cc6a 100644 --- a/pkg/providers/yt/provider/snapshot.go +++ b/pkg/providers/yt/provider/snapshot.go @@ -20,13 +20,13 @@ import ( "go.ytsaurus.tech/yt/go/yt" ) -// 16k batches * 2 MiByte per batch should be enough to fill buffer of size 32GiB +// 16k batches * 2 MiByte per batch should be enough to fill buffer of size 32GiB. const ( PushBatchSize = 2 * humanize.MiByte MaxInflightCount = 16384 // Max number of successfuly AsyncPush'd batches for which we may wait response from pusher ) -// Parallel table reader settings. These values are taken from YT python wrapper default config +// Parallel table reader settings. These values are taken from YT python wrapper default config. const ( parallelReadBatchSize = 8 * humanize.MiByte parallelTableReaders = 10 diff --git a/pkg/providers/yt/provider/source.go b/pkg/providers/yt/provider/source.go index 7aa88557..8c56ef65 100644 --- a/pkg/providers/yt/provider/source.go +++ b/pkg/providers/yt/provider/source.go @@ -29,7 +29,7 @@ type source struct { metrics *stats.SourceStats } -// To verify providers contract implementation +// To verify providers contract implementation. var ( _ base.SnapshotProvider = (*source)(nil) ) diff --git a/pkg/providers/yt/recipe/env.go b/pkg/providers/yt/recipe/env.go index 57a9a8c7..a0100d20 100644 --- a/pkg/providers/yt/recipe/env.go +++ b/pkg/providers/yt/recipe/env.go @@ -23,7 +23,7 @@ func NewEnv(t *testing.T, opts ...yttest.Option) (*yttest.Env, func()) { } proxy, err := container.ConnectionHost(ctx) require.NoError(t, err) - require.NoError(t, os.Setenv("YT_PROXY", proxy)) + t.Setenv("YT_PROXY", proxy) ytClient, err := container.NewClient(ctx) require.NoError(t, err) logger, stopLogger := yttest.NewLogger(t) diff --git a/pkg/providers/yt/sink/common.go b/pkg/providers/yt/sink/common.go index 5a686d04..f5e4f02d 100644 --- a/pkg/providers/yt/sink/common.go +++ b/pkg/providers/yt/sink/common.go @@ -245,7 +245,7 @@ func unionSchemas(current, expected schema.Schema) (schema.Schema, error) { } } - //preserve order of deleted non key columns to avoid unnecessary alters if old rows would be inserted + // preserve order of deleted non key columns to avoid unnecessary alters if old rows would be inserted for _, col := range current.Columns { _, notAdded := currentColumns[col.Name] if notAdded { @@ -425,7 +425,7 @@ func Restore(colSchema abstract.ColSchema, val interface{}) (interface{}, error) return abstract.Restore(colSchema, val), nil } -// TODO: Completely remove this legacy hack +// TODO: Completely remove this legacy hack. func fixDatetime(c *abstract.ColSchema) schema.Type { return schema.Type(strings.ToLower(c.DataType)) } diff --git a/pkg/providers/yt/sink/ordered_table.go b/pkg/providers/yt/sink/ordered_table.go index 97c719aa..7ee046aa 100644 --- a/pkg/providers/yt/sink/ordered_table.go +++ b/pkg/providers/yt/sink/ordered_table.go @@ -496,8 +496,8 @@ func NewOrderedTable(ytClient yt.Client, path ypath.Path, schema []abstract.ColS ctx, ytClient, yt2.SafeChild(dir, "meta", tableName+"__partition_to_tablet_index"), - *new(PartitionToTabletIndexKey), - *new(PartitionToTabletIndexVal), + PartitionToTabletIndexKey{}, + PartitionToTabletIndexVal{}, cfg.CellBundle(), map[string]interface{}{ "primary_medium": cfg.PrimaryMedium(), @@ -513,8 +513,8 @@ func NewOrderedTable(ytClient yt.Client, path ypath.Path, schema []abstract.ColS ctx, ytClient, yt2.SafeChild(dir, "meta", tableName+"__lb_offset_to_row_index"), - *new(LbOffsetToRowIndexKey), - *new(LbOffsetToRowIndexVal), + LbOffsetToRowIndexKey{}, + LbOffsetToRowIndexVal{}, cfg.CellBundle(), map[string]interface{}{ "primary_medium": cfg.PrimaryMedium(), diff --git a/pkg/providers/yt/sink/single_static_table.go b/pkg/providers/yt/sink/single_static_table.go index 00e29578..a6f8ed24 100644 --- a/pkg/providers/yt/sink/single_static_table.go +++ b/pkg/providers/yt/sink/single_static_table.go @@ -27,7 +27,7 @@ var _ GenericTable = new(SingleStaticTable) // SingleStaticTable is a specialization of StaticTable but for single table // which behaves like GenericTable for dynamic sinker. // Assumed that only one goroutine works with SingleStaticTable at moment, -// otherwise all methods should be protected with primitives in future +// otherwise all methods should be protected with primitives in future. type SingleStaticTable struct { ytClient yt.Client tableName string @@ -148,7 +148,7 @@ func (t *SingleStaticTable) init() error { // Write writes change items to table // Note: after calling this method object is still usable, but if -// error returned: object is NOT usable +// error returned: object is NOT usable. func (t *SingleStaticTable) Write(input []abstract.ChangeItem) error { var rollbacks util.Rollbacks defer rollbacks.Do() @@ -202,7 +202,7 @@ func (t *SingleStaticTable) write(input []abstract.ChangeItem) error { } // Commit tries to commit the transaction and performs some sorting and moving operations -// After commit object is not usable +// After commit object is not usable. func (t *SingleStaticTable) Commit(ctx context.Context) error { defer func() { err := t.tx.Abort() diff --git a/pkg/providers/yt/sink/single_static_table_test.go b/pkg/providers/yt/sink/single_static_table_test.go index 343e741a..9155a40a 100644 --- a/pkg/providers/yt/sink/single_static_table_test.go +++ b/pkg/providers/yt/sink/single_static_table_test.go @@ -28,7 +28,7 @@ const ( DefaultTimeout = 3 * time.Minute ) -// test runner +// test runner. func TestSingleStaticTable(t *testing.T) { t.Run("Simple", simple) t.Run("NilConfigNotAllowed", nilConfigNotAllowed) @@ -44,7 +44,7 @@ func TestSingleStaticTable(t *testing.T) { // Utilities -// Haskell-like map :: (a -> b) -> [a] -> [b] +// Haskell-like map :: (a -> b) -> [a] -> [b]. func mapValuesToChangeItems(f func([]interface{}) abstract.ChangeItem) func([][]interface{}) []abstract.ChangeItem { return func(ls [][]interface{}) []abstract.ChangeItem { var result []abstract.ChangeItem @@ -55,7 +55,7 @@ func mapValuesToChangeItems(f func([]interface{}) abstract.ChangeItem) func([][] } } -// initializes YT client and sinker config +// initializes YT client and sinker config. func initYt(t *testing.T, path string) (testCfg yt2.YtDestinationModel, client yt.Client) { cfg := yt2.NewYtDestinationV1(yt2.YtDestination{ Path: path, @@ -75,7 +75,7 @@ func initYt(t *testing.T, path string) (testCfg yt2.YtDestinationModel, client y } // create schema config and change items generator common for tests -// Schema 1: key-value schema where key is designated as primary key +// Schema 1: key-value schema where key is designated as primary key. type arcWarden struct { Key int32 `yson:"key"` Val string `yson:"val"` @@ -96,7 +96,7 @@ var toArcWardenItems = mapValuesToChangeItems(func(values []interface{}) abstrac } }) -// Schema 2: the same scheme, but 'key' is not designated as primary +// Schema 2: the same scheme, but 'key' is not designated as primary. type withoutKeys struct { Key int32 `yson:"key"` Val string `yson:"val"` @@ -117,7 +117,7 @@ var toWithoutKeysItems = mapValuesToChangeItems(func(values []interface{}) abstr } }) -// Schema 3: all keys schema +// Schema 3: all keys schema. type allKeys struct { Key int32 `yson:"K"` Val int32 `yson:"V"` @@ -138,7 +138,7 @@ var toAllKeysSchema = mapValuesToChangeItems(func(values []interface{}) abstract } }) -// tests +// tests. func simple(t *testing.T) { // create single static table for change item consumption path := ypath.Path("//home/cdc/test/TM-1572") diff --git a/pkg/providers/yt/sink/sink.go b/pkg/providers/yt/sink/sink.go index 6f2c7ad6..01071712 100644 --- a/pkg/providers/yt/sink/sink.go +++ b/pkg/providers/yt/sink/sink.go @@ -839,7 +839,7 @@ func (s *sinker) commitSnapshot(tableName string) error { return nil } -// private wrapper function with receiver +// private wrapper function with receiver. func (s *sinker) newGenericTable(schema []abstract.ColSchema, table string) (GenericTable, error) { return NewGenericTable(s.ytClient, yt2.SafeChild(s.dir, table), schema, s.config, s.metrics, s.logger) } diff --git a/pkg/providers/yt/sink/sorted_table.go b/pkg/providers/yt/sink/sorted_table.go index 1cb72e16..020335d1 100644 --- a/pkg/providers/yt/sink/sorted_table.go +++ b/pkg/providers/yt/sink/sorted_table.go @@ -234,7 +234,7 @@ func indexRowDeleter(ctx context.Context, tx yt.TabletTx, tablePath ypath.Path, return tx.DeleteRows(ctx, tablePath, keys, nil) } -// Write accept input which will be collapsed as very first step +// Write accept input which will be collapsed as very first step. func (t *SortedTable) Write(input []abstract.ChangeItem) error { input = abstract.Collapse(input) if len(t.config.Index()) > 0 { diff --git a/pkg/providers/yt/sink/static_table.go b/pkg/providers/yt/sink/static_table.go index b070ab20..64e1c784 100644 --- a/pkg/providers/yt/sink/static_table.go +++ b/pkg/providers/yt/sink/static_table.go @@ -197,15 +197,6 @@ func staticYTSchema(item abstract.ChangeItem) []schema.Column { return result } -func findCorrespondingIndex(cols []abstract.ColSchema, name string) int { - for i, colSchema := range cols { - if colSchema.ColumnName == name { - return i - } - } - return -1 -} - func getNameFromTableID(tID abstract.TableID) string { if tID.Namespace == "public" || len(tID.Namespace) == 0 { return tID.Name @@ -240,7 +231,7 @@ func (t *StaticTable) Push(items []abstract.ChangeItem) error { switch item.Kind { case abstract.InsertKind: if prevTableID != tableID { - ok := false + var ok bool writer, ok = t.getWriter(tableID) if !ok { diff --git a/pkg/providers/yt/sink/v2/transactions/state_storage.go b/pkg/providers/yt/sink/v2/transactions/state_storage.go index 53bfff0e..e74907b5 100644 --- a/pkg/providers/yt/sink/v2/transactions/state_storage.go +++ b/pkg/providers/yt/sink/v2/transactions/state_storage.go @@ -54,7 +54,7 @@ func (s *ytStateStorage) SetState(tx yt.TxID) error { return nil } -// RemoveState removes state and return deleted tx id +// RemoveState removes state and return deleted tx id. func (s *ytStateStorage) RemoveState() (*yt.TxID, error) { id, err := s.getState() if err != nil { diff --git a/pkg/providers/yt/sink/versioned_table.go b/pkg/providers/yt/sink/versioned_table.go index 7b150d11..09574870 100644 --- a/pkg/providers/yt/sink/versioned_table.go +++ b/pkg/providers/yt/sink/versioned_table.go @@ -460,7 +460,7 @@ func (t *VersionedTable) buildTargetTable(schemas []abstract.ColSchema) (migrate } // less will check whether left *less* than right -// it will give asc order for standard slices sort +// it will give asc order for standard slices sort. func (t *VersionedTable) less(left, right interface{}) bool { switch schema.Type(t.versionCol.DataType) { case schema.TypeFloat64, schema.TypeFloat32: diff --git a/pkg/randutil/randutil.go b/pkg/randutil/randutil.go index e198ce31..1d33bc46 100644 --- a/pkg/randutil/randutil.go +++ b/pkg/randutil/randutil.go @@ -1,12 +1,10 @@ package randutil import ( - "math/rand" + "crypto/rand" ) -var ( - AlphanumericValues string -) +var AlphanumericValues string func init() { var alphanumericValues []byte diff --git a/pkg/runtime/local/logger_injestor.go b/pkg/runtime/local/logger_injestor.go index d1769415..b71bef72 100644 --- a/pkg/runtime/local/logger_injestor.go +++ b/pkg/runtime/local/logger_injestor.go @@ -5,7 +5,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// WithLogger temproray hack to injest global logger into dataplane +// WithLogger temproray hack to injest global logger into dataplane. func WithLogger(lgr log.Logger) { logger.Log = log.With(lgr, log.Any("component", "dataplane")) logger.Log.Info("override logger inside data plane") diff --git a/pkg/runtime/local/replication.go b/pkg/runtime/local/replication.go index 90d784b9..fdaedbec 100644 --- a/pkg/runtime/local/replication.go +++ b/pkg/runtime/local/replication.go @@ -35,8 +35,10 @@ type Spec struct { const ReplicationStatusMessagesCategory string = "replication" -const healthReportPeriod time.Duration = 1 * time.Minute -const replicationRetryInterval time.Duration = 10 * time.Second +const ( + healthReportPeriod time.Duration = 1 * time.Minute + replicationRetryInterval time.Duration = 10 * time.Second +) func RunReplicationWithMeteringTags(ctx context.Context, cp coordinator.Coordinator, transfer *model.Transfer, registry metrics.Registry, runtimeTags map[string]interface{}) error { metering.InitializeWithTags(transfer, nil, runtimeTags) @@ -90,7 +92,7 @@ func replicationAttempt(ctx context.Context, cp coordinator.Coordinator, transfe replicationStats.Running.Set(float64(0)) }() - var attemptErr error = nil + var attemptErr error healthReportTicker := time.NewTicker(healthReportPeriod) defer healthReportTicker.Stop() @@ -158,7 +160,7 @@ func reportTransferHealth(ctx context.Context, cp coordinator.Coordinator, trans } } -// Does not return unless an error occurs +// Does not return unless an error occurs. func iteration(ctx context.Context, cp coordinator.Coordinator, dataFlow *model.Transfer, registry metrics.Registry, lgr log.Logger) (err error) { defer func() { if r := recover(); r != nil { diff --git a/pkg/runtime/local/task_sync_runtime.go b/pkg/runtime/local/task_sync_runtime.go index b502a115..67ff5dd6 100644 --- a/pkg/runtime/local/task_sync_runtime.go +++ b/pkg/runtime/local/task_sync_runtime.go @@ -51,7 +51,7 @@ func (s *SyncTask) run() { } // NewSyncTask only used for local debug, can operate properly only on single machine transfer server installation -// with enable `all_in_one_binary` flag +// with enable `all_in_one_binary` flag. func NewSyncTask( task *model.TransferOperation, cp coordinator.Coordinator, diff --git a/pkg/runtime/shared/limits.go b/pkg/runtime/shared/limits.go index c8ef6d76..85301e38 100644 --- a/pkg/runtime/shared/limits.go +++ b/pkg/runtime/shared/limits.go @@ -18,7 +18,7 @@ const ( // Some amount of memory is reserved for unaccounted memory (e.g. CGO) and system needs // and for algorithmic imperfections of the limiter implementation. // The reserve amounts to memLimitReservePerc * 100 percents of RAM but at least memLimitReserveBytes -// See https://go.dev/doc/gc-guide#Memory_limit for futher details +// See https://go.dev/doc/gc-guide#Memory_limit for futher details. func calcMemLimit(ram uint64) uint64 { if int(float64(ram)*memLimitReservePerc) < memLimitReserveBytes { // Reserve nothing for extremely low RAM values diff --git a/pkg/schemaregistry/confluent/load_balancer_test.go b/pkg/schemaregistry/confluent/load_balancer_test.go index f5ca1fca..99bc8e63 100644 --- a/pkg/schemaregistry/confluent/load_balancer_test.go +++ b/pkg/schemaregistry/confluent/load_balancer_test.go @@ -42,7 +42,7 @@ func TestRoundRobinLoadBalancer(t *testing.T) { urls = append(urls, parsedURL) } - //stop one server + // stop one server servers[0].Close() balancedClient := newBalancedClient(newRoundRobinLoadBalancer(urls), http.DefaultClient) @@ -51,7 +51,10 @@ func TestRoundRobinLoadBalancer(t *testing.T) { body := fmt.Sprintf("iteration: %d, subiteration %d", i, j) req, err := http.NewRequest(http.MethodPost, "http://it-will-be-replaced-anyway.com", strings.NewReader(body)) require.NoError(t, err) - _, err = balancedClient.Do(req) + resp, err := balancedClient.Do(req) + if err == nil { + defer resp.Body.Close() + } if j == 0 { require.Error(t, err) } else { diff --git a/pkg/schemaregistry/format/common.go b/pkg/schemaregistry/format/common.go index 0449397d..fbb7760b 100644 --- a/pkg/schemaregistry/format/common.go +++ b/pkg/schemaregistry/format/common.go @@ -6,7 +6,7 @@ import ( "github.com/doublecloud/transfer/library/go/core/xerrors" ) -// KafkaJSONSchemaFromArr convert KafkaJSONSchema as []byte to struct KafkaJSONSchema +// KafkaJSONSchemaFromArr convert KafkaJSONSchema as []byte to struct KafkaJSONSchema. func KafkaJSONSchemaFromArr(rawJSONSchema []byte) (*KafkaJSONSchema, error) { var kafkaSchema KafkaJSONSchema if err := json.Unmarshal(rawJSONSchema, &kafkaSchema); err != nil { diff --git a/pkg/serializer/parquet.go b/pkg/serializer/parquet.go index 1c89956f..910092bb 100644 --- a/pkg/serializer/parquet.go +++ b/pkg/serializer/parquet.go @@ -15,7 +15,7 @@ type parquetStreamSerializer struct { tableSchema abstract.FastTableSchema } -// works via stream serializer +// works via stream serializer. type parquetBatchSerializer struct { schema *parquet.Schema tableSchema abstract.FastTableSchema diff --git a/pkg/serializer/parquet_format.go b/pkg/serializer/parquet_format.go index 2b434a2a..5d5007cf 100644 --- a/pkg/serializer/parquet_format.go +++ b/pkg/serializer/parquet_format.go @@ -38,7 +38,7 @@ var primitiveTypesMap = map[schema.Type]parquet.Node{ } // Parses map[string] -> parquet.Row. Doesn't support repeated -// fields or composite values. Json fields are saved as string +// fields or composite values. Json fields are saved as string. func toParquetValue(column parquet.Field, col abstract.ColSchema, value any, idx int) (*parquet.Value, error) { defLevel := 0 var leafValue parquet.Value diff --git a/pkg/serializer/queue/debezium_serializer.go b/pkg/serializer/queue/debezium_serializer.go index acf2587a..a2557c7f 100644 --- a/pkg/serializer/queue/debezium_serializer.go +++ b/pkg/serializer/queue/debezium_serializer.go @@ -84,7 +84,7 @@ func (s *DebeziumSerializer) SerializeImpl(input []abstract.ChangeItem, threadsN } // Serialize - serializes []abstract.ChangeItem into map: topic->[]SerializedMessage via debezium emitter -// It's optimized version - with multithreading and caches optimizations +// It's optimized version - with multithreading and caches optimizations. func (s *DebeziumSerializer) Serialize(input []abstract.ChangeItem) (map[abstract.TablePartID][]SerializedMessage, error) { // runtime.NumCPU()*4 showed the best performance in benchmark (pkg/debezium/bench) // chunkSize==16 showed one of the best performance (with dropKeys:false) diff --git a/pkg/serializer/queue/mirror_serializer.go b/pkg/serializer/queue/mirror_serializer.go index 065bafcc..5f12ee7d 100644 --- a/pkg/serializer/queue/mirror_serializer.go +++ b/pkg/serializer/queue/mirror_serializer.go @@ -25,7 +25,7 @@ func (s *MirrorSerializer) serialize(changeItem *abstract.ChangeItem) ([]Seriali } // Serialize -// naive implementation - can be boosted by multi-threading +// naive implementation - can be boosted by multi-threading. func (s *MirrorSerializer) Serialize(input []abstract.ChangeItem) (map[abstract.TablePartID][]SerializedMessage, error) { if len(input) == 0 { return nil, nil @@ -65,7 +65,7 @@ func (s *MirrorSerializer) SerializeLB(changeItem *abstract.ChangeItem) ([]Seria // GroupAndSerializeLB // For logbroker-destination logic should be absolute another! // ChangeItems should be grouped by Key (it's ProducerID) -// And for every Key should be extracted extras (extras - unique for every producer) +// And for every Key should be extracted extras (extras - unique for every producer). func (s *MirrorSerializer) GroupAndSerializeLB(input []abstract.ChangeItem) (map[abstract.TablePartID][]SerializedMessage, map[abstract.TablePartID]map[string]string, error) { idToGroup := make(map[abstract.TablePartID][]SerializedMessage) extras := make(map[abstract.TablePartID]map[string]string) diff --git a/pkg/serializer/queue/native_serializer.go b/pkg/serializer/queue/native_serializer.go index ed33c9cf..b01adb60 100644 --- a/pkg/serializer/queue/native_serializer.go +++ b/pkg/serializer/queue/native_serializer.go @@ -5,7 +5,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract/model" ) -// NativeSerializer - for legacy compatibility: transfers named: realty-rent-prod/realty-rent-test +// NativeSerializer - for legacy compatibility: transfers named: realty-rent-prod/realty-rent-test. type NativeSerializer struct { batchingSettings model.Batching saveTxOrder bool @@ -26,7 +26,7 @@ func (s *NativeSerializer) serializeOneTableID(input []abstract.ChangeItem) []Se } // Serialize - serializes []abstract.ChangeItem into map: topic->[]SerializedMessage via json marshalling -// naive implementation - can be boosted by multi-threading +// naive implementation - can be boosted by multi-threading. func (s *NativeSerializer) Serialize(input []abstract.ChangeItem) (map[abstract.TablePartID][]SerializedMessage, error) { if len(input) == 0 { return nil, nil diff --git a/pkg/serverutil/endpoint.go b/pkg/serverutil/endpoint.go index fd373a19..32f6cf63 100644 --- a/pkg/serverutil/endpoint.go +++ b/pkg/serverutil/endpoint.go @@ -13,7 +13,7 @@ import ( ) func PingFunc(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "Application/json") res, _ := json.Marshal(map[string]interface{}{"ping": "pong", "ts": time.Now()}) if _, err := w.Write(res); err != nil { diff --git a/pkg/sink/sink.go b/pkg/sink/sink.go index 8b6dc683..f5d36838 100644 --- a/pkg/sink/sink.go +++ b/pkg/sink/sink.go @@ -22,13 +22,13 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// NoAsyncSinkErr error which indicate that we should try to create sinker from SyncSink +// NoAsyncSinkErr error which indicate that we should try to create sinker from SyncSink. var NoAsyncSinkErr = xerrors.NewSentinel("no applicable AsyncSink for this transfer") // MakeAsyncSink creates a ready-to-use complete sink pipeline, topped with an asynchronous sink wrapper. // The pipeline may include multiple middlewares and transformations. Their concrete set depends on transfer settings, its source and destination. func MakeAsyncSink(transfer *model.Transfer, lgr log.Logger, mtrcs metrics.Registry, cp coordinator.Coordinator, config middlewares.Config, opts ...abstract.SinkOption) (abstract.AsyncSink, error) { - var pipelineAsync abstract.AsyncSink = nil + var pipelineAsync abstract.AsyncSink middleware, err := syncMiddleware(transfer, lgr, mtrcs, cp, opts...) if err != nil { return nil, xerrors.Errorf("error building sync middleware pipeline: %w", err) @@ -87,7 +87,7 @@ func syncMiddleware(transfer *model.Transfer, lgr log.Logger, mtrcs metrics.Regi }, nil } -// ConstructBaseSink creates a sink of proper type +// ConstructBaseSink creates a sink of proper type. func ConstructBaseSink(transfer *model.Transfer, lgr log.Logger, mtrcs metrics.Registry, cp coordinator.Coordinator, config middlewares.Config) (abstract.Sinker, error) { switch dst := transfer.Dst.(type) { case *model.MockDestination: diff --git a/pkg/source/eventsource/source.go b/pkg/source/eventsource/source.go index 7efdfadc..6f2b3638 100644 --- a/pkg/source/eventsource/source.go +++ b/pkg/source/eventsource/source.go @@ -19,7 +19,7 @@ type eventSourceSource struct { logger log.Logger } -// NewSource constructs a wrapper over the given base.EventSource with the abstract.Source interface +// NewSource constructs a wrapper over the given base.EventSource with the abstract.Source interface. func NewSource(logger log.Logger, source base.EventSource, cleanupPolicy model.CleanupType, tmpPolicy *model.TmpPolicyConfig) abstract.Source { return &eventSourceSource{ source: source, diff --git a/pkg/stats/fallbacks.go b/pkg/stats/fallbacks.go index 186a4b3e..ba4d796e 100644 --- a/pkg/stats/fallbacks.go +++ b/pkg/stats/fallbacks.go @@ -13,7 +13,7 @@ type FallbackStats struct { Errors metrics.Counter } -// FallbackStatsCombination is an object unifying stats for source and target fallbacks +// FallbackStatsCombination is an object unifying stats for source and target fallbacks. type FallbackStatsCombination struct { registry metrics.Registry diff --git a/pkg/stats/middleware_bufferer.go b/pkg/stats/middleware_bufferer.go index fa14df55..e965d0d0 100644 --- a/pkg/stats/middleware_bufferer.go +++ b/pkg/stats/middleware_bufferer.go @@ -28,7 +28,7 @@ type MiddlewareBuffererStats struct { CountToFlush metrics.Histogram } -// ShortEvenDurationBuckets returns buckets adapted for short durations and distributed approximately evenly +// ShortEvenDurationBuckets returns buckets adapted for short durations and distributed approximately evenly. func ShortEvenDurationBuckets() metrics.DurationBuckets { return metrics.NewDurationBuckets( 500*time.Millisecond, @@ -73,7 +73,7 @@ func ShortEvenDurationBuckets() metrics.DurationBuckets { ) } -// Exponential10Buckets returns a set of buckets with borders at 10^[1..10] +// Exponential10Buckets returns a set of buckets with borders at 10^[1..10]. func Exponential10Buckets() metrics.Buckets { return metrics.MakeExponentialBuckets(10, 10, 10) } diff --git a/pkg/stats/middleware_transformer.go b/pkg/stats/middleware_transformer.go index 46050912..7134212d 100644 --- a/pkg/stats/middleware_transformer.go +++ b/pkg/stats/middleware_transformer.go @@ -25,7 +25,7 @@ func NewMiddlewareTransformerStats(r metrics.Registry) *MiddlewareTransformerSta } } -// MillisecondDurationBuckets returns buckets adapted for durations between 1 millisecond and 1 second +// MillisecondDurationBuckets returns buckets adapted for durations between 1 millisecond and 1 second. func MillisecondDurationBuckets() metrics.DurationBuckets { return metrics.NewDurationBuckets( 500*time.Microsecond, diff --git a/pkg/stats/pool.go b/pkg/stats/pool.go index 9ea17d74..64295e73 100644 --- a/pkg/stats/pool.go +++ b/pkg/stats/pool.go @@ -26,7 +26,7 @@ func (s PoolStats) SetStatusCount(status string, count int) { _, ok := s.StatusCount[status] if !ok { s.StatusCount[status] = s.pm.WithTags(map[string]string{ - "status": string(status), + "status": status, }).Gauge("transfers.status") } s.StatusCount[status].Set(float64(count)) diff --git a/pkg/stringutil/stringutil.go b/pkg/stringutil/stringutil.go index 1e8828b7..ee664058 100644 --- a/pkg/stringutil/stringutil.go +++ b/pkg/stringutil/stringutil.go @@ -4,12 +4,12 @@ import ( "strings" ) -// HasPrefixCI checks if string has prefix in case-insensitive mode +// HasPrefixCI checks if string has prefix in case-insensitive mode. func HasPrefixCI(s, prefix string) bool { return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix) } -// TrimPrefixCI trims first found prefix in case-insensitive mode +// TrimPrefixCI trims first found prefix in case-insensitive mode. func TrimPrefixCI(s string, prefixes ...string) (string, bool) { for _, prefix := range prefixes { if HasPrefixCI(s, prefix) { @@ -23,7 +23,7 @@ type UTF8String interface { string | []byte } -// TruncateUTF8 truncates utf-8 string +// TruncateUTF8 truncates utf-8 string. func TruncateUTF8[T UTF8String](s T, limit int) T { if len(s) <= limit { return s diff --git a/pkg/transformer/abstract.go b/pkg/transformer/abstract.go index 84b5d4d2..b19bec9e 100644 --- a/pkg/transformer/abstract.go +++ b/pkg/transformer/abstract.go @@ -16,7 +16,7 @@ type WithID interface { // Transformer serializeable one-of wrapper, for backward compatibility // will hold both type discriminator and config value itself, config is just an any object -// on later stages it materializes as real config object, specific to transformer +// on later stages it materializes as real config object, specific to transformer. type Transformer map[abstract.TransformerType]interface{} func (t Transformer) Type() abstract.TransformerType { diff --git a/pkg/transformer/registry/clickhouse/clickhouse_local.go b/pkg/transformer/registry/clickhouse/clickhouse_local.go index b75351cc..3c10156b 100644 --- a/pkg/transformer/registry/clickhouse/clickhouse_local.go +++ b/pkg/transformer/registry/clickhouse/clickhouse_local.go @@ -350,7 +350,7 @@ func (s *ClickhouseTransformer) ResultSchema(schema *abstract.TableSchema) (*abs inputStructure := strings.Join(inputCols, ",") cmd := exec.Command(s.clickhousePath, "local", "--input-format", "JSONEachRow", "--output-format", "JSONCompact", "--structure", inputStructure, "--query", s.query, "--no-system-tables") buffer := bytes.Buffer{} - buffer.Write([]byte("")) + buffer.WriteString("") cmd.Stdin = &buffer output, err := cmd.CombinedOutput() s.logger.Infof("exec: \n%s", strings.Join(cmd.Args, " ")) diff --git a/pkg/transformer/registry/filter/filter.go b/pkg/transformer/registry/filter/filter.go index 433a80ac..a7152769 100644 --- a/pkg/transformer/registry/filter/filter.go +++ b/pkg/transformer/registry/filter/filter.go @@ -47,21 +47,21 @@ func (f *Filter) Empty() bool { return len(f.IncludeRegexp) == 0 && len(f.ExcludeRegexp) == 0 } -func NewFilter(IncludeRegexp, ExcludeRegexp []string) (Filter, error) { +func NewFilter(includeRegexp, excludeRegexp []string) (Filter, error) { res := &Filter{ - IncludeRegexp: IncludeRegexp, + IncludeRegexp: includeRegexp, compiledInclude: nil, - ExcludeRegexp: ExcludeRegexp, + ExcludeRegexp: excludeRegexp, compiledExclude: nil, } - for _, reg := range IncludeRegexp { + for _, reg := range includeRegexp { creg, err := regexp.Compile(reg) if err != nil { return *res, xerrors.Errorf("unable to compile include regexp: %s: %w", reg, err) } res.compiledInclude = append(res.compiledInclude, creg) } - for _, reg := range ExcludeRegexp { + for _, reg := range excludeRegexp { creg, err := regexp.Compile(reg) if err != nil { return *res, xerrors.Errorf("unable to compile exclude regexp: %s: %w", reg, err) diff --git a/pkg/transformer/registry/filter_rows/util.go b/pkg/transformer/registry/filter_rows/util.go index 3c96ebc1..348599db 100644 --- a/pkg/transformer/registry/filter_rows/util.go +++ b/pkg/transformer/registry/filter_rows/util.go @@ -10,9 +10,7 @@ import ( "github.com/doublecloud/transfer/pkg/util/set" ) -var ( - errIntOverflow = xerrors.Errorf("Provided value overflows int64") -) +var errIntOverflow = xerrors.Errorf("Provided value overflows int64") func stringToTime(str string) (time.Time, bool) { layouts := []string{ @@ -55,7 +53,7 @@ func toInt64E(i interface{}) (int64, error) { case int: return int64(v), nil case int64: - return int64(v), nil + return v, nil case int32: return int64(v), nil case int16: @@ -80,24 +78,6 @@ func toInt64E(i interface{}) (int64, error) { } } -func trimZeroDecimal(s string) string { - // It is copy-paste of https://github.com/spf13/cast/blob/master/caste.go trimZeroDecimal function. - var foundZero bool - for i := len(s); i > 0; i-- { - switch s[i-1] { - case '.': - if foundZero { - return s[:i-1] - } - case '0': - foundZero = true - default: - return s - } - } - return s -} - func valuesListToSet(valList filter.Value) (*set.Set[interface{}], error) { values := make([]interface{}, 0) if valList.IsIntList() { diff --git a/pkg/transformer/registry/lambda/lambda_test.go b/pkg/transformer/registry/lambda/lambda_test.go index ce5b8625..012fe19f 100644 --- a/pkg/transformer/registry/lambda/lambda_test.go +++ b/pkg/transformer/registry/lambda/lambda_test.go @@ -34,7 +34,7 @@ func TestLambdaTransformer(t *testing.T) { } js, err := json.Marshal(data) require.NoError(t, err) - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") _, err = w.Write(js) require.NoError(t, err) diff --git a/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper.go b/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper.go index 8c027afc..3e1746b7 100644 --- a/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper.go +++ b/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper.go @@ -77,7 +77,7 @@ func (r *CdcHistoryGroupTransformer) Apply(input []abstract.ChangeItem) abstract cols, values := r.collectParsedData(changeItem) changeItem.ColumnNames = cols changeItem.ColumnValues = values - //only insert new lines for cdc history + // only insert new lines for cdc history changeItem.Kind = abstract.InsertKind } @@ -99,10 +99,6 @@ func (r *CdcHistoryGroupTransformer) Apply(input []abstract.ChangeItem) abstract } } -func (r *CdcHistoryGroupTransformer) containsAllKeys(colNames []string) bool { - return allFieldsPresent(colNames, rawCdcDocFields, r.Keys) -} - func (r *CdcHistoryGroupTransformer) containsAllFields(colNames []string) bool { return allFieldsPresent(colNames, rawCdcDocFields, append(r.Keys, r.Fields...)) } @@ -140,7 +136,6 @@ func (r *CdcHistoryGroupTransformer) Description() string { } func (r *CdcHistoryGroupTransformer) collectParsedData(changeItem abstract.ChangeItem) ([]string, []interface{}) { - newCols := make([]string, 0, len(r.Keys)+len(r.Fields)) newValues := make([]interface{}, 0, len(r.Keys)+len(r.Fields)) docData := make(map[string]interface{}, len(changeItem.ColumnNames)) @@ -157,7 +152,7 @@ func (r *CdcHistoryGroupTransformer) collectParsedData(changeItem abstract.Chang columnValues = changeItem.ColumnValues } - //firstly adding data from original columns and collecting doc + // firstly adding data from original columns and collecting doc for idx, colName := range columnNames { colValue := columnValues[idx] docData[colName] = colValue @@ -166,7 +161,7 @@ func (r *CdcHistoryGroupTransformer) collectParsedData(changeItem abstract.Chang newValues = append(newValues, colValue) } } - //adding system columns + // adding system columns newCols = append(newCols, etlUpdatedField) newValues = append(newValues, time.Unix(0, int64(changeItem.CommitTime))) @@ -180,7 +175,6 @@ func (r *CdcHistoryGroupTransformer) collectParsedData(changeItem abstract.Chang } func NewCdcHistoryGroupTransformer(config RawCDCDocGrouperConfig) (*CdcHistoryGroupTransformer, error) { - keys := config.Keys var fields []string if len(config.Fields) > 0 { @@ -190,7 +184,7 @@ func NewCdcHistoryGroupTransformer(config RawCDCDocGrouperConfig) (*CdcHistoryGr for _, name := range []string{etlUpdatedField, deletedField, rawDataField} { if !slices.Contains(keys, name) && !slices.Contains(fields, name) { if name == etlUpdatedField { - //by default to the beginning + // by default to the beginning keys = append([]string{etlUpdatedField}, keys...) } else { fields = append(fields, name) @@ -224,5 +218,4 @@ func NewCdcHistoryGroupTransformer(config RawCDCDocGrouperConfig) (*CdcHistoryGr targetSchemas: make(map[string]*abstract.TableSchema), schemasLock: sync.RWMutex{}, }, nil - } diff --git a/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper_test.go b/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper_test.go index 3b535321..496c73cd 100644 --- a/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper_test.go +++ b/pkg/transformer/registry/raw_doc_grouper/raw_cdc_doc_grouper_test.go @@ -13,6 +13,7 @@ import ( func TestRawCdcDocGroupTransformer(t *testing.T) { t.Parallel() t.Run("duplicate keys fail", func(t *testing.T) { + t.Parallel() _, err := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Keys: []string{"key1", "key1"}, }) @@ -20,6 +21,7 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("duplicate keys and non-keys fail", func(t *testing.T) { + t.Parallel() _, err := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Keys: []string{"key1", "key3"}, Fields: []string{"key1"}, @@ -28,6 +30,7 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("different keys and special fields ok", func(t *testing.T) { + t.Parallel() _, err := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Keys: []string{"key3", "key2"}, Fields: []string{"etl_updated_at"}, @@ -36,6 +39,7 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("Suitable check by tables", func(t *testing.T) { + t.Parallel() transformer, _ := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Tables: filter.Tables{IncludeTables: []string{"table1"}}, Keys: []string{"col1", "col2", "etl_updated_at"}, @@ -62,6 +66,7 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("Wrong columns fail transfer", func(t *testing.T) { + t.Parallel() transformer, _ := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Tables: filter.Tables{IncludeTables: []string{"table1"}}, Keys: []string{"col1", "col2"}, @@ -82,6 +87,7 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("Values are parsed", func(t *testing.T) { + t.Parallel() transformer, _ := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Keys: []string{"col1", "col2"}, Fields: []string{"col3"}, @@ -114,17 +120,18 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("Different schemas are processed", func(t *testing.T) { + t.Parallel() transformer, _ := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Keys: []string{"col1", "col2"}, Fields: []string{"col3"}, }) testChangeItems := []abstract.ChangeItem{ - dummyItem([]colParams{col1NotKey, col2NotKey, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 1 - dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 1, diff values - dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey2, col5NotKey, restCol}), //schema 1, diff values - dummyItem([]colParams{col1NotKey2, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 2 - diff type of col2 - dummyItem([]colParams{col2Key, col1NotKey, col3Key, restCol}), //schema 3 + dummyItem([]colParams{col1NotKey, col2NotKey, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 1 + dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 1, diff values + dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey2, col5NotKey, restCol}), // schema 1, diff values + dummyItem([]colParams{col1NotKey2, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 2 - diff type of col2 + dummyItem([]colParams{col2Key, col1NotKey, col3Key, restCol}), // schema 3 } differentSchemas := make([]string, 0) @@ -147,6 +154,7 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { }) t.Run("System events schema fixed", func(t *testing.T) { + t.Parallel() transformer, _ := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Keys: []string{"col1", "col2"}, }) @@ -173,8 +181,8 @@ func TestRawCdcDocGroupTransformer(t *testing.T) { } func getExpectedValuesCdc(chI abstract.ChangeItem, chItemNumber int) map[string]interface{} { - var valuesByColName = make(map[string]interface{}) - var docField = make(map[string]interface{}) + valuesByColName := make(map[string]interface{}) + docField := make(map[string]interface{}) var colNames []string var colvalues []interface{} @@ -193,6 +201,6 @@ func getExpectedValuesCdc(chI abstract.ChangeItem, chItemNumber int) map[string] } valuesByColName[etlUpdatedField] = time.Unix(0, int64(updateTime)) valuesByColName[rawDataField] = docField - valuesByColName[deletedField] = chItemNumber == 0 //chItem 0 is DeleteKind + valuesByColName[deletedField] = chItemNumber == 0 // chItem 0 is DeleteKind return valuesByColName } diff --git a/pkg/transformer/registry/raw_doc_grouper/raw_data_utils.go b/pkg/transformer/registry/raw_doc_grouper/raw_data_utils.go index 829a89ec..66a3c1bf 100644 --- a/pkg/transformer/registry/raw_doc_grouper/raw_data_utils.go +++ b/pkg/transformer/registry/raw_doc_grouper/raw_data_utils.go @@ -29,8 +29,8 @@ func MakeStubFieldWithOriginalType(colName string, ytType schema.Type, primaryKe func getOriginalType(original abstract.TableColumns, ytType schema.Type) string { for _, column := range original { - //checking if data is from pg and at least 1 original type is present - //in this case setting pg original for our column + // checking if data is from pg and at least 1 original type is present + // in this case setting pg original for our column if strings.Contains(column.OriginalType, pgPrefix) { return getPgTypeFor(ytType) } @@ -46,7 +46,7 @@ func getPgTypeFor(ytType schema.Type) string { if ytType == schema.TypeDatetime || ytType == schema.TypeTimestamp { return pgTimestamp } - //may be throw exception here? then need to change protocol + // may be throw exception here? then need to change protocol return pgText } diff --git a/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper.go b/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper.go index 4cf73aa2..2c811c50 100644 --- a/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper.go +++ b/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper.go @@ -99,10 +99,6 @@ func (r *RawDocGroupTransformer) Apply(input []abstract.ChangeItem) abstract.Tra } } -func (r *RawDocGroupTransformer) containsAllKeys(colNames []string) bool { - return allFieldsPresent(colNames, rawDocFields, r.Keys) -} - func (r *RawDocGroupTransformer) containsAllFields(colNames []string) bool { return allFieldsPresent(colNames, rawDocFields, append(r.Keys, r.Fields...)) } @@ -214,5 +210,4 @@ func NewRawDocGroupTransformer(config RawDocGrouperConfig) (*RawDocGroupTransfor targetSchemas: make(map[string]*abstract.TableSchema), schemasLock: sync.RWMutex{}, }, nil - } diff --git a/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper_test.go b/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper_test.go index d86a5195..08cd2b74 100644 --- a/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper_test.go +++ b/pkg/transformer/registry/raw_doc_grouper/raw_doc_grouper_test.go @@ -14,6 +14,7 @@ func TestRawDocGroupTransformer(t *testing.T) { t.Parallel() t.Run("duplicate keys fail", func(t *testing.T) { + t.Parallel() _, err := NewRawDocGroupTransformer(RawDocGrouperConfig{ Keys: []string{"key1", "key1"}, }) @@ -21,6 +22,7 @@ func TestRawDocGroupTransformer(t *testing.T) { }) t.Run("different keys ok", func(t *testing.T) { + t.Parallel() _, err := NewRawDocGroupTransformer(RawDocGrouperConfig{ Keys: []string{"key3", "key2"}, }) @@ -28,6 +30,7 @@ func TestRawDocGroupTransformer(t *testing.T) { }) t.Run("Suitable check by table and fields", func(t *testing.T) { + t.Parallel() transformer, _ := NewRawDocGroupTransformer(RawDocGrouperConfig{ Tables: filter.Tables{IncludeTables: []string{"table1"}}, Keys: []string{"col1", "col2"}, @@ -54,6 +57,7 @@ func TestRawDocGroupTransformer(t *testing.T) { }) t.Run("Wrong columns fail transfer", func(t *testing.T) { + t.Parallel() transformer, _ := NewCdcHistoryGroupTransformer(RawCDCDocGrouperConfig{ Tables: filter.Tables{IncludeTables: []string{"table1"}}, Keys: []string{"col1", "col2"}, @@ -74,14 +78,15 @@ func TestRawDocGroupTransformer(t *testing.T) { }) t.Run("Values are parsed", func(t *testing.T) { + t.Parallel() transformer, _ := NewRawDocGroupTransformer(RawDocGrouperConfig{ Keys: []string{"col1", "col2"}, Fields: []string{"col3"}, }) testChangeItems := []abstract.ChangeItem{ - dummyItem([]colParams{col1NotKey, col2NotKey, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 1 - dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), //same schema 1, diff values + dummyItem([]colParams{col1NotKey, col2NotKey, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 1 + dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), // same schema 1, diff values dummyItem([]colParams{col2Key, col1NotKey, col3Key, restCol}), // schema 2 } @@ -105,17 +110,18 @@ func TestRawDocGroupTransformer(t *testing.T) { }) t.Run("Different schemas are processed", func(t *testing.T) { + t.Parallel() transformer, _ := NewRawDocGroupTransformer(RawDocGrouperConfig{ Keys: []string{"col1", "col2"}, Fields: []string{"col3"}, }) testChangeItems := []abstract.ChangeItem{ - dummyItem([]colParams{col1NotKey, col2NotKey, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 1 - dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 1, diff values - dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey2, col5NotKey, restCol}), //schema 1, diff values - dummyItem([]colParams{col1NotKey2, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), //schema 2 - diff type of col2 - dummyItem([]colParams{col2Key, col1NotKey, col3Key, restCol}), //schema 3 + dummyItem([]colParams{col1NotKey, col2NotKey, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 1 + dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 1, diff values + dummyItem([]colParams{col1NotKey, col2NotKey2, col3NotKey, col4NotKey2, col5NotKey, restCol}), // schema 1, diff values + dummyItem([]colParams{col1NotKey2, col2NotKey2, col3NotKey, col4NotKey, col5NotKey, restCol}), // schema 2 - diff type of col2 + dummyItem([]colParams{col2Key, col1NotKey, col3Key, restCol}), // schema 3 } differentSchemas := make([]string, 0) @@ -138,6 +144,7 @@ func TestRawDocGroupTransformer(t *testing.T) { }) t.Run("System events schema fixed", func(t *testing.T) { + t.Parallel() transformer, _ := NewRawDocGroupTransformer(RawDocGrouperConfig{ Keys: []string{"col1", "col2"}, }) @@ -161,12 +168,11 @@ func TestRawDocGroupTransformer(t *testing.T) { []string{"col1", "col2", "etl_updated_at", "doc"}, "Changed schema chItems columns be returned!!") } }) - } func getExpectedValues(changeItem abstract.ChangeItem) map[string]interface{} { - var valuesByColName = make(map[string]interface{}) - var docField = make(map[string]interface{}) + valuesByColName := make(map[string]interface{}) + docField := make(map[string]interface{}) for i, name := range changeItem.ColumnNames { if name == restField { diff --git a/pkg/transformer/registry/raw_doc_grouper/raw_doc_test_utils.go b/pkg/transformer/registry/raw_doc_grouper/raw_doc_test_utils.go index e2919907..33ad3c31 100644 --- a/pkg/transformer/registry/raw_doc_grouper/raw_doc_test_utils.go +++ b/pkg/transformer/registry/raw_doc_grouper/raw_doc_test_utils.go @@ -114,7 +114,7 @@ func transformAndCheckFields(t *testing.T, transformResult []abstract.ChangeItem columns := transformResult[0].TableSchema.Columns() require.Equal(t, len(columns), len(keys)+len(fields), "Wrong number of columns in result!") - //checking all pkeys and order + // checking all pkeys and order for i := range keys { require.Equal(t, keys[i], columns[i].ColumnName, "%d column should be key %s", i, keys[i]) require.True(t, columns[i].PrimaryKey, "Key column should be PK!") diff --git a/pkg/transformer/registry/to_string/to_string_test.go b/pkg/transformer/registry/to_string/to_string_test.go index a4c1e8cb..59450f3c 100644 --- a/pkg/transformer/registry/to_string/to_string_test.go +++ b/pkg/transformer/registry/to_string/to_string_test.go @@ -103,7 +103,8 @@ func TestToStringTransformer(t *testing.T) { "value1", time.Date(1703, 1, 2, 0, 0, 0, 0, time.UTC), 123.123, - float32(312.321)}, + float32(312.321), + }, TableSchema: table2Schema, } item3 := abstract.ChangeItem{ @@ -128,7 +129,7 @@ func TestToStringTransformer(t *testing.T) { func TestAllTypesToStringTransformer(t *testing.T) { t.Parallel() - var testCases = []struct { + testCases := []struct { originalValue interface{} originalType schema.Type expectedValue string @@ -153,7 +154,7 @@ func TestAllTypesToStringTransformer(t *testing.T) { {time.Date(14124, 1, 12, 0, 0, 0, 0, time.UTC), schema.TypeDate, "14124-01-12"}, {time.Date(2311, 12, 1, 1, 2, 4, 5, time.UTC), schema.TypeDatetime, "2311-12-01T01:02:04.000000005Z"}, {time.Date(1231, 5, 23, 9, 8, 7, 6, time.UTC), schema.TypeTimestamp, "1231-05-23T09:08:07.000000006Z"}, - {time.Duration(12*time.Hour + 53*time.Minute + 21*time.Second + 87*time.Millisecond + 182*time.Microsecond + 124*time.Nanosecond), schema.TypeInterval, "12h53m21.087182124s"}, + {12*time.Hour + 53*time.Minute + 21*time.Second + 87*time.Millisecond + 182*time.Microsecond + 124*time.Nanosecond, schema.TypeInterval, "12h53m21.087182124s"}, // check nulls {nil, schema.TypeDate, ""}, {nil, schema.TypeDatetime, ""}, diff --git a/pkg/util/backoff.go b/pkg/util/backoff.go index 6cafe6c2..df37683a 100644 --- a/pkg/util/backoff.go +++ b/pkg/util/backoff.go @@ -7,7 +7,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// BackoffLogger uses level "warn" by default +// BackoffLogger uses level "warn" by default. func BackoffLogger(logger log.Logger, msg string) func(error, time.Duration) { return BackoffLoggerWarn(logger, msg) } diff --git a/pkg/util/castx/caste.go b/pkg/util/castx/caste.go index b39a838f..f739ef0e 100644 --- a/pkg/util/castx/caste.go +++ b/pkg/util/castx/caste.go @@ -73,7 +73,7 @@ func ToStringE(i interface{}) (string, error) { case uint: return strconv.FormatUint(uint64(s), 10), nil case uint64: - return strconv.FormatUint(uint64(s), 10), nil + return strconv.FormatUint(s, 10), nil case uint32: return strconv.FormatUint(uint64(s), 10), nil case uint16: diff --git a/pkg/util/cli/spinner.go b/pkg/util/cli/spinner.go index c5c539f3..cf68291c 100644 --- a/pkg/util/cli/spinner.go +++ b/pkg/util/cli/spinner.go @@ -7,7 +7,7 @@ import ( "syscall" "time" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) var spin = []string{"--", "\\", "|", "/"} @@ -22,7 +22,7 @@ type spinner struct { delay time.Duration } -// NewSpinner creates pause filler in CLI to make it visually alive +// NewSpinner creates pause filler in CLI to make it visually alive. func NewSpinner(width int, delay time.Duration) *spinner { if width < 0 { width = 3 @@ -36,7 +36,7 @@ func NewSpinner(width int, delay time.Duration) *spinner { var stdout interface{} = syscall.Stdout stdoutFD, ok := stdout.(int) - isTerminal := terminal.IsTerminal(stdoutFD) + isTerminal := term.IsTerminal(stdoutFD) return &spinner{ isTty: ok && isTerminal, diff --git a/pkg/util/diff/diff.go b/pkg/util/diff/diff.go index fe7b5638..04bd02cc 100644 --- a/pkg/util/diff/diff.go +++ b/pkg/util/diff/diff.go @@ -6,20 +6,8 @@ import ( "github.com/doublecloud/transfer/library/go/core/xerrors" "github.com/doublecloud/transfer/library/go/slices" "github.com/doublecloud/transfer/pkg/util" - "golang.org/x/exp/constraints" ) -// TODO can be removed after go1.21 -func min[T constraints.Ordered](x T, y ...T) T { - result := x - for _, y := range y { - if y < result { - result = y - } - } - return result -} - // LineDiff takes two strings and calculates diff in terms of lines that came from a or from b. // It calculates DiffItems, which is array of []DiffItem -- an array of minimal length that // describes what lines came from a, what lines came from b, and what lines are common. @@ -29,17 +17,17 @@ func LineDiff(a, b string) (DiffItems, error) { } // DiffItems is array of diff items, that has useful `AsString` method which can give you a -// readable format. You can use default String method if you want default settings +// readable format. You can use default String method if you want default settings. type DiffItems []DiffItem -// String converts DiffItems to string with AsString method with default string conversion options +// String converts DiffItems to string with AsString method with default string conversion options. func (d DiffItems) String() string { return d.AsString(nil) } // AsString converts array of DiffItem into single string. You can control the // result with parameter `options *DiffItemsAsStringOptions`, or set it to null -// for default configuration +// for default configuration. func (d DiffItems) AsString(options *diffItemsAsStringOptions) string { opts := util.Coalesce(options, makeDiffAsStringOptions()) items := slices.Filter(d, func(item DiffItem) bool { @@ -78,19 +66,19 @@ type diffItemsAsStringOptions struct { JoinWith string } -// DiffOnlySources is used when you want to get only diff lines +// DiffOnlySources is used when you want to get only diff lines. func (l *diffItemsAsStringOptions) DiffOnlySources() *diffItemsAsStringOptions { l.DiffSourceExcludeFilter = []DiffSource{DiffSourceBoth} return l } -// CommonOnlySources is used when you want to get only common lines +// CommonOnlySources is used when you want to get only common lines. func (l *diffItemsAsStringOptions) CommonOnlySources() *diffItemsAsStringOptions { l.DiffSourceExcludeFilter = []DiffSource{DiffSourceLeft, DiffSourceRight} return l } -// AllSources is used when you want to get all lines: diff and common +// AllSources is used when you want to get all lines: diff and common. func (l *diffItemsAsStringOptions) AllSources() *diffItemsAsStringOptions { l.DiffSourceExcludeFilter = nil return l diff --git a/pkg/util/generics.go b/pkg/util/generics.go index 555ca0f9..e22c0209 100644 --- a/pkg/util/generics.go +++ b/pkg/util/generics.go @@ -1,6 +1,6 @@ package util -// NewIfNil returns specified ptr, initializing it with new object if ptr is nil +// NewIfNil returns specified ptr, initializing it with new object if ptr is nil. func NewIfNil[T any](ptr **T) *T { if *ptr == nil { *ptr = new(T) diff --git a/pkg/util/glob/glob.go b/pkg/util/glob/glob.go index 39feaa9c..b36c1c90 100644 --- a/pkg/util/glob/glob.go +++ b/pkg/util/glob/glob.go @@ -2,7 +2,6 @@ package glob import ( "bytes" - "reflect" "strings" "unsafe" ) @@ -45,20 +44,10 @@ func Match(pattern, subj string) bool { } // get underlying bytes of pattern - patternBytes := unsafe.Slice( - (*byte)(unsafe.Pointer( - (*reflect.StringHeader)(unsafe.Pointer(&pattern)).Data), - ), - len(pattern), - ) + patternBytes := unsafe.Slice(unsafe.StringData(pattern), len(pattern)) // get underlying bytes of subject - subjBytes := unsafe.Slice( - (*byte)(unsafe.Pointer( - (*reflect.StringHeader)(unsafe.Pointer(&subj)).Data), - ), - len(subj), - ) + subjBytes := unsafe.Slice(unsafe.StringData(subj), len(subj)) for { // find first glob index diff --git a/pkg/util/hash.go b/pkg/util/hash.go index b8ab7886..cc18f0ea 100644 --- a/pkg/util/hash.go +++ b/pkg/util/hash.go @@ -29,7 +29,7 @@ func HashSha256(data []byte) string { return hex.EncodeToString(hasher.Sum(data)) } -// use hasher with once allocated slice [sha256.Size]byte for result instead of sha256.Sum256() function to avoid extra memory allocations +// use hasher with once allocated slice [sha256.Size]byte for result instead of sha256.Sum256() function to avoid extra memory allocations. type HasherSha256 struct { hasher hash.Hash result [sha256.Size]byte diff --git a/pkg/util/jsonx/json_null.go b/pkg/util/jsonx/json_null.go index 961f99d0..bef52dd9 100644 --- a/pkg/util/jsonx/json_null.go +++ b/pkg/util/jsonx/json_null.go @@ -1,7 +1,7 @@ package jsonx // JSONNull denotes a JSON 'null' value. -// It represents pure 'null' JSON value, not a value inside a JSON object or array. A value inside a JSON object is just golang's 'nil' +// It represents pure 'null' JSON value, not a value inside a JSON object or array. A value inside a JSON object is just golang's 'nil'. type JSONNull struct{} func (n JSONNull) MarshalJSON() ([]byte, error) { diff --git a/pkg/util/jsonx/value_decoder.go b/pkg/util/jsonx/value_decoder.go index 0067b342..5a76bd44 100644 --- a/pkg/util/jsonx/value_decoder.go +++ b/pkg/util/jsonx/value_decoder.go @@ -6,19 +6,19 @@ import ( "github.com/doublecloud/transfer/library/go/core/xerrors" ) -// ValueDecoder is a decoder which deserializes JSONs into the Data Transfer's JSON representation +// ValueDecoder is a decoder which deserializes JSONs into the Data Transfer's JSON representation. type ValueDecoder struct { base *json.Decoder } -// NewValueDecoder constructs a decoder which deserializes JSONs into the Data Transfer's JSON representation +// NewValueDecoder constructs a decoder which deserializes JSONs into the Data Transfer's JSON representation. func NewValueDecoder(base *json.Decoder) *ValueDecoder { return &ValueDecoder{ base: base, } } -// Decode uses the decoder from the constructor to deserialize the input into a Data Transfer JSON +// Decode uses the decoder from the constructor to deserialize the input into a Data Transfer JSON. func (d *ValueDecoder) Decode() (any, error) { var result any if err := d.base.Decode(&result); err != nil { diff --git a/pkg/util/make_chan_with_error.go b/pkg/util/make_chan_with_error.go index d7c5cc10..daf9e5b9 100644 --- a/pkg/util/make_chan_with_error.go +++ b/pkg/util/make_chan_with_error.go @@ -1,6 +1,6 @@ package util -// MakeChanWithError constructs a buffered channel with capacity 1, writes the given error into it and returns the channel +// MakeChanWithError constructs a buffered channel with capacity 1, writes the given error into it and returns the channel. func MakeChanWithError(err error) chan error { result := make(chan error, 1) result <- err diff --git a/pkg/util/map_keys_in_order.go b/pkg/util/map_keys_in_order.go index 63ed9274..cfa5264b 100644 --- a/pkg/util/map_keys_in_order.go +++ b/pkg/util/map_keys_in_order.go @@ -6,7 +6,7 @@ import ( "golang.org/x/exp/constraints" ) -// MapKeysInOrder returns a sorted slice of all the keys in the given map +// MapKeysInOrder returns a sorted slice of all the keys in the given map. func MapKeysInOrder[K constraints.Ordered, V any](m map[K]V) []K { result := make([]K, 0, len(m)) for k := range m { diff --git a/pkg/util/multibuf/pooledmultibuf.go b/pkg/util/multibuf/pooledmultibuf.go index 7b437a9c..1a428c7d 100644 --- a/pkg/util/multibuf/pooledmultibuf.go +++ b/pkg/util/multibuf/pooledmultibuf.go @@ -7,7 +7,7 @@ import ( ) // PooledMultiBuffer provides io.Reader interface over multiple bytes.Buffers (like io.MultiReader) -// buffers are taken from sync.Pool to reduce GC pressure +// buffers are taken from sync.Pool to reduce GC pressure. type PooledMultiBuffer struct { buffers []io.Reader bufMu sync.Mutex @@ -21,7 +21,7 @@ func (q *PooledMultiBuffer) initReaderOnce() { } // Read implements io.Reader interface -// It is guaranteed that internal buffers read order is the same as the order of AcquireBuffer calls +// It is guaranteed that internal buffers read order is the same as the order of AcquireBuffer calls. func (q *PooledMultiBuffer) Read(p []byte) (int, error) { q.initReaderOnce() return q.rd.Read(p) @@ -29,7 +29,7 @@ func (q *PooledMultiBuffer) Read(p []byte) (int, error) { // AcquireBuffer provides a buffer. It may allocate new buffer in memory or take it from internal memory pool. // To optimize memory allocations you can set initial buffer capacity if it is known. -// It is guaranteed that internal buffers read order is the same as the order of AcquireBuffer calls +// It is guaranteed that internal buffers read order is the same as the order of AcquireBuffer calls. func (q *PooledMultiBuffer) AcquireBuffer(initialCap int) *bytes.Buffer { var buf *bytes.Buffer if b := q.pool.Get(); b != nil { @@ -45,7 +45,7 @@ func (q *PooledMultiBuffer) AcquireBuffer(initialCap int) *bytes.Buffer { } // Close release all acquired buffers and return them into memory pool -// Read should not be called after close +// Read should not be called after close. func (q *PooledMultiBuffer) Close() { for _, buf := range q.buffers { q.pool.Put(buf) diff --git a/pkg/util/oneof/oneof_value.go b/pkg/util/oneof/oneof_value.go index 1f50abef..6999547f 100644 --- a/pkg/util/oneof/oneof_value.go +++ b/pkg/util/oneof/oneof_value.go @@ -7,7 +7,7 @@ import ( var FieldNotFoundError = xerrors.New("Can't find field") -// Value get single one-of value. Extract actual one-of message instead of `IsOneOfFoo` interface wrapper +// Value get single one-of value. Extract actual one-of message instead of `IsOneOfFoo` interface wrapper. func Value(root protoreflect.Message) (protoreflect.ProtoMessage, error) { fields := root.Descriptor().Fields() for i := 0; i < fields.Len(); i++ { @@ -22,7 +22,7 @@ func Value(root protoreflect.Message) (protoreflect.ProtoMessage, error) { return nil, FieldNotFoundError } -// SetValue will assign one-of value to a container with corresponding one-of +// SetValue will assign one-of value to a container with corresponding one-of. func SetValue(container protoreflect.Message, value protoreflect.Message) error { fields := container.Descriptor().Fields() for i := 0; i < fields.Len(); i++ { diff --git a/pkg/util/pool/impl.go b/pkg/util/pool/impl.go index 6cfc72ac..98ce3c3c 100644 --- a/pkg/util/pool/impl.go +++ b/pkg/util/pool/impl.go @@ -49,7 +49,7 @@ func (t *defaultPool) worker() { } // NewDefaultPool creates simple task pool. -// taskFn is called to process each task. Capacity sets maximum number of tasks to be processed simultaneously +// taskFn is called to process each task. Capacity sets maximum number of tasks to be processed simultaneously. func NewDefaultPool(taskFn TaskFn, capacity uint64) Pool { return &defaultPool{ taskCh: make(chan interface{}), diff --git a/pkg/util/pool/pool.go b/pkg/util/pool/pool.go index 10bfd16f..b5c64277 100644 --- a/pkg/util/pool/pool.go +++ b/pkg/util/pool/pool.go @@ -7,11 +7,11 @@ import ( var ErrPoolClosed = xerrors.New("pool is closed") // TaskFn is used to process items (tasks) -// which are put into pool +// which are put into pool. type TaskFn func(interface{}) // Pool should be used to process set of homogeneous items (called tasks) -// in parallel +// in parallel. type Pool interface { // Add puts task into pool and waits until task processing is started. // Error is returned if task cannot be put into pool (ex. pool is closed), not if the processing itself has failed diff --git a/pkg/util/ports.go b/pkg/util/ports.go index 6de010dc..2c96dafe 100644 --- a/pkg/util/ports.go +++ b/pkg/util/ports.go @@ -7,7 +7,7 @@ import ( ) // GetFreePort returns free port in operating system that -// you can use for deploying your service +// you can use for deploying your service. func GetFreePort() (port int, resultError error) { lsnr, err := net.Listen("tcp", ":0") if err != nil { diff --git a/pkg/util/queues/sequencer/sequencer.go b/pkg/util/queues/sequencer/sequencer.go index 32fdf455..50bf0449 100644 --- a/pkg/util/queues/sequencer/sequencer.go +++ b/pkg/util/queues/sequencer/sequencer.go @@ -122,7 +122,7 @@ func (s *Sequencer) StartProcessing(messages []QueueMessage) error { if _, ok := s.processing[partitionName]; !ok { s.processing[partitionName] = &partitionToOffsets{ topic: message.Topic, - partition: int(message.Partition), + partition: message.Partition, offsets: make([]int64, 0), maxAppendedOffset: 0, } diff --git a/pkg/util/queues/sequencer/util_kafka.go b/pkg/util/queues/sequencer/util_kafka.go index abe0fc7b..51b0e3c2 100644 --- a/pkg/util/queues/sequencer/util_kafka.go +++ b/pkg/util/queues/sequencer/util_kafka.go @@ -4,14 +4,14 @@ import ( "fmt" ) -// BuildMapPartitionToOffsetsRange - is used only in logging +// BuildMapPartitionToOffsetsRange - is used only in logging. func BuildMapPartitionToOffsetsRange(messages []QueueMessage) string { sequencer := NewSequencer() _ = sequencer.StartProcessing(messages) return sequencer.ToStringRanges() } -// BuildPartitionOffsetLogLine - is used only in logging +// BuildPartitionOffsetLogLine - is used only in logging. func BuildPartitionOffsetLogLine(messages []QueueMessage) string { if len(messages) == 0 { return "" @@ -23,7 +23,7 @@ func BuildPartitionOffsetLogLine(messages []QueueMessage) string { return result[0 : len(result)-1] } -// BuildMapTopicPartitionToOffsetsRange - is used only in logging +// BuildMapTopicPartitionToOffsetsRange - is used only in logging. func BuildMapTopicPartitionToOffsetsRange(messages []QueueMessage) string { sequencer := NewSequencer() _ = sequencer.StartProcessing(messages) diff --git a/pkg/util/runtime.go b/pkg/util/runtime.go index 09e31c24..a51b62ec 100644 --- a/pkg/util/runtime.go +++ b/pkg/util/runtime.go @@ -19,7 +19,7 @@ func GetMiniCallstack(depth int) []string { for offset := 1; offset <= depth; offset++ { _, file, no, ok := runtime.Caller(offset) if ok { - caller := "" + var caller string callerPath := strings.Split(fmt.Sprintf("%s:%d", file, no), "/") if len(callerPath) > 1 { caller = strings.Join(callerPath[len(callerPath)-2:], "/") diff --git a/pkg/util/smart_timer.go b/pkg/util/smart_timer.go index 4721ddd8..a7f4078b 100644 --- a/pkg/util/smart_timer.go +++ b/pkg/util/smart_timer.go @@ -94,7 +94,7 @@ func (t *SmartTimer) Pause() { } // deactivateLastRoutine deactivates the last routine run by smart timer and cleans up the alarm channel. -// This method must be run with mtx already taken +// This method must be run with mtx already taken. func (t *SmartTimer) deactivateLastRoutine() { if t.lastRoutineCanceler == nil { return diff --git a/pkg/util/snaker.go b/pkg/util/snaker.go index b3748f64..e9ac5684 100644 --- a/pkg/util/snaker.go +++ b/pkg/util/snaker.go @@ -5,8 +5,10 @@ import ( "strings" ) -var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") -var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") +var ( + matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") + matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") +) func Snakify(str string) string { snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}") @@ -30,14 +32,15 @@ func camelCase(s string, upper bool) string { stringIter(s, func(prev, curr, next rune) { if !isDelimiter(curr) { - if isDelimiter(prev) || (upper && prev == 0) { + switch { + case isDelimiter(prev) || (upper && prev == 0): buffer = append(buffer, toUpper(curr)) - } else if isLower(prev) { + case isLower(prev): buffer = append(buffer, curr) - } else if isUpper(prev) && isUpper(curr) && isLower(next) { + case isUpper(prev) && isUpper(curr) && isLower(next): // Assume a case like "R" for "XRequestId" buffer = append(buffer, curr) - } else { + default: buffer = append(buffer, toLower(curr)) } } diff --git a/pkg/util/throttler/throttler.go b/pkg/util/throttler/throttler.go index 6267d338..a0de08e1 100644 --- a/pkg/util/throttler/throttler.go +++ b/pkg/util/throttler/throttler.go @@ -23,7 +23,7 @@ func (t *MemoryThrottler) AddInflight(size uint64) { func (t *MemoryThrottler) ReduceInflight(size uint64) { t.inflightMutex.Lock() defer t.inflightMutex.Unlock() - t.inflightBytes = t.inflightBytes - size + t.inflightBytes -= size } func NewMemoryThrottler(bufferSize uint64) *MemoryThrottler { diff --git a/pkg/util/time.go b/pkg/util/time.go index c8b21d39..b38b2112 100644 --- a/pkg/util/time.go +++ b/pkg/util/time.go @@ -2,12 +2,12 @@ package util import "time" -// AnnoDominiBeginning returns the first time instant of the AD (modern) epoch in the given location +// AnnoDominiBeginning returns the first time instant of the AD (modern) epoch in the given location. func AnnoDominiBeginning(loc *time.Location) time.Time { return time.Date(1, 1, 1, 0, 0, 0, 0, loc) } -// BeforeChristEnding returns the first time instant AFTER the end of the BC (pre-modern) epoch in the given location +// BeforeChristEnding returns the first time instant AFTER the end of the BC (pre-modern) epoch in the given location. func BeforeChristEnding(loc *time.Location) time.Time { return time.Date(0, 1, 1, 0, 0, 0, 0, loc) } diff --git a/pkg/util/validators/validators.go b/pkg/util/validators/validators.go index c3bbdfc7..6ce61726 100644 --- a/pkg/util/validators/validators.go +++ b/pkg/util/validators/validators.go @@ -27,7 +27,7 @@ func HostPort(host string, port string) error { } func Host(host string) error { - matchedHostname, _ := regexp.Match(hostnameRegex, []byte(host)) + matchedHostname, _ := regexp.MatchString(hostnameRegex, host) if !matchedHostname { // host is ip if ip := net.ParseIP(host); ip == nil { diff --git a/pkg/util/xd_array.go b/pkg/util/xd_array.go index 364dd9f0..14b31513 100644 --- a/pkg/util/xd_array.go +++ b/pkg/util/xd_array.go @@ -2,7 +2,7 @@ package util import "github.com/doublecloud/transfer/library/go/core/xerrors" -// XDArray Multidimensional array with arbitrary item type +// XDArray Multidimensional array with arbitrary item type. type XDArray struct { Data []interface{} } @@ -80,7 +80,7 @@ func XDIndex(i int, fullSize []int) []int { index := make([]int, len(fullSize)) for d, l := range fullSize { index[d] = i / l - i = i % l + i %= l } return index } diff --git a/pkg/worker/tasks/activate_delivery.go b/pkg/worker/tasks/activate_delivery.go index bc7398f5..3a21b0a2 100644 --- a/pkg/worker/tasks/activate_delivery.go +++ b/pkg/worker/tasks/activate_delivery.go @@ -174,7 +174,7 @@ func ActivateDelivery(ctx context.Context, task *model.TransferOperation, cp coo return nil } -// ObtainAllSrcTables uses a temporary Storage for transfer source to obtain a list of tables +// ObtainAllSrcTables uses a temporary Storage for transfer source to obtain a list of tables. func ObtainAllSrcTables(transfer *model.Transfer, registry metrics.Registry) (abstract.TableMap, error) { srcStorage, err := storage.NewStorage(transfer, coordinator.NewFakeClient(), registry) if err != nil { diff --git a/pkg/worker/tasks/add_tables.go b/pkg/worker/tasks/add_tables.go index 51d82c07..807cab30 100644 --- a/pkg/worker/tasks/add_tables.go +++ b/pkg/worker/tasks/add_tables.go @@ -2,7 +2,6 @@ package tasks import ( "context" - "encoding/json" "sort" "github.com/doublecloud/transfer/internal/logger" @@ -143,13 +142,3 @@ func setSourceTables(source model.Source, tableSet map[string]bool) { src.DBTables = result } } - -func prepareSourceParamsToStore(source model.Source) string { - var params string - switch src := source.(type) { - case *postgres.PgSource: - j, _ := json.Marshal(src) - params = string(j) - } - return params -} diff --git a/pkg/worker/tasks/checksum.go b/pkg/worker/tasks/checksum.go index 075dcc8e..76d117bb 100644 --- a/pkg/worker/tasks/checksum.go +++ b/pkg/worker/tasks/checksum.go @@ -1014,7 +1014,7 @@ func compareSlices(lVal []interface{}, lSchema abstract.ColSchema, rVal []interf return equal, nil } -// colSchemaForSliceElement transforms the given schema in such a way that it is suitable for comparison of an individual slice element +// colSchemaForSliceElement transforms the given schema in such a way that it is suitable for comparison of an individual slice element. func colSchemaForSliceElement(s abstract.ColSchema) abstract.ColSchema { s.OriginalType = strings.TrimSuffix(s.OriginalType, "[]") return s diff --git a/pkg/worker/tasks/load_sharded_snapshot.go b/pkg/worker/tasks/load_sharded_snapshot.go index 46060b56..a50a1b08 100644 --- a/pkg/worker/tasks/load_sharded_snapshot.go +++ b/pkg/worker/tasks/load_sharded_snapshot.go @@ -69,7 +69,7 @@ func (l *SnapshotLoader) GetShardState(ctx context.Context) error { return nil } -// OperationStateExists returns true if the state of the operation of the given task exists (is not nil) +// OperationStateExists returns true if the state of the operation of the given task exists (is not nil). func (l *SnapshotLoader) OperationStateExists(ctx context.Context) (bool, error) { result, err := backoff.RetryNotifyWithData( func() (bool, error) { diff --git a/pkg/worker/tasks/load_snapshot.go b/pkg/worker/tasks/load_snapshot.go index ad927312..b3936795 100644 --- a/pkg/worker/tasks/load_snapshot.go +++ b/pkg/worker/tasks/load_snapshot.go @@ -135,7 +135,7 @@ func (l *SnapshotLoader) CheckIncludeDirectives(tables []abstract.TableDescripti return nil } -// TODO Remove, legacy hacks +// TODO Remove, legacy hacks. func (l *SnapshotLoader) endpointsPreSnapshotActions(sourceStorage abstract.Storage) { switch specificStorage := sourceStorage.(type) { case *greenplum.Storage: @@ -153,7 +153,7 @@ func (l *SnapshotLoader) endpointsPreSnapshotActions(sourceStorage abstract.Stor } } -// TODO Remove, legacy hacks +// TODO Remove, legacy hacks. func (l *SnapshotLoader) endpointsPostSnapshotActions() { switch dst := l.transfer.Dst.(type) { case model.HackableTarget: diff --git a/pkg/worker/tasks/s3coordinator/load_sharded_snapshot_test.go b/pkg/worker/tasks/s3coordinator/load_sharded_snapshot_test.go index 78d87868..e9ec4340 100644 --- a/pkg/worker/tasks/s3coordinator/load_sharded_snapshot_test.go +++ b/pkg/worker/tasks/s3coordinator/load_sharded_snapshot_test.go @@ -80,7 +80,7 @@ func (f fakeSinker) Push(input []abstract.ChangeItem) error { return nil } -// Obsolete, remove or refactor after all transfers move to revision after TM-5319, TM-5321 +// Obsolete, remove or refactor after all transfers move to revision after TM-5319, TM-5321. func TestShardedUploadCoordinator(t *testing.T) { cp, err := s3coordinator.NewS3Recipe(os.Getenv("S3_BUCKET")) require.NoError(t, err) diff --git a/pkg/worker/tasks/snapshot_table_metrics_tracker.go b/pkg/worker/tasks/snapshot_table_metrics_tracker.go index 0311f8db..8e3280d9 100644 --- a/pkg/worker/tasks/snapshot_table_metrics_tracker.go +++ b/pkg/worker/tasks/snapshot_table_metrics_tracker.go @@ -138,7 +138,7 @@ func (t *SnapshotTableMetricsTracker) run() { // Close // Safe to close few time, not thread safe; -// But safe to use with defer and standalone call in same time; +// But safe to use with defer and standalone call in same time;. func (t *SnapshotTableMetricsTracker) Close() { if t.closed { return diff --git a/pkg/worker/tasks/snapshot_table_progress_tracker.go b/pkg/worker/tasks/snapshot_table_progress_tracker.go index c0719c2d..af4971b8 100644 --- a/pkg/worker/tasks/snapshot_table_progress_tracker.go +++ b/pkg/worker/tasks/snapshot_table_progress_tracker.go @@ -72,7 +72,7 @@ func (t *SnapshotTableProgressTracker) run() { // Close // Safe to close few time, not thread safe; -// But safe to use with defer and standalone call in same time; +// But safe to use with defer and standalone call in same time;. func (t *SnapshotTableProgressTracker) Close() { if t.closed { return @@ -137,14 +137,14 @@ func (t *SnapshotTableProgressTracker) Flush() { t.pushProgress() } -// AddGetProgress TODO: Remove, A2 thing +// AddGetProgress TODO: Remove, A2 thing. func (t *SnapshotTableProgressTracker) AddGetProgress(part *model.OperationTablePart, progressFunc func()) { t.progressUpdateMutex.Lock() defer t.progressUpdateMutex.Unlock() t.progressFuncs[part.Key()] = progressFunc } -// RemoveGetProgress TODO: Remove, A2 thing +// RemoveGetProgress TODO: Remove, A2 thing. func (t *SnapshotTableProgressTracker) RemoveGetProgress(part *model.OperationTablePart) { t.progressUpdateMutex.Lock() defer t.progressUpdateMutex.Unlock() diff --git a/pkg/worker/tasks/transitional_upload.go b/pkg/worker/tasks/transitional_upload.go index 4f9bab63..85d31230 100644 --- a/pkg/worker/tasks/transitional_upload.go +++ b/pkg/worker/tasks/transitional_upload.go @@ -137,7 +137,7 @@ func TransitUpload(ctx context.Context, cp coordinator.Coordinator, transfer mod return nil } -// TransitReupload is shitty method mainly for transfers with LB in the middle, same as TransitUpload +// TransitReupload is shitty method mainly for transfers with LB in the middle, same as TransitUpload. func TransitReupload(ctx context.Context, cp coordinator.Coordinator, transfer model.Transfer, task model.TransferOperation, registry metrics.Registry) error { snapshotLoader := NewSnapshotLoader(cp, task.OperationID, &transfer, registry) if !transfer.IsMain() { @@ -221,7 +221,7 @@ func TransitReupload(ctx context.Context, cp coordinator.Coordinator, transfer m return nil } -// TransitionalAddTables same as above +// TransitionalAddTables same as above. func TransitionalAddTables(ctx context.Context, cp coordinator.Coordinator, transfer model.Transfer, task model.TransferOperation, tables []string, registry metrics.Registry) error { linkedTransfers, err := GetLeftTerminalTransfers(cp, transfer) if err != nil { diff --git a/pkg/worker/tasks/update_transfer.go b/pkg/worker/tasks/update_transfer.go index 1f10002a..53ec29d6 100644 --- a/pkg/worker/tasks/update_transfer.go +++ b/pkg/worker/tasks/update_transfer.go @@ -15,7 +15,7 @@ import ( // UpdateTransfer is system task, generated by transfer update for `Running` transfers with active replication // `control plane` will prepare `UpdateDataObjectsParams` update transfer and start operation // `data plane` for all new object we will generate homo-DDL and upload data -// this operation similar to `AddTables` but without endpoint mutation +// this operation similar to `AddTables` but without endpoint mutation. func UpdateTransfer(ctx context.Context, cp coordinator.Coordinator, transfer model.Transfer, task model.TransferOperation, registry metrics.Registry, objects abstract.UpdateTransferParams) error { tables, err := objects.AddedTables() if err != nil { diff --git a/pkg/xtls/create.go b/pkg/xtls/create.go index 0bd66c6f..9bfbcd7b 100644 --- a/pkg/xtls/create.go +++ b/pkg/xtls/create.go @@ -3,7 +3,7 @@ package xtls import ( "crypto/tls" "crypto/x509" - "io/ioutil" + "os" "github.com/doublecloud/transfer/library/go/core/xerrors" ) @@ -14,7 +14,7 @@ func Pool(rootCACertPaths []string) (*x509.CertPool, error) { return nil, err } for _, certPath := range rootCACertPaths { - caCertificatePEM, err := ioutil.ReadFile(certPath) + caCertificatePEM, err := os.ReadFile(certPath) if err != nil { return nil, xerrors.Errorf("Cannot read file %s: %w", certPath, err) } From b15ced6fb737246c8c0af913e05c666811dc828e Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 18:34:16 -0300 Subject: [PATCH 2/7] linting checkpoint Signed-off-by: Henrique Goncalves --- tests/canon/all_replication_sequences.go | 2 +- tests/canon/clickhouse/canon_test.go | 3 +- tests/canon/mongo/canon_docs.go | 4 +- tests/canon/mongo/canon_test.go | 2 +- tests/canon/mysql/canon_test.go | 5 +- tests/canon/postgres/canon_test.go | 2 +- tests/canon/reference/table.go | 10 +- tests/canon/s3/csv/canon_test.go | 6 +- tests/canon/s3/jsonline/canon_test.go | 2 +- tests/canon/s3/parquet/canon_test.go | 6 +- tests/canon/sequences/sequences_test.go | 6 +- .../kafka2ch/replication_mv/check_db_test.go | 5 +- tests/e2e/mongo2mock/slots/slot_test.go | 6 +- .../bson_obj_too_large/check_db_test.go | 6 +- .../mongo2mongo/replication/check_db_test.go | 2 +- .../replication_filter_test/check_db_test.go | 7 +- .../rps/replication_source/rps_test.go | 2 +- tests/e2e/mongo2mongo/rps/rps.go | 16 +- .../document_key_updates/rps_test.go | 346 ------------------ .../nested_shard_key/nested_shard_key_test.go | 272 -------------- .../mongo2yt/rotator/rotator_test_common.go | 2 +- .../debezium_replication/check_db_test.go | 6 +- tests/e2e/mysql2mock/views/check_db_test.go | 2 - tests/e2e/mysql2yt/alters/check_db_test.go | 9 - tests/e2e/mysql2yt/date_time/check_db_test.go | 3 +- tests/e2e/pg2ch/dbt/check_db_test.go | 4 +- .../check_db_test.go | 5 +- .../debezium_replication/check_db_test.go | 6 +- .../debezium_replication/check_db_test.go | 6 +- .../pg2mock/retry_conn_leak/check_db_test.go | 3 +- .../some_parts/partitioned_tables_test.go | 16 +- tests/e2e/pg2pg/replication/check_db_test.go | 2 - .../snapshot_missing_public/check_db_test.go | 2 +- .../replication_toasted/check_db_test.go | 2 +- .../check_db_test.go | 2 +- .../bulk_jsonb_pkey/bulk_json_generator.go | 14 +- .../bulk_json_generator_test.go | 11 +- .../cdc_partial_activate/check_db_test.go | 21 +- tests/e2e/pg2yt/enum/enum_join_test.go | 2 +- .../pg2yt/sql_transformer/check_db_test.go | 2 +- .../e2e/ydb2mock/incremental/check_db_test.go | 2 +- .../check_db_test.go | 10 +- tests/helpers/activate_delivery_wrapper.go | 4 +- .../endpoint_matcher.go | 2 +- tests/helpers/debezium_pg_array_comparator.go | 2 +- tests/helpers/http_proxy/proxy.go | 1 + tests/helpers/testsflag/testsflag.go | 2 +- tests/helpers/utils.go | 4 +- tests/helpers/ydb.go | 70 ++-- .../docker-compose/elasticsearch2pg_test.go | 3 + .../pg/permissions/permissions_test.go | 2 +- tests/tcrecipes/azure/azurite.go | 9 +- tests/tcrecipes/azure/credentials.go | 4 +- tests/tcrecipes/azure/eventhub.go | 8 +- tests/tcrecipes/clickhouse/clickhouse.go | 10 +- tests/tcrecipes/clickhouse/zookeeper.go | 2 +- tests/tcrecipes/k3s/k3s.go | 6 +- tests/tcrecipes/k3s/types.go | 14 +- tests/tcrecipes/kafka/kafka.go | 4 +- tests/tcrecipes/localstack/types.go | 10 +- tests/tcrecipes/postgres/postrges.go | 12 +- 61 files changed, 191 insertions(+), 820 deletions(-) delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go delete mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go diff --git a/tests/canon/all_replication_sequences.go b/tests/canon/all_replication_sequences.go index bc00a4f3..1bd08eda 100644 --- a/tests/canon/all_replication_sequences.go +++ b/tests/canon/all_replication_sequences.go @@ -13,7 +13,7 @@ import ( "github.com/doublecloud/transfer/pkg/abstract" ) -// SequenceTestCases contains a list of all sequence test cases. Each new case must be added here +// SequenceTestCases contains a list of all sequence test cases. Each new case must be added here. var SequenceTestCases = []string{ "insert_update_delete", "updatepk", diff --git a/tests/canon/clickhouse/canon_test.go b/tests/canon/clickhouse/canon_test.go index 3f20bb30..a665eb9a 100644 --- a/tests/canon/clickhouse/canon_test.go +++ b/tests/canon/clickhouse/canon_test.go @@ -1,7 +1,6 @@ package clickhouse import ( - "os" "sort" "strings" "testing" @@ -50,7 +49,7 @@ func getBaseType(colSchema abstract.ColSchema) string { } func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga Source := &model.ChSource{ ShardsList: []model.ClickHouseShard{ { diff --git a/tests/canon/mongo/canon_docs.go b/tests/canon/mongo/canon_docs.go index 0dca6712..a90db2b7 100644 --- a/tests/canon/mongo/canon_docs.go +++ b/tests/canon/mongo/canon_docs.go @@ -48,7 +48,7 @@ var ( }}}, bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12c4")}, {Key: "nil", Value: nil}}, } - // this piece is valid only for repackable hetero transfer + // this piece is valid only for repackable hetero transfer. ExtraSnapshotDocuments = []interface{}{ bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12b4")}, {Key: "nan", Value: math.NaN()}, {Key: "inf+", Value: math.Inf(1)}, {Key: "inf-", Value: math.Inf(-1)}}, } @@ -79,7 +79,7 @@ var ( }}}, bson.D{{Key: "_id", Value: oid("62e98ebcb28dbd2fffdf12cf")}, {Key: "nil", Value: primitive.Null{}}}, // primitive.Null is the same as nil } - // Update existing + // Update existing. IncrementUpdates = []IncrementUpdate{ { Filter: bson.D{{Key: "_id", Value: "omg"}}, diff --git a/tests/canon/mongo/canon_test.go b/tests/canon/mongo/canon_test.go index df3b2aaf..14aaacd1 100644 --- a/tests/canon/mongo/canon_test.go +++ b/tests/canon/mongo/canon_test.go @@ -16,7 +16,7 @@ import ( ) func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga databaseName := "canondb" t.Run("vanilla hetero case", func(t *testing.T) { snapshotPlusIncrementScenario(t, databaseName, "hetero_repack", false, false) diff --git a/tests/canon/mysql/canon_test.go b/tests/canon/mysql/canon_test.go index 8e2f8ec5..86bc9909 100644 --- a/tests/canon/mysql/canon_test.go +++ b/tests/canon/mysql/canon_test.go @@ -29,7 +29,7 @@ func execBatch(t *testing.T, conn *sql.DB, sqlCommands string) { } func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga Source := &mysql.MysqlSource{ ClusterID: os.Getenv("CLUSTER_ID"), Host: os.Getenv("RECIPE_MYSQL_HOST"), @@ -47,7 +47,6 @@ func TestCanonSource(t *testing.T) { }() tableCase := func(tableName string) func(t *testing.T) { - return func(t *testing.T) { connParams, err := mysql.NewConnectionParams(Source.ToStorageParams()) require.NoError(t, err) @@ -106,5 +105,5 @@ func TestCanonSource(t *testing.T) { t.Run("numeric_types_int", tableCase("numeric_types_int")) t.Run("string_types", tableCase("string_types")) t.Run("string_types_emoji", tableCase("string_types_emoji")) - //t.Run("spatial_types", tableCase("spatial_types")) + // t.Run("spatial_types", tableCase("spatial_types")) } diff --git a/tests/canon/postgres/canon_test.go b/tests/canon/postgres/canon_test.go index 353d6c8c..2fa35a2c 100644 --- a/tests/canon/postgres/canon_test.go +++ b/tests/canon/postgres/canon_test.go @@ -24,7 +24,7 @@ func TestCanonSource(t *testing.T) { if tcrecipes.Enabled() { _ = pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump")) } - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga srcPort := helpers.GetIntFromEnv("PG_LOCAL_PORT") Source := &postgres.PgSource{ ClusterID: os.Getenv("PG_CLUSTER_ID"), diff --git a/tests/canon/reference/table.go b/tests/canon/reference/table.go index 0365bfc8..d4a87078 100644 --- a/tests/canon/reference/table.go +++ b/tests/canon/reference/table.go @@ -39,7 +39,7 @@ type Row struct { Date time.Time `yson:"t_date"` DateTime time.Time `yson:"t_datetime"` Timestamp time.Time `yson:"t_timestamp"` - //Interval int64 `yson:"t_interval"` + // Interval int64 `yson:"t_interval"` Yson string `yson:"t_yson"` Decimal string `yson:"t_decimal"` @@ -73,7 +73,7 @@ func Table() []abstract.ChangeItem { {ColumnName: "t_date", DataType: string(schema.TypeDate)}, {ColumnName: "t_datetime", DataType: string(schema.TypeDatetime)}, {ColumnName: "t_timestamp", DataType: string(schema.TypeTimestamp)}, - //{ColumnName: "t_interval", DataType: schema.TypeInterval}, + // {ColumnName: "t_interval", DataType: schema.TypeInterval}, {ColumnName: "t_list", DataType: string(schema.TypeAny)}, {ColumnName: "t_struct", DataType: string(schema.TypeAny)}, {ColumnName: "t_tuple", DataType: string(schema.TypeAny)}, @@ -82,8 +82,8 @@ func Table() []abstract.ChangeItem { {ColumnName: "t_dict", DataType: string(schema.TypeAny)}, {ColumnName: "t_tagged", DataType: string(schema.TypeAny)}, // Complex types - //{ColumnName: "t_yson", DataType: schema.Optional{Item: schema.TypeAny}}, - //{ ColumnName: "t_decimal"}, + // {ColumnName: "t_yson", DataType: schema.Optional{Item: schema.TypeAny}}, + // { ColumnName: "t_decimal"}, }) colNames := slices.Map(cols.Columns(), func(colSchema abstract.ColSchema) string { return colSchema.ColumnName @@ -148,7 +148,7 @@ func Table() []abstract.ChangeItem { inRow.Date, inRow.DateTime, inRow.Timestamp, - //inRow.Interval, + // inRow.Interval, inRow.List, inRow.Struct, inRow.Tuple, diff --git a/tests/canon/s3/csv/canon_test.go b/tests/canon/s3/csv/canon_test.go index 416cb59a..4d0d9dda 100644 --- a/tests/canon/s3/csv/canon_test.go +++ b/tests/canon/s3/csv/canon_test.go @@ -17,7 +17,7 @@ import ( ) func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga testCasePath := "test_csv_all_types" src := s3.PrepareCfg(t, "", "") @@ -189,7 +189,7 @@ func TestCanonSource(t *testing.T) { var processed []abstract.ChangeItem func TestNativeS3WithProvidedSchemaAndSystemCols(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga processed = make([]abstract.ChangeItem, 0) testCasePath := "test_csv_all_types" @@ -258,7 +258,7 @@ func storeItems(item []abstract.ChangeItem) []abstract.ChangeItem { } func TestNativeS3MissingColumnsAreFilled(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga processed = make([]abstract.ChangeItem, 0) testCasePath := "test_csv_all_types" diff --git a/tests/canon/s3/jsonline/canon_test.go b/tests/canon/s3/jsonline/canon_test.go index dd0a63fe..88dd6645 100644 --- a/tests/canon/s3/jsonline/canon_test.go +++ b/tests/canon/s3/jsonline/canon_test.go @@ -15,7 +15,7 @@ import ( ) func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga testCasePath := "test_jsonline_all_types" src := s3.PrepareCfg(t, "", "") diff --git a/tests/canon/s3/parquet/canon_test.go b/tests/canon/s3/parquet/canon_test.go index a81e8ca2..2d1ffe41 100644 --- a/tests/canon/s3/parquet/canon_test.go +++ b/tests/canon/s3/parquet/canon_test.go @@ -18,7 +18,7 @@ import ( ) func TestUnsopportedData(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga absPath, err := filepath.Abs("unsupported_data") require.NoError(t, err) files, err := os.ReadDir(absPath) @@ -51,7 +51,7 @@ func TestUnsopportedData(t *testing.T) { } } -// rowsCutter will limit number of rows pushed to child sink +// rowsCutter will limit number of rows pushed to child sink. type rowsCutter struct { sink abstract.Sinker pushed bool @@ -82,7 +82,7 @@ func (r *rowsCutter) Push(items []abstract.ChangeItem) error { } func TestCanonSource(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga absPath, err := filepath.Abs("data") require.NoError(t, err) files, err := os.ReadDir(absPath) diff --git a/tests/canon/sequences/sequences_test.go b/tests/canon/sequences/sequences_test.go index 2a46ea01..cb3b5dea 100644 --- a/tests/canon/sequences/sequences_test.go +++ b/tests/canon/sequences/sequences_test.go @@ -29,7 +29,7 @@ var ( ) func TestCanonizeSequences(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga Source := &pgcommon.PgSource{ ClusterID: os.Getenv("PG_CLUSTER_ID"), Hosts: []string{"localhost"}, @@ -82,7 +82,7 @@ func TestCanonizeSequences(t *testing.T) { // new cases should be added here. The name of the cases MUST be placed in canon.SequenceTestCases so that the change in the set of cases is automatically propagated to all sinks under test } -// DropNonRowKindsExceptRowMiddleware drops all non-row items except the ones whose kind is among the given kinds +// DropNonRowKindsExceptRowMiddleware drops all non-row items except the ones whose kind is among the given kinds. func DropNonRowKindsExceptRowMiddleware(preserveKinds ...abstract.Kind) func([]abstract.ChangeItem) []abstract.ChangeItem { preserveKindsMap := make(map[abstract.Kind]bool) for _, k := range preserveKinds { @@ -100,7 +100,7 @@ func DropNonRowKindsExceptRowMiddleware(preserveKinds ...abstract.Kind) func([]a } } -// SynthesizeCommitTimeRowMiddleware sets synthetic sequential CommitTime for all items that it processes +// SynthesizeCommitTimeRowMiddleware sets synthetic sequential CommitTime for all items that it processes. func SynthesizeCommitTimeRowMiddleware() func([]abstract.ChangeItem) []abstract.ChangeItem { return func(items []abstract.ChangeItem) []abstract.ChangeItem { for i := range items { diff --git a/tests/e2e/kafka2ch/replication_mv/check_db_test.go b/tests/e2e/kafka2ch/replication_mv/check_db_test.go index 4e011d07..bcb67ae3 100644 --- a/tests/e2e/kafka2ch/replication_mv/check_db_test.go +++ b/tests/e2e/kafka2ch/replication_mv/check_db_test.go @@ -22,9 +22,8 @@ var ( kafkaTopic = "topic1" source = *kafkasink.MustSourceRecipe() - chDatabase = "public" - target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) - targetAsSource = *chrecipe.MustSource(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) + chDatabase = "public" + target = *chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(chDatabase)) timestampToUse = time.Date(2024, 03, 19, 0, 0, 0, 0, time.Local) ) diff --git a/tests/e2e/mongo2mock/slots/slot_test.go b/tests/e2e/mongo2mock/slots/slot_test.go index 5bfd29aa..81ad4240 100644 --- a/tests/e2e/mongo2mock/slots/slot_test.go +++ b/tests/e2e/mongo2mock/slots/slot_test.go @@ -83,19 +83,19 @@ func setOplogSize(ctx context.Context, client *mongo.MongoClientWrapper, sizeInS return nil } -// just mock sinker +// just mock sinker. type mockSinker struct{} func (m mockSinker) Close() error { return nil } func (m mockSinker) Push(items []abstract.ChangeItem) error { return nil } -// controlplane that catches replication failure +// controlplane that catches replication failure. type mockCPFailRepl struct { cpclient.CoordinatorNoOp err error } -// test data structure +// test data structure. type Pepe struct { DayOfTheWeek string DayOfTheWeekID int diff --git a/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go b/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go index c734b9dc..dd8acdfd 100644 --- a/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go +++ b/tests/e2e/mongo2mongo/bson_obj_too_large/check_db_test.go @@ -193,8 +193,8 @@ func LoadFromchangestream(t *testing.T) { time.Sleep(time.Second) tryingsCount := 30 - tries := 0 - var dstTableSize uint64 = 0 + var tries int + var dstTableSize uint64 for tries = 0; tries < tryingsCount; tries++ { allOk := true for _, td := range []abstract.TableDescription{ @@ -283,7 +283,7 @@ func LoadFrompurecursor(t *testing.T) { time.Sleep(time.Second) const tryingsCount int = 30 - var dstTableSize uint64 = 0 + var dstTableSize uint64 for tries := 0; tries < tryingsCount; tries++ { allOk := true for _, td := range []abstract.TableDescription{ diff --git a/tests/e2e/mongo2mongo/replication/check_db_test.go b/tests/e2e/mongo2mongo/replication/check_db_test.go index 8d935ac5..8e089a03 100644 --- a/tests/e2e/mongo2mongo/replication/check_db_test.go +++ b/tests/e2e/mongo2mongo/replication/check_db_test.go @@ -158,7 +158,7 @@ func Load(t *testing.T) { require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) } -// define mock sinker to report error +// define mock sinker to report error. var MockSinkerError = xerrors.New("You picked the wrong house, fool!") type mockSinker struct { diff --git a/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go b/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go index 40962460..698ff9ea 100644 --- a/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go +++ b/tests/e2e/mongo2mongo/replication_filter_test/check_db_test.go @@ -23,7 +23,7 @@ import ( "go.mongodb.org/mongo-driver/bson" ) -// creates source from environment settings/recipe +// creates source from environment settings/recipe. func sourceFromConfig() (*mongodataagent.MongoSource, error) { srcPort, err := strconv.Atoi(os.Getenv("MONGO_LOCAL_PORT")) if err != nil { @@ -66,16 +66,17 @@ func makeTransfer(id string, source *mongodataagent.MongoSource, target *mongoda } func TestGroup(t *testing.T) { + t.Parallel() sourcePort, err := strconv.Atoi(os.Getenv("MONGO_LOCAL_PORT")) require.NoError(t, err) targetPort, err := strconv.Atoi(os.Getenv("DB0_MONGO_LOCAL_PORT")) require.NoError(t, err) - defer func() { + t.Cleanup(func() { require.NoError(t, helpers.CheckConnections( helpers.LabeledPort{Label: "Mongo source", Port: sourcePort}, helpers.LabeledPort{Label: "Mongo target", Port: targetPort}, )) - }() + }) t.Run("Empty Collection List Means Include All", testEmptyCollectionListIncludesAll) t.Run("Include All Collections Test", testCollectionFilterIncludeWholeDB) diff --git a/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go b/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go index c2b10b6b..4e20ec53 100644 --- a/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go +++ b/tests/e2e/mongo2mongo/rps/replication_source/rps_test.go @@ -326,7 +326,7 @@ func RpsTestFactory(testParameters RpsTestParameters) func(t *testing.T) { // wait for replication to catch up lag rowCount := uint64(len(rpsModel.Persistent)) tryingsCount := 30 - tries := 0 + var tries int for tries = 0; tries < tryingsCount; tries++ { td := abstract.TableID{Namespace: DB, Name: Collection} dstTableSize, err := dstStorage.ExactTableRowsCount(td) // TODO(@kry127;@timmyb32r) TM2409 change on GetRowsCount() diff --git a/tests/e2e/mongo2mongo/rps/rps.go b/tests/e2e/mongo2mongo/rps/rps.go index dcf07fbc..4510160a 100644 --- a/tests/e2e/mongo2mongo/rps/rps.go +++ b/tests/e2e/mongo2mongo/rps/rps.go @@ -18,7 +18,7 @@ import ( "go.ytsaurus.tech/library/go/core/log" ) -// KV is Key-Document object for key-value database (Mongo or in-memory) +// KV is Key-Document object for key-value database (Mongo or in-memory). type KV struct { Key string `bson:"_id"` Document interface{} `bson:"document"` @@ -37,7 +37,7 @@ func (k *KV) String() string { } // GenerateKV produces random Key-Document object with key and value as -// string of `keysize` and `valsize` length respectfully +// string of `keysize` and `valsize` length respectfully. func GenerateKV(keysize, valsize int) KV { return KV{ Key: randutil.GenerateAlphanumericString(keysize), @@ -46,14 +46,14 @@ func GenerateKV(keysize, valsize int) KV { } // RpsSpec defines specification of requests count per Delay -// applied in order: delete, create, update +// applied in order: delete, create, update. type RpsSpec struct { DeleteCount, CreateCount, UpdateCount, ReplaceCount uint KVConstructor func() KV `json:"-"` Delay time.Duration } -// RpsNotifier contains callbacks for different events of RPS generator +// RpsNotifier contains callbacks for different events of RPS generator. type RpsCallbacks struct { Tick func(ctx context.Context, index int, rps *RpsModel) bool OnDelete func(ctx context.Context, key string) @@ -63,7 +63,7 @@ type RpsCallbacks struct { } var ( - // default spec immediately gives control to user + // default spec immediately gives control to user. defaultRpsSpec = RpsSpec{ KVConstructor: func() KV { return GenerateKV(10, 10) @@ -110,7 +110,7 @@ func HistoryToString(history []historyDescription) string { // 1. rate-limiter for requests // 2. in-mem KV storage // -// Anyone can access in-memory Persistent state, NonPersistent(deleted) state, and change history as ModelHistory +// Anyone can access in-memory Persistent state, NonPersistent(deleted) state, and change history as ModelHistory. type RpsModel struct { // we will use this for generating RPS: every 'period' milliseconds the 'timer' hits timer *time.Timer @@ -129,7 +129,7 @@ func (r *RpsModel) Close() { r.timer.Stop() } -// SetSpec sets RPS specification +// SetSpec sets RPS specification. func (r *RpsModel) SetSpec(spec *RpsSpec) { if spec == nil { spec = &defaultRpsSpec @@ -285,7 +285,7 @@ func (r *RpsModel) Start() { // NewRpsModel creates RPS model // use initialSpec to set frequency and value of operations // use onCreate, onDelete and onUpdate from RpsCallbacks to make actions (e.g. with database) -// use Delay in RpsCallbacks to reconfigure RPS in time +// use Delay in RpsCallbacks to reconfigure RPS in time. func NewRpsModel(ctx context.Context, notifiers *RpsCallbacks) *RpsModel { newCtx, newCtxCancel := context.WithCancel(ctx) diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go b/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go deleted file mode 100644 index ffe1afb0..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go +++ /dev/null @@ -1,346 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/transfer/internal/logger" - "github.com/doublecloud/transfer/pkg/abstract" - cpclient "github.com/doublecloud/transfer/pkg/abstract/coordinator" - "github.com/doublecloud/transfer/pkg/abstract/model" - mongostorage "github.com/doublecloud/transfer/pkg/providers/mongo" - "github.com/doublecloud/transfer/pkg/randutil" - "github.com/doublecloud/transfer/pkg/runtime/local" - "github.com/doublecloud/transfer/pkg/worker/tasks" - mongoshardedcluster "github.com/doublecloud/transfer/recipe/mongo/pkg/cluster" - "github.com/doublecloud/transfer/tests/e2e/mongo2mongo/rps" - "github.com/doublecloud/transfer/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "go.ytsaurus.tech/library/go/core/log" -) - -func TestGroup(t *testing.T) { - t.Skip("TM-5255 temporary skip tests") - - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("RpsTest", RpsTest) - }) -} - -const ( - slotIDAkaTransferID = "dtt_shard_to_shard" - DB = "db1" - Collection = "coll1" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = &mongostorage.MongoSource{ - Hosts: []string{os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: model.SecretString(os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Collections: []mongostorage.MongoCollection{ - {DatabaseName: DB, CollectionName: Collection}, - }, - SlotID: slotIDAkaTransferID, - } - Target = mongostorage.MongoDestination{ - Hosts: []string{os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: model.SecretString(os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Cleanup: model.DisabledCleanup, - } -) - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongostorage.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongostorage.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func MakeDstClient(t *mongostorage.MongoDestination) (*mongostorage.MongoClientWrapper, error) { - return mongostorage.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) -} - -func ShardSourceCollection(t *testing.T, client *mongostorage.MongoClientWrapper) { - adminDB := client.Database("admin") - - res := adminDB.RunCommand(context.TODO(), - bson.D{ - {Key: "enableSharding", Value: DB}, - }) - require.NoError(t, res.Err()) - - var runCmdResult bson.M - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection)}, - {Key: "key", Value: bson.D{ - {Key: "document.a", Value: "hashed"}, - {Key: "document.b", Value: 1}, - {Key: "document.c", Value: 1}, - }}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) -} - -func ShardTargetCollection(t *testing.T, client *mongostorage.MongoClientWrapper) { - adminDB := client.Database("admin") - - res := adminDB.RunCommand(context.TODO(), - bson.D{ - {Key: "enableSharding", Value: DB}, - }) - require.NoError(t, res.Err()) - - key := bson.D{ - {Key: "document.x", Value: "hashed"}, - {Key: "document.y", Value: 1}, - {Key: "document.z", Value: 1}, - } - - var runCmdResult bson.M - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection)}, - {Key: "key", Value: key}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(Source) - client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := MakeDstClient(&Target) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func clearStorage(t *testing.T, client *mongostorage.MongoClientWrapper) { - t.Helper() - var err error - - db := client.Database(DB) - _ = db.Collection(Collection).Drop(context.Background()) - err = db.CreateCollection(context.Background(), Collection) - require.NoError(t, err) -} - -func RpsTest(t *testing.T) { - for _, rsName := range []mongostorage.MongoReplicationSource{ - mongostorage.MongoReplicationSourcePerDatabaseFullDocument, - mongostorage.MongoReplicationSourcePerDatabaseUpdateDocument, - } { - t.Run(string(rsName), func(t *testing.T) { - RpsTestForRS(t, rsName) - }) - } -} - -func RpsTestForRS(t *testing.T, rs mongostorage.MongoReplicationSource) { - ctx := context.Background() - - clientSource, err := mongostorage.Connect(ctx, Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - defer func() { _ = clientSource.Close(context.Background()) }() - - // recreate collections on source - clearStorage(t, clientSource) - dbSource := clientSource.Database(DB) - collectionSource := dbSource.Collection(Collection) - - // make connection to the target - clientTarget, err := mongostorage.Connect(ctx, Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - // drop collection on target before sharding - clearStorage(t, clientTarget) - - // shard source - //ShardSourceCollection(t, clientSource) //IS it recipe limitation? - // shard target - ShardTargetCollection(t, clientTarget) - - mongoSource := Source - mongoSource.ReplicationSource = rs - transfer := helpers.MakeTransfer(helpers.TransferID, mongoSource, &Target, TransferType) - - // activate transfer - err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - // start local worker for activation - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - errChan := make(chan error, 1) - go func() { - errChan <- localWorker.Run() // like .Start(), but we in control for processing error in test - }() - - dstStorage, err := mongostorage.NewStorage(Target.ToStorageParams()) - require.NoError(t, err) - - // configure desired RPS - rpsContext, rpsCancel := context.WithCancel(ctx) - defer rpsCancel() - rpsModel := rps.NewRpsModel(rpsContext, &rps.RpsCallbacks{ - OnCreate: func(ctx context.Context, entity rps.KV) { - _, err := collectionSource.InsertOne(ctx, entity) - require.NoError(t, err) - }, - OnUpdate: func(ctx context.Context, previous rps.KV, actual rps.KV) { - opts := options.Update() - doc, ok := previous.Document.(bson.D) - require.True(t, ok) - filter := bson.D{ - {Key: "_id", Value: previous.Key}, - {Key: "document.a", Value: doc.Map()["a"]}, - {Key: "document.b", Value: doc.Map()["b"]}, - {Key: "document.c", Value: doc.Map()["c"]}, - } - update := bson.D{{Key: "$set", Value: bson.D{{Key: "document", Value: actual.Document}}}} - result, err := collectionSource.UpdateOne(ctx, filter, update, opts) - require.NoError(t, err) - require.Equal(t, int64(1), result.ModifiedCount) - }, - OnReplace: func(ctx context.Context, previous rps.KV, actual rps.KV) { - opts := options.Replace() - filter := bson.D{{Key: "_id", Value: previous.Key}} - result, err := collectionSource.ReplaceOne(ctx, filter, actual, opts) - require.NoError(t, err) - require.Equal(t, int64(1), result.ModifiedCount) - }, - OnDelete: func(ctx context.Context, key string) { - filter := bson.D{{Key: "_id", Value: key}} - result, err := collectionSource.DeleteOne(ctx, filter, nil) - require.NoError(t, err) - require.Equal(t, int64(1), result.DeletedCount) - }, - Tick: func(ctx context.Context, tickId int, model *rps.RpsModel) bool { - if tickId > 12 { - // stop generation on last Delay - logger.Log.Info("RPS stopping", log.Int("tickId", tickId)) - return false - } - return true - }, - }) - - rpsModel.SetSpec(&rps.RpsSpec{ - DeleteCount: 100, - CreateCount: 100, - UpdateCount: 100, - ReplaceCount: 100, - KVConstructor: func() rps.KV { - return rps.KV{ - Key: randutil.GenerateAlphanumericString(16), - Document: bson.D{ - {Key: "a", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "b", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "c", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "x", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "y", Value: randutil.GenerateAlphanumericString(8)}, - {Key: "z", Value: randutil.GenerateAlphanumericString(8)}, - }, - } - }, - Delay: 0, - }) - - logger.Log.Info("Start RPS generator") - rpsModelDone := make(chan struct{}) - go func() { - defer rpsModel.Close() - defer close(rpsModelDone) - rpsModel.Start() - }() - select { - case <-rpsModelDone: - break - case <-ctx.Done(): - t.Fatal("Couldn't wait for RPS to close") - } - - // wait for replication to catch up lag - rowCount := uint64(len(rpsModel.Persistent)) - tryingsCount := 30 - tries := 0 - for tries = 0; tries < tryingsCount; tries++ { - td := abstract.TableID{Namespace: DB, Name: Collection} - dstTableSize, err := dstStorage.ExactTableRowsCount(td) // TODO(@kry127;@timmyb32r) TM2409 change on GetRowsCount() - require.NoError(t, err) - - t.Logf("Table: %s, count rows. Expected: %d, actual: %d", td.Fqtn(), rowCount, dstTableSize) - if dstTableSize == rowCount { - break - } - time.Sleep(time.Second) - } - if tries == tryingsCount { - // nevermind, further test is unpassable - t.Logf("Tries are over: %d out of %d", tries, tryingsCount) - } - - // wait a little bit (push batch delay is recomended) - time.Sleep(3 * mongostorage.DefaultBatchFlushInterval) - - // stop worker - logger.Log.Info("Stop local worker") - err = localWorker.Stop() - require.NoError(t, err) - - // wait for appropriate error from replication - select { - case err := <-errChan: - require.NoError(t, err) - case <-ctx.Done(): - t.Fatalf("Couldn't wait until replication ended: %v", ctx.Err()) - } - - dbTarget := clientTarget.Database(DB) - collectionTarget := dbTarget.Collection(Collection) - - // check that 'persistent' is present in source and target, and they values are equal - // and check that 'not persistent' neither on source nor target - logger.Log.Info("Validation of source and target databases") - for fromWhere, coll := range map[string]*mongo.Collection{"source": collectionSource, "target": collectionTarget} { - rpsModel.CheckValid(t, ctx, fromWhere, coll) - } - - logger.Log.Info("All values validated, tear down") -} diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go b/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go deleted file mode 100644 index ea00e7f8..00000000 --- a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package shmongo - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/doublecloud/transfer/internal/logger" - "github.com/doublecloud/transfer/pkg/abstract" - cpclient "github.com/doublecloud/transfer/pkg/abstract/coordinator" - "github.com/doublecloud/transfer/pkg/abstract/model" - mongodataagent "github.com/doublecloud/transfer/pkg/providers/mongo" - "github.com/doublecloud/transfer/pkg/runtime/local" - "github.com/doublecloud/transfer/pkg/worker/tasks" - mongoshardedcluster "github.com/doublecloud/transfer/recipe/mongo/pkg/cluster" - "github.com/doublecloud/transfer/tests/helpers" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" -) - -func init() { - _ = os.Setenv("YC", "1") // to not go to vanga -} - -func TestGroup(t *testing.T) { - defer func() { - require.NoError(t, helpers.CheckConnections( - helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, - helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, - )) - }() - - t.Run("Group after port check", func(t *testing.T) { - t.Run("Ping", Ping) - t.Run("Load", Load) - }) -} - -const ( - slotIDAkaTransferID = "dtt_shard_to_shard" - DB = "db1" - Collection1 = "coll1" - Collection2 = "coll2" - Collection3 = "coll3" -) - -var ( - TransferType = abstract.TransferTypeSnapshotAndIncrement - Source = &mongodataagent.MongoSource{ - Hosts: []string{os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: model.SecretString(os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Collections: []mongodataagent.MongoCollection{ - {DatabaseName: DB, CollectionName: Collection1}, - {DatabaseName: DB, CollectionName: Collection2}, - {DatabaseName: DB, CollectionName: Collection3}, - }, - SlotID: slotIDAkaTransferID, - } - Target = mongodataagent.MongoDestination{ - Hosts: []string{os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, - Port: helpers.GetIntFromEnv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPort), - User: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterUsername), - Password: model.SecretString(os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), - AuthSource: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), - Cleanup: model.DisabledCleanup, - } -) - -func init() { - Source.WithDefaults() -} - -//--------------------------------------------------------------------------------------------------------------------- -// utils - -func LogMongoSource(s *mongodataagent.MongoSource) { - fmt.Printf("Source.Hosts: %v\n", s.Hosts) - fmt.Printf("Source.Port: %v\n", s.Port) - fmt.Printf("Source.User: %v\n", s.User) - fmt.Printf("Source.Password: %v\n", s.Password) -} - -func LogMongoDestination(s *mongodataagent.MongoDestination) { - fmt.Printf("Target.Hosts: %v\n", s.Hosts) - fmt.Printf("Target.Port: %v\n", s.Port) - fmt.Printf("Target.User: %v\n", s.User) - fmt.Printf("Target.Password: %v\n", s.Password) -} - -func ShardTargetCollections(t *testing.T, client *mongodataagent.MongoClientWrapper) { - adminDB := client.Database("admin") - - res := adminDB.RunCommand(context.TODO(), - bson.D{ - {Key: "enableSharding", Value: DB}, - }) - require.NoError(t, res.Err()) - - key1 := bson.D{ - {Key: "_id.x", Value: "hashed"}, - } - - key2 := bson.D{ - {Key: "_id.x", Value: "hashed"}, - {Key: "city", Value: 1}, - } - - key3 := bson.D{ - {Key: "_id.x", Value: "hashed"}, - {Key: "_id.y", Value: 1}, - } - - var runCmdResult bson.M - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection1)}, - {Key: "key", Value: key1}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) - - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection2)}, - {Key: "key", Value: key2}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) - - require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ - {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection3)}, - {Key: "key", Value: key3}, - {Key: "unique", Value: false}, - }).Decode(&runCmdResult)) -} - -func Ping(t *testing.T) { - // ping src - LogMongoSource(Source) - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - defer func() { _ = client.Close(context.Background()) }() - require.NoError(t, err) - err = client.Ping(context.TODO(), nil) - require.NoError(t, err) - - // ping dst - LogMongoDestination(&Target) - client2, err := mongodataagent.Connect(context.Background(), Target.ConnectionOptions([]string{}), nil) - defer func() { _ = client2.Close(context.Background()) }() - require.NoError(t, err) - err = client2.Ping(context.TODO(), nil) - require.NoError(t, err) -} - -func insertOne(t *testing.T, coll *mongo.Collection, row any) { - _, err := coll.InsertOne(context.Background(), row) - require.NoError(t, err) -} - -func updateOne(t *testing.T, coll *mongo.Collection, filter, update bson.D) { - _, err := coll.UpdateOne(context.Background(), filter, update) - require.NoError(t, err) -} - -func Load(t *testing.T) { - client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - //------------------------------------------------------------------------------------ - // create source collections - - db := client.Database(DB) - err = db.CreateCollection(context.Background(), Collection1) - require.NoError(t, err) - err = db.CreateCollection(context.Background(), Collection2) - require.NoError(t, err) - err = db.CreateCollection(context.Background(), Collection3) - require.NoError(t, err) - - coll1 := db.Collection(Collection1) - coll2 := db.Collection(Collection2) - coll3 := db.Collection(Collection3) - - type CompositeID struct { - X string `bson:"x,omitempty"` - Y string `bson:"y,omitempty"` - } - - type Citizen struct { - ID CompositeID `bson:"_id"` - Name string `bson:"name,omitempty"` - Age int `bson:"age,omitempty"` - City string `bson:"city,omitempty"` - } - - jamesGordon := Citizen{ - ID: CompositeID{"x1", "y1"}, - Name: "James Gordon", - Age: 33, - City: "Gotham", - } - - insertOne(t, coll1, jamesGordon) - insertOne(t, coll2, jamesGordon) - insertOne(t, coll3, jamesGordon) - - //------------------------------------------------------------------------------------ - // shard target collections - - targetClient, err := mongodataagent.Connect(context.Background(), Target.ConnectionOptions([]string{}), nil) - require.NoError(t, err) - - targetDB := targetClient.Database(DB) - err = targetDB.CreateCollection(context.Background(), Collection1) - require.NoError(t, err) - err = targetDB.CreateCollection(context.Background(), Collection2) - require.NoError(t, err) - err = targetDB.CreateCollection(context.Background(), Collection3) - require.NoError(t, err) - - ShardTargetCollections(t, targetClient) - - //------------------------------------------------------------------------------------ - // activate - - transfer := helpers.MakeTransfer(helpers.TransferID, Source, &Target, abstract.TransferTypeSnapshotAndIncrement) - - err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) - require.NoError(t, err) - - localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) - localWorker.Start() - defer localWorker.Stop() //nolint - - //------------------------------------------------------------------------------------ - // replicate update one record and insert one - - leslieThompkins := Citizen{ - ID: CompositeID{"x2", "y2"}, - Name: "Leslie Thompkins", - Age: 29, - City: "Gotham", - } - - insertOne(t, coll1, leslieThompkins) - insertOne(t, coll2, leslieThompkins) - insertOne(t, coll3, leslieThompkins) - - leslieFilter := bson.D{{Key: "_id", Value: leslieThompkins.ID}} - leslieUpdate := bson.D{{Key: "$set", Value: bson.D{{Key: "city", Value: "Atlanta"}}}} - - updateOne(t, coll1, leslieFilter, leslieUpdate) - updateOne(t, coll2, leslieFilter, leslieUpdate) - updateOne(t, coll3, leslieFilter, leslieUpdate) - - jamesFilter := bson.D{{Key: "_id", Value: jamesGordon.ID}} - jamesUpdate := bson.D{{Key: "$set", Value: bson.D{{Key: "age", Value: 34}}}} - - updateOne(t, coll1, jamesFilter, jamesUpdate) - updateOne(t, coll2, jamesFilter, jamesUpdate) - updateOne(t, coll3, jamesFilter, jamesUpdate) - - //------------------------------------------------------------------------------------ - // check results - - require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection1, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection2, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection3, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) - require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) -} diff --git a/tests/e2e/mongo2yt/rotator/rotator_test_common.go b/tests/e2e/mongo2yt/rotator/rotator_test_common.go index deb13e63..b38d881f 100644 --- a/tests/e2e/mongo2yt/rotator/rotator_test_common.go +++ b/tests/e2e/mongo2yt/rotator/rotator_test_common.go @@ -22,7 +22,7 @@ import ( "go.ytsaurus.tech/yt/go/yttest" ) -// constants (begin) +// constants (begin). const ( TimeColumnName = "partition_time" ) diff --git a/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go b/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go index f76ae1b0..c220a0c4 100644 --- a/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go +++ b/tests/e2e/mysql2mock/debezium/debezium_replication/check_db_test.go @@ -31,13 +31,13 @@ func init() { //--------------------------------------------------------------------------------------------------------------------- -// fill 't' by giant random string +// fill 't' by giant random string. var update1Stmt = `UPDATE customers3 SET text_ = 'LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\u003c4SaNJTHkL@1?6YcDf\u003eHI[862bUb4gT@k\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^\u003e\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4\u003cI_@d]\u003eF1e]hj_XJII862[N\u003cj=bYA\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\u003e0UDDBb8h]65C\u003efC\u003c[02jRT]bJ\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\u003eH2X\\]_\u003cEE3@?U2_L67UV8FNQecS2Y=@6\u003ehb1\\3F66UE[W9\u003c]?HH\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJ\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\u003e@A\u003e5\u003cK\\d4QM:7:41B^_c\\FCI=\u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[T\u003cIQI4S_g\u003e;gf[BF_EN\u003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\u003c\u003e[=0W3Of;6;RFY=Q\\OK\\7[\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\u003cOK6gV=EMGC?\\F\u003cXaa_\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;\u003eMZG\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\u003e7cU]M[\u003c72c\u003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:\u003ea5a;j\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\u003c84==_9FJbjbEhQeOV\u003eWDP4MV^W1_]=TeAa66jLObKG\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\u003ebGaJ2f;VB\u003eG\\3\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\u003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\u003ek\u003ePUHD6\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\u003e=\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@f\u003ciDV?6i0WVXj\u003c@ZPd5d\\5B]O?7h=C=8O:L:IR8I\u003e^6\u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLE\u003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\u003e7UBbR58G?[X_O1b\\:[65\u003eP9Z6\u003c]S8=a\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\u003e^GMUPS2]b\u003e]DN?aUKNL^@RV\u003cFTBh:Q[Q3E5VHbK?5=RTKI\u003eggZZ\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\u003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[H\u003cUb4EE^\u003ckWO7\u003eR8fD9JQHR\u003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5\u003cBA\\3IVT@gG\\4\u003cRRS459YROd=_H1OM=a_hd\u003cSMLOd=S6^:eG\u003ejPgQ4_^d\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\u003cDa;\\Ni[AC\u003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9\u003eT12E?FZ9cYCLQbH[2O\u003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihE\u003ehMVaDF\u003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\u003eHga5eW[E8\u003c9jdYO7\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Q\u003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NR\u003eTK07=]7Ecdej\u003cUj\u003cDe1H\u003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_I\u003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\u003cSV@b[GVEU3Xh;R7\u003cXeTNgN\u003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhR\u003e]@GIYf[L55g\u003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\u003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\u003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9He\u003e1L[3\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\u003eCLdAe]6L2AD0aYHc5\u003e=fM7h\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]\u003eKE\u003cea\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\u003eLW5VIfJL:eQ4K3a1^WN5T=\\X=\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1C\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\u003eTQQWYJ_@FAX\\]9jh\u003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6\u003eBgES\u003e5EaeOFeG:i\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\u003eRQ]5Z9jA@Y9V1ZI6TDkC\u003eNZ_f_DR\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\u003eFW\u003cJ6X?IiJ\u003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;f\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHI\u003eI]gBS\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34\u003eh_2@i3kd02G\u003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\u003c^U7Hk]7Q6P:QZS;Ge@:\u003c\u003cfT6PK7j4?;cdC@c5GI:gS[W\u003cf26;\u003cBG7fMXFTWJcbB\\9QT\u003eh3HdV8Pb3Rh\u003e^?Ue:7RP[=jT4AE\u003ebiL_1dYW1\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\u003cA9LXQbECIc2M\u003c^I\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;c\u003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\u003c_F9P\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\u003e=R4U3W1G;\u003chN\\WFO_=DD\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5T\u003eY?bFOMZO\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\u003efaaP8P4KDVSCiQ=2\u003c=Ef:\u003eP\u003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_\u003c@5Z8fDPJAE8DcGUIb8C\u003c_L7XhP=\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\u003e]LW\u003ee^b\u003e?0G9Ie\u003c\u003c@UT4e9\u003cGM_jME7[6TFEN:\u003c\\H\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\u003cL42d\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\u003eEJQi8j;]L5CILgXdR_\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\u003cKHA:\\[CW7SRYVhE1[MD\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8k\u003eQb]UVVZ:18fe_8M?\\?\u003e\u003eLf4QSG@jO@\u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\u003cR]Ofg:TNGW0L\u003ePOC_CP\u003e^PI[aZ:KY^V@Q;;ME_k\\K0\u003eYP]1D5QSc51SfZ]FIP1Y6\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\u003e0]RJGeQiJG5\\=O8TRG5U\u003eLGa\u003eRi2K\u003c3=1TVHN=FhTJYajbIP\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\u003e93HU2ig?7\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1J\u003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^G\u003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1a\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\u003cA\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\u003e@^\u003e[4\u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVde\u003cUVVNH2EJ^=ALOFKUX:^\u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2S\u003cX?9bC7Ebc5V5E]\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SC\u003e\u003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\u003ceM8?j]NZai4\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\u003eI=?@f\u003cG349NMId8[T^@Sf\u003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@d\u003cc\u003cMhS3K;b\u003eZbHAf[GKME9igTY7iVFba\u003e4D;WFVb=dQ4Abj2\u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\u003e@TM9eO\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\u003c@=gPHLhQFDC@:T\u003cREdY\u003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1ad\u003cIiK1O7fbD[7[\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\u003e=FFMHPSBf8:\\XRZ91D:2D[1Y\u003eX\\bfj4BEQZe:1A\u003cQj^@7SAK]C_NCM\\0\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\u003e2\u003e4X:9JYPXk\u003eX_?;DAfL\u003ec?HF\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\u003e1\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aE\u003cY^MPd\u003e6M^iNNe=P6i6Lf::P6ebjX;\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\u003c93\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\u003c8@j5e\u003eVA76=g2=gD4V1eYF0bZd0EZ\u003cMk2M4g[Z=baJ]cVY\u003c[D=U2RUdBNdW=69=8UB4E1@\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;D\u003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4ia\u003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fa\u003e:Vj=BR7EW0_hV4=]DaSeQ\u003c?8]?9X4GbZF41h;FS\u003c9Pa=^SQT\u003cL:GAIP3XX[\\4RKJVLFabj20Oc\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\W\u003cHg9FWd\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\u003eS\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\u003e67\u003eW\u003cQNZRKDH@]_j^M_AV9g4\u003chIF\u003eaSDhbj9GMdjh=F=j:\u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaS\u003eO]caAKi\u003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\u003cWkC\u003c^KSgbI7?aGVaRkbA2?_Raf^\u003e9DID]07\u003cS431;BaRhX:hNJj]\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\u003cN?J\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\u003e8]\u003eU2:HGATaUBPG\u003c\\c0aX@_D;_EOK=]Sjk=1:VGK\u003e=4P^K\\OD\\D008D\u003cgY[GfMjeM\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNf\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\u003ceaB7BV;\u003eIRKK' WHERE pk=1;` -// TOASTed update +// TOASTed update. var update2Stmt = `UPDATE customers3 SET bool1=true WHERE bool1=false;` -// update with pkey change +// update with pkey change. var update3Stmt = `UPDATE customers3 SET pk=2 WHERE pk=1;` var deleteStmt = `DELETE FROM customers3 WHERE 1=1;` var insertStmt = ` diff --git a/tests/e2e/mysql2mock/views/check_db_test.go b/tests/e2e/mysql2mock/views/check_db_test.go index 24feb60e..2c717901 100644 --- a/tests/e2e/mysql2mock/views/check_db_test.go +++ b/tests/e2e/mysql2mock/views/check_db_test.go @@ -108,7 +108,6 @@ func TestMySQLHeteroViewsInteraction(t *testing.T) { require.NoError(t, err) require.NoError(t, rows.Close()) } - notesPerTable = 3 } notesPerTable = 2 require.Equal(t, params.checkTableLength, func() int { @@ -126,5 +125,4 @@ func TestMySQLHeteroViewsInteraction(t *testing.T) { }) }(testCase) } - } diff --git a/tests/e2e/mysql2yt/alters/check_db_test.go b/tests/e2e/mysql2yt/alters/check_db_test.go index 431b2501..f84a85c2 100644 --- a/tests/e2e/mysql2yt/alters/check_db_test.go +++ b/tests/e2e/mysql2yt/alters/check_db_test.go @@ -27,15 +27,6 @@ import ( var ( Source = *helpers.WithMysqlInclude(helpers.RecipeMysqlSource(), []string{"__test_a", "__test_b", "__test_c", "__test_d"}) Target = yt_helpers.RecipeYtTarget("//home/cdc/test/mysql2yt_e2e_alters") - - db = os.Getenv("RECIPE_MYSQL_SOURCE_DATABASE") - - tableNames = []abstract.TableDescription{ - {Name: fmt.Sprintf("%s___test_a", db)}, - {Name: fmt.Sprintf("%s___test_b", db)}, - {Name: fmt.Sprintf("%s___test_c", db)}, - {Name: fmt.Sprintf("%s___test_d", db)}, - } ) func init() { diff --git a/tests/e2e/mysql2yt/date_time/check_db_test.go b/tests/e2e/mysql2yt/date_time/check_db_test.go index 519b4f5d..0ec97f01 100644 --- a/tests/e2e/mysql2yt/date_time/check_db_test.go +++ b/tests/e2e/mysql2yt/date_time/check_db_test.go @@ -33,7 +33,6 @@ const ( var ( source = *helpers.WithMysqlInclude(helpers.RecipeMysqlSource(), []string{tableName}) - tablePath = ypath.Path(fmt.Sprintf("//home/cdc/test/mysql2yt/date_time/%s_%s", source.Database, tableName)) targetCluster = os.Getenv("YT_PROXY") ) @@ -86,7 +85,7 @@ func TestDateTime(t *testing.T) { )) }() - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga ctx := context.Background() diff --git a/tests/e2e/pg2ch/dbt/check_db_test.go b/tests/e2e/pg2ch/dbt/check_db_test.go index 278b731f..b2ec4ac2 100644 --- a/tests/e2e/pg2ch/dbt/check_db_test.go +++ b/tests/e2e/pg2ch/dbt/check_db_test.go @@ -20,8 +20,8 @@ import ( func TestSnapshot(t *testing.T) { t.Skip() - _ = os.Setenv("DBT_CONTAINER_REGISTRY", "12197361.preprod") - _ = os.Setenv("DBT_IMAGE_TAG", "public.ecr.aws/t9p9v8b9") + t.Setenv("DBT_CONTAINER_REGISTRY", "12197361.preprod") + t.Setenv("DBT_IMAGE_TAG", "public.ecr.aws/t9p9v8b9") source := pgrecipe.RecipeSource( pgrecipe.WithInitFiles(yatest.SourcePath("transfer_manager/go/tests/e2e/pg2ch/dbt/init_pg.sql")), diff --git a/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go b/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go index 62bc900c..b29be3bd 100644 --- a/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go +++ b/tests/e2e/pg2ch/snapshot_and_replication_canon_types/check_db_test.go @@ -3,7 +3,6 @@ package replication import ( "context" "fmt" - "os" "testing" "time" @@ -25,11 +24,11 @@ var ( ) func TestSnapshotAndIncrement(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga Source := pgrecipe.RecipeSource(pgrecipe.WithPrefix("")) Target := chrecipe.MustTarget(chrecipe.WithInitDir("dump/ch"), chrecipe.WithDatabase(databaseName)) - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga helpers.InitSrcDst(helpers.TransferID, Source, Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable defer func() { require.NoError(t, helpers.CheckConnections( diff --git a/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go b/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go index 1d2a7f62..dbfab40d 100644 --- a/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go +++ b/tests/e2e/pg2kafkamock/debezium_replication/check_db_test.go @@ -43,13 +43,13 @@ func init() { //--------------------------------------------------------------------------------------------------------------------- -// fill 't' by giant random string +// fill 't' by giant random string. var update1Stmt = `UPDATE public.basic_types SET t = 'LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\u003c4SaNJTHkL@1?6YcDf\u003eHI[862bUb4gT@k\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^\u003e\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4\u003cI_@d]\u003eF1e]hj_XJII862[N\u003cj=bYA\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\u003e0UDDBb8h]65C\u003efC\u003c[02jRT]bJ\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\u003eH2X\\]_\u003cEE3@?U2_L67UV8FNQecS2Y=@6\u003ehb1\\3F66UE[W9\u003c]?HH\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJ\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\u003e@A\u003e5\u003cK\\d4QM:7:41B^_c\\FCI=\u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[T\u003cIQI4S_g\u003e;gf[BF_EN\u003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\u003c\u003e[=0W3Of;6;RFY=Q\\OK\\7[\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\u003cOK6gV=EMGC?\\F\u003cXaa_\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;\u003eMZG\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\u003e7cU]M[\u003c72c\u003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:\u003ea5a;j\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\u003c84==_9FJbjbEhQeOV\u003eWDP4MV^W1_]=TeAa66jLObKG\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\u003ebGaJ2f;VB\u003eG\\3\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\u003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\u003ek\u003ePUHD6\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\u003e=\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@f\u003ciDV?6i0WVXj\u003c@ZPd5d\\5B]O?7h=C=8O:L:IR8I\u003e^6\u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLE\u003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\u003e7UBbR58G?[X_O1b\\:[65\u003eP9Z6\u003c]S8=a\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\u003e^GMUPS2]b\u003e]DN?aUKNL^@RV\u003cFTBh:Q[Q3E5VHbK?5=RTKI\u003eggZZ\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\u003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[H\u003cUb4EE^\u003ckWO7\u003eR8fD9JQHR\u003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5\u003cBA\\3IVT@gG\\4\u003cRRS459YROd=_H1OM=a_hd\u003cSMLOd=S6^:eG\u003ejPgQ4_^d\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\u003cDa;\\Ni[AC\u003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9\u003eT12E?FZ9cYCLQbH[2O\u003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihE\u003ehMVaDF\u003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\u003eHga5eW[E8\u003c9jdYO7\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Q\u003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NR\u003eTK07=]7Ecdej\u003cUj\u003cDe1H\u003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_I\u003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\u003cSV@b[GVEU3Xh;R7\u003cXeTNgN\u003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhR\u003e]@GIYf[L55g\u003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\u003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\u003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9He\u003e1L[3\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\u003eCLdAe]6L2AD0aYHc5\u003e=fM7h\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]\u003eKE\u003cea\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\u003eLW5VIfJL:eQ4K3a1^WN5T=\\X=\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1C\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\u003eTQQWYJ_@FAX\\]9jh\u003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6\u003eBgES\u003e5EaeOFeG:i\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\u003eRQ]5Z9jA@Y9V1ZI6TDkC\u003eNZ_f_DR\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\u003eFW\u003cJ6X?IiJ\u003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;f\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHI\u003eI]gBS\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34\u003eh_2@i3kd02G\u003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\u003c^U7Hk]7Q6P:QZS;Ge@:\u003c\u003cfT6PK7j4?;cdC@c5GI:gS[W\u003cf26;\u003cBG7fMXFTWJcbB\\9QT\u003eh3HdV8Pb3Rh\u003e^?Ue:7RP[=jT4AE\u003ebiL_1dYW1\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\u003cA9LXQbECIc2M\u003c^I\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;c\u003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\u003c_F9P\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\u003e=R4U3W1G;\u003chN\\WFO_=DD\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5T\u003eY?bFOMZO\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\u003efaaP8P4KDVSCiQ=2\u003c=Ef:\u003eP\u003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_\u003c@5Z8fDPJAE8DcGUIb8C\u003c_L7XhP=\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\u003e]LW\u003ee^b\u003e?0G9Ie\u003c\u003c@UT4e9\u003cGM_jME7[6TFEN:\u003c\\H\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\u003cL42d\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\u003eEJQi8j;]L5CILgXdR_\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\u003cKHA:\\[CW7SRYVhE1[MD\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8k\u003eQb]UVVZ:18fe_8M?\\?\u003e\u003eLf4QSG@jO@\u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\u003cR]Ofg:TNGW0L\u003ePOC_CP\u003e^PI[aZ:KY^V@Q;;ME_k\\K0\u003eYP]1D5QSc51SfZ]FIP1Y6\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\u003e0]RJGeQiJG5\\=O8TRG5U\u003eLGa\u003eRi2K\u003c3=1TVHN=FhTJYajbIP\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\u003e93HU2ig?7\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1J\u003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^G\u003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1a\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\u003cA\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\u003e@^\u003e[4\u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVde\u003cUVVNH2EJ^=ALOFKUX:^\u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2S\u003cX?9bC7Ebc5V5E]\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SC\u003e\u003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\u003ceM8?j]NZai4\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\u003eI=?@f\u003cG349NMId8[T^@Sf\u003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@d\u003cc\u003cMhS3K;b\u003eZbHAf[GKME9igTY7iVFba\u003e4D;WFVb=dQ4Abj2\u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\u003e@TM9eO\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\u003c@=gPHLhQFDC@:T\u003cREdY\u003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1ad\u003cIiK1O7fbD[7[\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\u003e=FFMHPSBf8:\\XRZ91D:2D[1Y\u003eX\\bfj4BEQZe:1A\u003cQj^@7SAK]C_NCM\\0\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\u003e2\u003e4X:9JYPXk\u003eX_?;DAfL\u003ec?HF\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\u003e1\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aE\u003cY^MPd\u003e6M^iNNe=P6i6Lf::P6ebjX;\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\u003c93\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\u003c8@j5e\u003eVA76=g2=gD4V1eYF0bZd0EZ\u003cMk2M4g[Z=baJ]cVY\u003c[D=U2RUdBNdW=69=8UB4E1@\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;D\u003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4ia\u003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fa\u003e:Vj=BR7EW0_hV4=]DaSeQ\u003c?8]?9X4GbZF41h;FS\u003c9Pa=^SQT\u003cL:GAIP3XX[\\4RKJVLFabj20Oc\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\W\u003cHg9FWd\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\u003eS\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\u003e67\u003eW\u003cQNZRKDH@]_j^M_AV9g4\u003chIF\u003eaSDhbj9GMdjh=F=j:\u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaS\u003eO]caAKi\u003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\u003cWkC\u003c^KSgbI7?aGVaRkbA2?_Raf^\u003e9DID]07\u003cS431;BaRhX:hNJj]\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\u003cN?J\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\u003e8]\u003eU2:HGATaUBPG\u003c\\c0aX@_D;_EOK=]Sjk=1:VGK\u003e=4P^K\\OD\\D008D\u003cgY[GfMjeM\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNf\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\u003ceaB7BV;\u003eIRKK' WHERE i=1;` -// TOASTed update +// TOASTed update. var update2Stmt = `UPDATE public.basic_types SET bl=false WHERE bl=true;` -// update with pkey change +// update with pkey change. var update3Stmt = `UPDATE public.basic_types SET i=2 WHERE i=1;` var deleteStmt = `DELETE FROM public.basic_types WHERE 1=1;` var insertStmt = ` diff --git a/tests/e2e/pg2mock/debezium/debezium_replication/check_db_test.go b/tests/e2e/pg2mock/debezium/debezium_replication/check_db_test.go index 082feffd..3cb63436 100644 --- a/tests/e2e/pg2mock/debezium/debezium_replication/check_db_test.go +++ b/tests/e2e/pg2mock/debezium/debezium_replication/check_db_test.go @@ -34,13 +34,13 @@ func init() { //--------------------------------------------------------------------------------------------------------------------- -// fill 't' by giant random string +// fill 't' by giant random string. var update1Stmt = `UPDATE public.basic_types SET t = 'LidVY09K[5iKehWaIO^A7W;_jaMN^ij\\aUJb^eQdc1^XT?=F3NN[YBZO_=B]\u003c4SaNJTHkL@1?6YcDf\u003eHI[862bUb4gT@k\u003c6NUZfU;;WJ@EBU@P2X@9_B0I94F\\DEhJcS9^=Did^\u003e\u003e4cMTd;d2j;3HD7]6K83ekV2^cF[\\8ii=aKaZVZ\\Ue_1?e_DEfG?f2AYeWIU_GS1\u003c4bfZQWCLKEZE84Z3KiiM@WGf51[LU\\XYTSG:?[VZ4E4\u003cI_@d]\u003eF1e]hj_XJII862[N\u003cj=bYA\u003c]NUQ]NCkeDeWAcKiCcGKjI:LU9YKbkWTMA:?_M?Yb9E816DXM_Vgi7P7a1jXSBi]R^@aL6ja\u003e0UDDBb8h]65C\u003efC\u003c[02jRT]bJ\u003ehI4;IYO]0Ffi812K?h^LX_@Z^bCOY]]V;aaTOFFO\\ALdBODQL729fBcY9;=bhjM8C\\CY7bJHCCZbW@C^BKYTCG]NTTKS6SHJD[8KSQcfdR]Pb5C9P2]cIOE28U\u003eH2X\\]_\u003cEE3@?U2_L67UV8FNQecS2Y=@6\u003ehb1\\3F66UE[W9\u003c]?HH\u003cfi5^Q7L]GR1DI15LG;R1PBXYNKhCcEO^CTRd[3V7UVK3XPO4[55@G]ie=f=5@\\cSEJL5M7\u003c7]X:J=YMh^R=;D;5Q7BUG3NjHhKMJRYQDF\\]SJ?O=a]H:hL[4^EJacJ\u003ee[?KIa__QQGkf=WXUaU6PXdf8[^QiSKXbf6WZe\u003e@A\u003e5\u003cK\\d4QM:7:41B^_c\\FCI=\u003eOehJ7=[EBg3_dTB4[L7\\^ePVVfi48\u003cT2939F]OWYDZM=C_@2@H^2BCYh=W2FcVG1XPFJ428G\\UT4Ie6YBd[T\u003cIQI4S_g\u003e;gf[BF_EN\u003c68:QZ@?09jTEG:^K]QG0\\DfMVAAk_L6gA@M0P\\1YZU37_aRRGiR9BMUh^fgRG2NXBkYb[YPKCSQ8I8Y6@hH]SEPMA7eCURUT@LEi1_ASEI1M7aTG^19FEZcVa]iJDS4S4HR4\u003ccXRAY4HNX_BXiX3XPYMAWhU?0\u003eBH_GUW3;h\\?F?g:QT8=W]DB3k?X??fQWZgAGjLD[[ZjWdP@1]faO@8R?G@NV;4Be0SAk4U[_CZK\u003c\u003e[=0W3Of;6;RFY=Q\\OK\\7[\\\u003cELkX:KeI;7Ib:h]E4hgJU9jFXJ8_:djODj\u003cOK6gV=EMGC?\\F\u003cXaa_\u003cM?DAI=@hQ@95Z?2ELGbcZ6T5AAe77ZCThWeFd;CJJMO9\\QN=hE5WKY\\\\jVc6E;ZBbTX\\_1;\u003eMZG\u003e@eK=?PdZ=UK=@CBUO2gFVU7JUBW713EAiO=DHgR2G^B[6g\u003e7cU]M[\u003c72c\u003e3gSEdHc6\\@2CBI7T9=OGDG16d\\Bk^:\u003ea5a;j\u003e35jC6CUPI=XV]4j9552aG2TQ@JV6UUDXZD0VUE5b2[T6Z];_1;bU\\75H=Z2QG\\eGQP1eUdgEM34?\u003ec4?4fd2i=?W?a3j[JP@LJeDG?aIC6W\u003c:f?5_47]AFIP;LOff3;GN5[dDRBXXicad8fX\u003c1JMGc2RDPM?TXV6]Gj6hB^U@VK:^FbkGAM^9OFM4c\\XPG^B]^H[5;DEa_OU:FTQW6E_U[AYS2G8H:J:hbe22\u003eGd3eM=@7^g=8[bc1PK2gRK61U3cO4e]K^E@2UGPTh@KA0?Cgb^2cH5[g9VYTINiYPS5D8YAH96Y:F26\u003c84==_9FJbjbEhQeOV\u003eWDP4MV^W1_]=TeAa66jLObKG\u003cHg6gRDTfdXHOK4P?]cZ3Z9YBXO]4[:1a7S;ZN4HfSbj87_djNhYC5GU]fGaVQbMXJWGh[_cCVbJ]VD\\9@ILE68[MiF3c[?O8\u003c?f4RRf1CPE4YUN:jCA73^5IaeAR9YE5TIV;CWNd1RRV5]UH2[JcWZ9=cjf=3PVZ[jF\u003ebGaJ2f;VB\u003eG\\3\u003cUZf^g^]bkGVO7TeELB:eD56jGDF8GQ]5LP1?Bc?8?dWENQZjcdd\u003cij;ECQMY7@_Sb7X6?fjf@MLjKDcEPaD[;V@XEHh8k]hbdUg8Pf2aHOccX=HNQ7Y\u003cHFQ_CY_5VVi@R5M8VeVK^N8kfVQ2E]J[B\u003e3038WY6g@;\\]CGXibKLjKFU0Hj]bZ46]48e[akW6:HcMPKW0gUKB@KZ\u003e=QhAWZF_T6US][^;T@j9[V9VAUhP5W_B=\\TdKjX45BWb3J2VZ1JWi5hS2MXYAjg1SLQMPV_\u003cMbUOMDPB^=@c:ceWOThNOi6DJWajBU:_L_Cj9cAg5Q_?IYehBbKaQ:?\u003ek\u003ePUHD6\u003cW5EOFATg5bE^]B5T]fID5XQ4f6ZBJO6ecUA9\u003e=\u003e5R0bc5KVkdi4QP9KVb^5WA;R:_bC24P7UQiNVI8UB7ZcVbCAY6FFGQgQE^dGbINLjMjUf7?=\u003ei5dI:OOQef6aLLTEcK^Fg]cfG^2W0?U59JNCi2dchjXIJA^B\\QYXCQSZDTFDd0J1JhDIi=@f\u003ciDV?6i0WVXj\u003c@ZPd5d\\5B]O?7h=C=8O:L:IR8I\u003e^6\u003ejFgN?1G05Y^ThdQ:=^B\\h^fGE3Taga_A]CP^ZPcHCLE\u003c2OHa9]T49i7iRheH\\;:4[h^@:SAO_D3=9eFfNJ4LQ23MgK\u003e7UBbR58G?[X_O1b\\:[65\u003eP9Z6\u003c]S8=a\u003eb96I==_LhM@LN7=XbC]5cfi7RQ\u003e^GMUPS2]b\u003e]DN?aUKNL^@RV\u003cFTBh:Q[Q3E5VHbK?5=RTKI\u003eggZZ\u003cAEGWiZT8@EYCZ^h6UHE[UgC5EQ1@@ZLQ5d=3Sa;b;c:eV80AOE09AD\u003eVd?f9iGZ3@g5b^@Zi9db_0b5P\u003c5YMHg8B:3K8J:;Z6@QdP@bY9YM:PRY]WG?4CGFMJaVd0S76:kVJbDSPa]5HKb3c67;MMXgCCaC8IJ\u003eSJd2@=U3GeKc\\NZaUeD7R@Kd6^1P=?8V8:fE[H\u003cUb4EE^\u003ckWO7\u003eR8fD9JQHR\u003cP\\7eQbA]L8aaNS2M@QTNF;V@O_[5\u003cBA\\3IVT@gG\\4\u003cRRS459YROd=_H1OM=a_hd\u003cSMLOd=S6^:eG\u003ejPgQ4_^d\u003c_GZ1=Ni6ZQT;5MHXR;aMR4K7k2;_31TK[UX=S^h9G8\u003ecPfK[\\gAHHJST?WUc7EM_R6RO?iWMa;HAf9==jUU_4=IBd3;jHX^j^EN2C:O9EhJ@6WL5A6dECBW\u003cDa;\\Ni[AC\u003eCVGc_\\_=1eeMj;TcOg:;8N1C?PAjaT=9\u003eT12E?FZ9cYCLQbH[2O\u003e4bMT8LJ[XSiAT0VI?18Hdb\\EHS]8UAFY8cB@C[k1CiBgihE\u003ehMVaDF\u003c\\iidT??BG6TWJDWJWU\\TSXiaVKLL_bXPVIIeX[A^Ch=WTWD\u003eHga5eW[E8\u003c9jdYO7\u003eH^iYQAV^i?JAMb=Dg7kWL8dU7]CgAI9Y=7G^H3PFBjW_ad7\\17IM?A7F3JBDcK25RIbjLHE^G0Q\u003ceXie_FG3WNJZh[3;5e^O\\]k96]O7C\\00Yf5Bc\\BK]2NR\u003eTK07=]7Ecdej\u003cUj\u003cDe1H\u003ce91;U^=8DK\\Kc1=jG5b@43f3@?hAW9;:FJgSRA3C6O;7\\9Na1^d4YgDgdUS2_I\u003c:c8^JIa]NEgU558f6f:S\\MPU78WfPc5HkcbHYSf3OP8UX3[Scd;TG[\u003eNcfIH]N]FW:4?57_U?HCB8e:16^Ha2eYhC6ZagL\u003cSV@b[GVEU3Xh;R7\u003cXeTNgN\u003cdaBSW=3dY9WIOB^:EK6P2=\\Z7E=3cIgYZOFhR\u003e]@GIYf[L55g\u003cUiIFXP[eTSCPA23WjUf\\eB:S=f3BkjNUhgjULZN5BaTScX?bB:S\u003cK^_XXbkXaNB^JAHfkfjA\\SdT@8KRB3^]aRJNIJ;@hL3F]JA]E@46chZ85:ZG\u003eM934TQN3\\]k=Fk?W]Tg[_]JhcUW?b9He\u003e1L[3\u003cM3JBIIQ5;:11e^D]UiIdRAZA;PEG2HaD@feK5fKj[\u003eCLdAe]6L2AD0aYHc5\u003e=fM7h\u003cZI;JWOfPAfAD[QX[GE8?JFLEcS9_d\u003ejBeN=JB2[=B4hd[X@5_OP:jd2R3bFf5E=kbKI:L9F_=CXijg3_KSiJL01ObGJh\\WgS7F]TO8G\\K4ZJ0]\u003eKE\u003cea\u003cfE3B_03KgVRBG;aORRjVAIV3W6Hc0=4gR7\u003eF7Aa3fHECR;b9]a_3?K5eQM]Q[aMBh[W40M7feM\u003eLW5VIfJL:eQ4K3a1^WN5T=\\X=\u003e_98AGUhM?FHYbRSIV3LL4?8RD\\_5H1C\u003c:LMQ5J3DaK3X1V6WYR8]a@D:17?I9SVC38d8RgLHGO5H:;4c]=USMi]N52g\u003eTQQWYJ_@FAX\\]9jh\u003ebZKLBhJ4JO6F]ZhBFV\\;f6KSc@F1?B?61ZSCW1H6PNLB=ITS4E^jK\u003eSCOhD^@SdABLTiM142NPD[igD2A71\\ET4dQGWajP7A0[?M\\CO?ccja_Cc5Jda_NeX4ACeAc1Rc\\aFM9e\\1][bR3ZWMTM@6Gh:X@4i85P1aGGBPA3Q3^HUa7ABZ^Sa:Pkb4h8Fii\\E@AUCbX6\u003eBgES\u003e5EaeOFeG:i\u003c86R54CJDT4XJ]^Y4Z3Vi80_2P9ggDe8KjZQ32kHU444b]dROOhPCj4Lf0_8@_bbd?NdCRY;DR\\96@5VS4Z4jZc^c8QZhHR]W5VkWD:0fg91\u003c?V_CEcA5[4gcVVa3=SZB=ZiQeiL7M1F8XMXjRI3NAX97[EZKWg:UM3RidYKe4SZ]6H[Xa^;7KC=\u003cYgVEcjFcQD\\?_VDGE5M]:SSDY4Xg@Fcf[[[Y6T?JDO\u003ejbUEg77]AYEUGIBCXX;SGfC50gDJ@cX@ZBTVI[HZI]D;V8cCCLZ=__\u003e[9X01E@[WeF5T_2Q9c\\kT7B5bPdV^T_JT__dOK^eQGYEJ?OAjCASKSXA8Qgf9[E^O9W3UJh:aVP@e3QdGbMaK:8S[4Nd^cVB1BEV\\BSiEbcHI\\_@\u003eU[H]C70SXWeYi?DZQ9BON9GfR8YbFCR^5eeeZfNGQH5OWI?\u003eRQ]5Z9jA@Y9V1ZI6TDkC\u003eNZ_f_DR\u003eS8QecZd9jRAVS14YUHYhV;WJ6K^XYFLNN2HF\\BO[dFLaJ9KbbHL24g8OZ=4A[SC8h4JLCA;^7UhRL_jha3diRR^_W3O\u003eFW\u003cJ6X?IiJ\u003c549XOhWM^ZE\\@hO4TRSbh?3GE[V]Y5i^97KY47:baOS6L7:5X\\gUkj1DZX7H]5;f\u003cWT@^^8SB[Y_acdNT8T_:iNb4eT:6OF]8VOf^8=Ma1CYdbBYjgM9ejkieS8k8M\\@9@;gHHI\u003eI]gBS\u003e0R:M[4L[2FC9EKW6[Ge[_B91[fh2N;36EPaI1QKGdT\\D?b34\u003eh_2@i3kd02G\u003c5MQUCjUcI1\\2]4BT8Ec5:eD7hDkhFG9KdZ5;YZ38[_:MdK70aj5jcJ7^6]:MfUFUZQDIUK:IUWB5^Bf]HfUb1JU8\u003c^U7Hk]7Q6P:QZS;Ge@:\u003c\u003cfT6PK7j4?;cdC@c5GI:gS[W\u003cf26;\u003cBG7fMXFTWJcbB\\9QT\u003eh3HdV8Pb3Rh\u003e^?Ue:7RP[=jT4AE\u003ebiL_1dYW1\u003eM4JCSYhMc44H_AGHEX]SO[3C[g1Gi?e24DDV2A8dE\u003cA9LXQbECIc2M\u003c^I\u003c:GK4IOG]:I3BCHNTQjA7aUJ?NL\\Y?:fIPFMied[4B^FU;c\u003e\\bNcX9AgW]WE1a@JFVgDPa4S8bi]2ak]XNUEWfACXhXY^h9:S5N8eR[2IY_JO_==BbRi]cAJh8TeA^MFAU@cEB@36[Reh_\u003c_F9P\u003eJj3G8WAHJ_^ZH3R]EbKRGEO;PCPZc^9baPjMaHfU;V2\u003e=R4U3W1G;\u003chN\\WFO_=DD\u003ca:T]_^Gb1TVSX@VDA2OMj2=VG\\JU6^agiJY]=5T\u003eY?bFOMZO\u003eBO@O:W@TAFG7BEQj7^4[1]jc9NEcCd7UHG9Q3J:DQK6f162_:]ag\\Y5?3iRg4\u003cDKEeN_4bSUBZPC_R8iCie4WkCZhdV15iLJcj\u003efaaP8P4KDVSCiQ=2\u003c=Ef:\u003eP\u003cDNX^FW1AMcaVHe6\\PY4N?AQKNeFX9fcLIP?_\u003c@5Z8fDPJAE8DcGUIb8C\u003c_L7XhP=\u003cDILI8TDL99fIN3^FIH_@P8LDSS1Q8\u003e]LW\u003ee^b\u003e?0G9Ie\u003c\u003c@UT4e9\u003cGM_jME7[6TFEN:\u003c\\H\u003c8RU2]aBHJFBSRY5FXR[_BbHY;ebGV?S^a=S470NNB650;KX]\u003cL42d\\\u003e^SUJc==XJ3AN:A1XS7]TB=A3I]7KVcYJLCcCO61j8AMCRNk:U\\^gi4kGa7bMjPfKc_^Ge^F25cEWFDa06Tg4XgKN3Ck2cfMZZ?6S3LU8Cj^YCTYI=UMeQhHT?HV7C7a1GgUJH?Q[\u003eEJQi8j;]L5CILgXdR_\u003cYU=5RbOj65ZEJ9fGAeR3FWF_8CL1e@=SfJXLA\u003cKHA:\\[CW7SRYVhE1[MD\u003cN=M[G:NdKZDckNTZAaIbP4_d5OFI\\cV=SLT]iM=Xa5XCZG8k\u003eQb]UVVZ:18fe_8M?\\?\u003e\u003eLf4QSG@jO@\u003c57iZ]UIgVRaOEi1UZ@ch\\]1BEHSDgcP1iN\\[8:W^\\NB6LCZ;SR9CD:VYR=2N5RO35@_=JKk;iA@ITkU\u003cR]Ofg:TNGW0L\u003ePOC_CP\u003e^PI[aZ:KY^V@Q;;ME_k\\K0\u003eYP]1D5QSc51SfZ]FIP1Y6\u003cdRQXRC8RP7BaKGG2?L3bG]S];8_d\u003e0]RJGeQiJG5\\=O8TRG5U\u003eLGa\u003eRi2K\u003c3=1TVHN=FhTJYajbIP\u003eN:LjQB=9@@TLBaLfLdIY?FBY57XfQ\u003e93HU2ig?7\u003cO[WaP9]12;ZAQ1kV8XQYeZ\\BD_@@3GLR78HWA:YCEHTfITQQ@7?;b1M;_]Kc9gJ@4bgD1UWF2@AKdb29iADBak6SKi\\FG1J\u003eh^?RKUT[e4T\\6]ZG6OXgN_Oi\\@D8A^G\u003eQVa1?J\\:NDfT7U0=9Y9WLYU=iiF?\\]MBGCCW]3@H[eNEe[MSe94R^AP\\W_MHB_U7LG:AWR1Q5FKc2Z16A_GaQ3U2Kga@Qh\\h71TY29]HTS@VBA\\S68IV;4YVkOfQLVMSX6AZ?37cVFNgX?O]GhIQ16\u003c1U7Q6]3ZI9j8H2?@XU^TB284I6Mj7S;7=BYD4\\3Me2UC4dS\\NFEIMdbSFaZi1a\u003cCOPG@Re;TOMXH5IfK^[d@U[ckQRiRH:fgZB\u003cA\u003cGe[dR8ik3J]^C3H2fHSMF;eP6b?H3PSJICC0JAkMZ]@2X5[5X=Lc71hi@E1iK\u003e@^\u003e[4\u003e=^kM;eO@R\\\\Id]Gb2\\cbYC5j5CZ9QggPI\\ETVde\u003cUVVNH2EJ^=ALOFKUX:^\u003e5Z^NK88511BWWh:4iNN\\[_=?:XdbaW5fEcJ0Rf2S\u003cX?9bC7Ebc5V5E]\u003eWSe]N?Uh4UOjW7;DED;YKPODU:Hjj:=V]7H@F2=JW\\ICcTX=hbfHGJ\\2T91SC\u003e\u003e5EVE[XS:DDRX;;DH8;CPS\\ATEJUh]c;b=a=gN_6b8XOCcc[k33PV_?:?d71\\Bdi85eVdkM1X0DQc5Pf85Qge6:Y\u003c;JN3GV8A@2A]3i]GOUL4PS:6O4eU=SaH1DKIjTZ?U01Xi^4MHPRh8[3W_hA2P7JQKejJNYY8YZaWNe:fJ[cRLf?@cPBHW[i7VhQ9V?ACi7kL19GKe?3E:AU2agJMWHTBD:KjI\\CHcBddL@DEOF[YXE[NA:0hQT?f_Ze=K=UBON;j]OEAf4jRIZ5Zc5WJZfENU?[5KEGjbRjT6Ce1HdSaSYPK^\u003ceM8?j]NZai4\u003ehfgOf?JgWCPMe=2E0??MFNL81;ij?\u003cg:1cYg78d^KH?EVB[VPj8gMT4N_2M3\u003eI=?@f\u003cG349NMId8[T^@Sf\u003c5O?SCB5FPNS_^Ok:R4C6Q\\iXLRK\\:Eg@d\u003cc\u003cMhS3K;b\u003eZbHAf[GKME9igTY7iVFba\u003e4D;WFVb=dQ4Abj2\u003eJNSSLP;:V:11V?5jK\\E6SRj8V@kUB=4aaVBEbL11A22gA6f\\b@bJbaRM7R7I_;?UaPjX1kXB2Z\u003eC94WIf6@]X]c?dA24PWe5VR6V?HWiVj__3K=iQM[\u003e@TM9eO\u003cJ;6OaXVLg38eZ7XN:8[8Y=cgMLIVFhb8hEjTjJP3RJ\\Y7?c?k0h=deZECE[@;PH8eG]daBgI[X6bhi6gj49bhc\u003c@=gPHLhQFDC@:T\u003cREdY\u003caWB]VFgMC_YS1U7J64jMHB\\Rfh9@abLWN^I99EVL9E4:j;S5?SRWeC=?F55=Q\\\\D:eMNPiWe1ad\u003cIiK1O7fbD[7[\u003chEhYY6S;T88@2:6eFOcaPGiK?B;E1kQiENW3T?\u003e=FFMHPSBf8:\\XRZ91D:2D[1Y\u003eX\\bfj4BEQZe:1A\u003cQj^@7SAK]C_NCM\\0\u003eSf=V=Q=gKFi@W:aVg6]OF=BY1_1NP2[8hh^:Nk6iF4\u003e2\u003e4X:9JYPXk\u003eX_?;DAfL\u003ec?HF\u003eNETRSWWDj^XEKXR8LaC7?@E7O\\M]@bGbJ2W6FVf:C?U0b]LX6@_EP9K4ehb:_\u003e1\u003e@XDWD?WNJWE=82CHaWhj82d5d2d648F\\K25Zb\\=BHROPTbhJNeHVgA[_CTfG\\A8\u003cC=f:i8LFZ0fCbc]D]:jYKZM_CH;3YC@1O;\u003cMCXc2X^EOV7cHAb6\\QTPc1ZgZ2;\\RFh4YUg[BZ5aE\u003cY^MPd\u003e6M^iNNe=P6i6Lf::P6ebjX;\u003cFhYfag1CZka=e3]k1cLg2VL8PCiPj9[E6IAgEB@4B6A\u003c93\u003c:fX5iCQ6cd4Hc=8=CQN?fOk6TAB]DNg@:1\u003eMRDEKH]CUePgK3;FcZFiDW@61^1@h2NJTb_4?QGcKggk0BcZXa3D69Ed:Ua\u003c8@j5e\u003eVA76=g2=gD4V1eYF0bZd0EZ\u003cMk2M4g[Z=baJ]cVY\u003c[D=U2RUdBNdW=69=8UB4E1@\u003cbZiYEWe507Y3YCfkaV4f_A2IR6_TFkJ5i9JU2OV9=XbPTaFILJC@[FZBLMfbMEgKNF6Pe[Y7IOW2F3JbM^7=8aOTCJK_G@A]FaV6O]O4JPIMk@i]H;f\u003eZOQ8jFgEV=703^6RPUVj:4K:DJg\\UbjDEOLDeHZOUaPXSV@8@f7JjSTC2P4WG3j\\RK5Lc_0MUP:=;JFJDMdC5MV72[]I]\\;D\u003c@44QYE[fO:AjN^cbcEMjH=\\ajM1CZA8^EhD3B4ia\u003e?\\2XSf25dJAU@@7ASaQ\\TfYghk0fa\u003e:Vj=BR7EW0_hV4=]DaSeQ\u003c?8]?9X4GbZF41h;FS\u003c9Pa=^SQT\u003cL:GAIP3XX[\\4RKJVLFabj20Oc\u003eBK_fW?53PNSS;ABgDeG^Pc9FZ8HZW@gi[[cGkhKPK37UCJQXDgKc_T?M\\W\u003cHg9FWd\u003e4d;NHVQP@ejaQB]1;QVI3G5@_1H:XAH[:S\u003eS\u003e7NY6C@H5ASVg1ZC6i76GA^XYNbA]JNQR1?XDO5IX4\\Y^4_\\:e8KX9;XIh7hNXh]EAAJZ66_b_RfSC5MKP:@YEg7A34_[1Q5BbN2hUIGZ1ZM9EWI30E:BH\u003e67\u003eW\u003cQNZRKDH@]_j^M_AV9g4\u003chIF\u003eaSDhbj9GMdjh=F=j:\u003c^Wj3C8jGDgY;VBOS8N\\P0UNhbe:a4FT[EW2MVIaS\u003eO]caAKi\u003cNa1]WfgMiB6YW]\\9H:jjHN]@D3[BcgX\\aJI\\FfZY1HE]9N:CL:ZjgjCjZUbVJNG?h0DZZ1[8FNAcXTEbCD^BW\\1ASW[63j3bjGRZHBb]8VM[jC3C6EjcF@K20Q5jTgikNXHN:TV6F_II8P^7G9Hb;HG@G1;E0Y2HNPR7;G=R\u003cWkC\u003c^KSgbI7?aGVaRkbA2?_Raf^\u003e9DID]07\u003cS431;BaRhX:hNJj]\u003eQS9DaBY?62169=Y=AZHSPkP=9M[TLMb36kGgB4;H6\u003cN?J\u003cLZfeCKdcX2EHVbeMd0M@g^E7;KDYZ]e;M5_?iWg01DWc\u003e8]\u003eU2:HGATaUBPG\u003c\\c0aX@_D;_EOK=]Sjk=1:VGK\u003e=4P^K\\OD\\D008D\u003cgY[GfMjeM\u003cfVbB65O:UBVEai6:j6BCB=02TgOSa1_[WU2]ZRhDdRYYQ_cOf:b=Gb?0^^ST_FDK0F=Zh93\\\\OAQGLQWYhNhhAZPeNf\u003eifT:UPDYF4JdF0@;Lab9]F6ZW?QC:^A5GKZg_HBcb;\u003ebKICA@L3VQ^BG2cZ;Vj@3Jjj\u003eFA6=LD4g]G=3c@YI305cO@ONPQhNP\u003ceaB7BV;\u003eIRKK' WHERE i=1;` -// TOASTed update +// TOASTed update. var update2Stmt = `UPDATE public.basic_types SET bl=false WHERE bl=true;` -// update with pkey change +// update with pkey change. var update3Stmt = `UPDATE public.basic_types SET i=2 WHERE i=1;` var deleteStmt = `DELETE FROM public.basic_types WHERE 1=1;` var insertStmt = ` diff --git a/tests/e2e/pg2mock/retry_conn_leak/check_db_test.go b/tests/e2e/pg2mock/retry_conn_leak/check_db_test.go index faaf89be..93d777f3 100644 --- a/tests/e2e/pg2mock/retry_conn_leak/check_db_test.go +++ b/tests/e2e/pg2mock/retry_conn_leak/check_db_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "os" "sync" "testing" "time" @@ -35,7 +34,7 @@ func (s *mockSinker) Push(input []abstract.ChangeItem) error { } func TestReplication(t *testing.T) { - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga sinker := &mockSinker{} transfer := model.Transfer{ ID: "test_id", diff --git a/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go b/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go index 47e4001e..45697745 100644 --- a/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go +++ b/tests/e2e/pg2pg/partitioned_tables/some_parts/partitioned_tables_test.go @@ -3,7 +3,6 @@ package replication import ( "context" "os" - "strconv" "testing" "time" @@ -29,8 +28,7 @@ var ( ), pgrecipe.WithEdit(func(pg *postgres.PgSource) { pg.UseFakePrimaryKey = true })) - dstPort, _ = strconv.Atoi(os.Getenv("DB0_PG_LOCAL_PORT")) - Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) + Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) ) func init() { @@ -178,24 +176,24 @@ func Load(t *testing.T) { compareParams := helpers.NewCompareStorageParams() compareParams.TableFilter = func(tables abstract.TableMap) []abstract.TableDescription { return []abstract.TableDescription{ - abstract.TableDescription{ + { Name: "measurement_inherited", Schema: "public", }, - abstract.TableDescription{ + { Name: "measurement_inherited_y2006m02", Schema: "public", }, - abstract.TableDescription{ + { Name: "measurement_inherited_y2006m04", Schema: "public", }, - //skip measurement_declarative because of turned UseFakePrimaryKey option on (limitation of outdated 10.5 PG version) - abstract.TableDescription{ + // skip measurement_declarative because of turned UseFakePrimaryKey option on (limitation of outdated 10.5 PG version) + { Name: "measurement_declarative_y2006m02", Schema: "public", }, - abstract.TableDescription{ + { Name: "measurement_declarative_y2006m04", Schema: "public", }, diff --git a/tests/e2e/pg2pg/replication/check_db_test.go b/tests/e2e/pg2pg/replication/check_db_test.go index 8a3c5c45..18f96c5b 100644 --- a/tests/e2e/pg2pg/replication/check_db_test.go +++ b/tests/e2e/pg2pg/replication/check_db_test.go @@ -3,7 +3,6 @@ package replication import ( "context" "os" - "strconv" "testing" "time" @@ -21,7 +20,6 @@ import ( var ( TransferType = abstract.TransferTypeSnapshotAndIncrement Source = *pgrecipe.RecipeSource(pgrecipe.WithInitDir("dump"), pgrecipe.WithPrefix("")) - dstPort, _ = strconv.Atoi(os.Getenv("DB0_PG_LOCAL_PORT")) Target = *pgrecipe.RecipeTarget(pgrecipe.WithPrefix("DB0_")) ) diff --git a/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go b/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go index 99297abf..81f30552 100644 --- a/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go +++ b/tests/e2e/pg2pg/snapshot_missing_public/check_db_test.go @@ -62,7 +62,7 @@ func Snapshot(t *testing.T) { require.False(t, exists) } -// CheckTableExistence is a helper function for PostgreSQL to check existence of the given table +// CheckTableExistence is a helper function for PostgreSQL to check existence of the given table. func CheckTableExistence(ctx context.Context, conn *pgxpool.Pool, tableSchema string, tableName string) (bool, error) { var result bool err := conn.QueryRow(context.Background(), ` diff --git a/tests/e2e/pg2ydb/replication_toasted/check_db_test.go b/tests/e2e/pg2ydb/replication_toasted/check_db_test.go index e91e789f..8fcf8fcf 100644 --- a/tests/e2e/pg2ydb/replication_toasted/check_db_test.go +++ b/tests/e2e/pg2ydb/replication_toasted/check_db_test.go @@ -31,7 +31,7 @@ func TestSnapshotAndIncrement(t *testing.T) { Instance: helpers.GetEnvOfFail(t, "YDB_ENDPOINT"), } - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga helpers.InitSrcDst(helpers.TransferID, Source, Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable defer func() { diff --git a/tests/e2e/pg2ydb/snapshot_replication_pk_update/check_db_test.go b/tests/e2e/pg2ydb/snapshot_replication_pk_update/check_db_test.go index 678c4354..6012be60 100644 --- a/tests/e2e/pg2ydb/snapshot_replication_pk_update/check_db_test.go +++ b/tests/e2e/pg2ydb/snapshot_replication_pk_update/check_db_test.go @@ -32,7 +32,7 @@ func TestSnapshotAndIncrement(t *testing.T) { Instance: helpers.GetEnvOfFail(t, "YDB_ENDPOINT"), } - _ = os.Setenv("YC", "1") // to not go to vanga + t.Setenv("YC", "1") // to not go to vanga helpers.InitSrcDst(helpers.TransferID, Source, Target, TransferType) // to WithDefaults() & FillDependentFields(): IsHomo, helpers.TransferID, IsUpdateable defer func() { diff --git a/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator.go b/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator.go index 01501aa2..eb8e3845 100644 --- a/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator.go +++ b/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator.go @@ -8,15 +8,15 @@ import ( // FCoalgebra is one JSON node builder. // F-coalgeras accepting zero amount of childs are primitive JSONs (constructors) // arbitrary F-coalgebra may constructs JSON with N childs passed as strings -// if it unable to do it, it should panic +// if it unable to do it, it should panic. type FCoalgebra func([]string) string // GenerationRules is just a map of child count to F-coalgebras which accepts that count of childs -// you should care about arity of F-coalgebras +// you should care about arity of F-coalgebras. type GenerationRules map[uint][]FCoalgebra var ( - // see example of default generation rules + // see example of default generation rules. DefaultGenerationRules GenerationRules = map[uint][]FCoalgebra{ 0: {numberConstructor, stringConstructor, arrayConstructor}, 1: {fCoalgebraJSONA, fCoalgebraJSONB, arrayConstructor}, @@ -24,7 +24,7 @@ var ( 3: {arrayConstructor}, } - // this rules do not generate collisions like 0 == "0" + // this rules do not generate collisions like 0 == "0". WithoutCollisionGenerationRules GenerationRules = map[uint][]FCoalgebra{ 0: {numberConstructor, arrayConstructor}, 1: {fCoalgebraJSONB, arrayConstructor}, @@ -43,7 +43,7 @@ func checkLength(ch []string, l int) { } } -// F-coalgebras +// F-coalgebras. func numberConstructor(nest []string) string { checkLength(nest, 0) return "0" @@ -76,7 +76,7 @@ func fCoalgebraJSONC(nest []string) string { // // note, that result length of array may differ with 'count' parameter, because of not having zero-arity constructors, // or not having non-zero-arity constructors -// the algorithm is deterministic with no side effects +// the algorithm is deterministic with no side effects. func (j *JSONGenerator) generateSequence(count int) []string { rules := j.generationRules if rules == nil { @@ -161,7 +161,7 @@ func (j *JSONGenerator) generateSequence(count int) []string { } // NewJSONGenerator creates JSONGenerator with generation rules -// if generationRules rules are nil, default will be used +// if generationRules rules are nil, default will be used. func NewJSONGenerator(generationRules GenerationRules) *JSONGenerator { return &JSONGenerator{ generationRules: generationRules, diff --git a/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator_test.go b/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator_test.go index 22be4de4..c3b4a77b 100644 --- a/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator_test.go +++ b/tests/e2e/pg2yt/bulk_jsonb_pkey/bulk_json_generator_test.go @@ -37,6 +37,7 @@ func TestMain(m *testing.M) { } func TestRunner(t *testing.T) { + t.Parallel() t.Run("CheckSimpleValidity", checkSimpleValidity) t.Run("CheckGeneratorSequenceUniqueness", checkGeneratorSequenceUniqueness) t.Run("CheckGeneratorIsDeterministic", checkGeneratorIsDeterministic) @@ -44,11 +45,11 @@ func TestRunner(t *testing.T) { targetPort, err := helpers.GetPortFromStr(ytDest.Cluster()) require.NoError(t, err) - defer func() { + t.Cleanup(func() { require.NoError(t, helpers.CheckConnections( helpers.LabeledPort{Label: "YT target", Port: targetPort}, )) - }() + }) defer cancel() t.Run("PumpDatabaseToYt_NoCollisions", testFactoryPumpDatabaseToYt(ytDest, "bulk_jsonb_no_collision", WithoutCollisionGenerationRules)) @@ -77,7 +78,7 @@ type ValuesWithKind struct { kind abstract.Kind } -// Haskell-like map :: (a -> b) -> [a] -> [b] +// Haskell-like map :: (a -> b) -> [a] -> [b]. func mapValuesToChangeItems(f func(ValuesWithKind) abstract.ChangeItem) func([]ValuesWithKind) []abstract.ChangeItem { return func(vwk []ValuesWithKind) []abstract.ChangeItem { var result []abstract.ChangeItem @@ -103,7 +104,7 @@ func teardown(env *yttest.Env, p string) { } // initializes YT client and sinker config -// do not forget to call testTeardown when resources are not needed anymore +// do not forget to call testTeardown when resources are not needed anymore. func initYt(t *testing.T) (testEnv *yttest.Env, testCfg yt_provider.YtDestinationModel, testTeardown func()) { env, cancel := yttest.NewEnv(t) cypressPath := "//home/cdc/test/TM-1893" @@ -120,7 +121,7 @@ var whoMakesJSONbAsKeyQuestionMarkSchema = abstract.NewTableSchema([]abstract.Co {DataType: ytschema.TypeInt32.String(), ColumnName: "value"}, }) -// absolutely unreadable..... but this thing is a factory depending on pgSource that generates function that accepts only values and kinds and returns appropriate change items +// absolutely unreadable..... but this thing is a factory depending on pgSource that generates function that accepts only values and kinds and returns appropriate change items. func whoMakesJSONbAsKeyQuestionMarkItemGenerator(schema, table string) func([]ValuesWithKind) []abstract.ChangeItem { return mapValuesToChangeItems(func(vwk ValuesWithKind) abstract.ChangeItem { return abstract.ChangeItem{ diff --git a/tests/e2e/pg2yt/cdc_partial_activate/check_db_test.go b/tests/e2e/pg2yt/cdc_partial_activate/check_db_test.go index 3b53b50d..67cbe6f7 100644 --- a/tests/e2e/pg2yt/cdc_partial_activate/check_db_test.go +++ b/tests/e2e/pg2yt/cdc_partial_activate/check_db_test.go @@ -18,13 +18,14 @@ import ( ) var ( - srcPort = helpers.GetIntFromEnv("PG_LOCAL_PORT") - Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithDBTables("public.__test")) - Target = yt_helpers.RecipeYtTarget("//home/cdc/test/pg2yt_e2e") + Source = *pgrecipe.RecipeSource(pgrecipe.WithPrefix(""), pgrecipe.WithInitDir("dump"), pgrecipe.WithDBTables("public.__test")) + Target = yt_helpers.RecipeYtTarget("//home/cdc/test/pg2yt_e2e") ) -const CursorField = "id" -const CursorValue = "5" +const ( + CursorField = "id" + CursorValue = "5" +) func init() { _ = os.Setenv("YC", "1") // to not go to vanga @@ -78,11 +79,11 @@ func Load(t *testing.T) { ] } `)) - //start cdc + // start cdc worker := helpers.Activate(t, transfer) require.NotNil(t, worker, "Transfer is not activated") - //check snapshot loaded + // check snapshot loaded conn, err := pgcommon.MakeConnPoolFromSrc(&Source, logger.Log) require.NoError(t, err) @@ -93,12 +94,12 @@ func Load(t *testing.T) { require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "__test", storage, 60*time.Second, expectedYtRows), "Wrong row number after first snapshot round!") - //add some data to pg + // add some data to pg expectedYtRows = addSomeDataAndGetExpectedCount(t, conn) require.NoError(t, helpers.WaitDestinationEqualRowsCount("public", "__test", helpers.GetSampleableStorageByModel(t, Target.LegacyModel()), 60*time.Second, expectedYtRows)) worker.Close(t) - //read data from target + // read data from target require.NoError(t, storage.LoadTable(context.Background(), abstract.TableDescription{ Name: "__test", Schema: "", @@ -144,7 +145,7 @@ func addSomeDataAndGetExpectedCount(t *testing.T, conn *pgxpool.Pool) uint64 { extraItems += 3 _, err = conn.Exec(context.Background(), "delete from __test where str='rrr' or str='eee';") require.NoError(t, err) - extraItems += 2 //item before deletion + deleted event + extraItems += 2 // item before deletion + deleted event return currentDBRows + extraItems } diff --git a/tests/e2e/pg2yt/enum/enum_join_test.go b/tests/e2e/pg2yt/enum/enum_join_test.go index ce0f0a6d..e1a4d5b3 100644 --- a/tests/e2e/pg2yt/enum/enum_join_test.go +++ b/tests/e2e/pg2yt/enum/enum_join_test.go @@ -60,7 +60,7 @@ func teardown(env *yttest.Env, p string) { } // initializes YT client and sinker config -// do not forget to call testTeardown when resources are not needed anymore +// do not forget to call testTeardown when resources are not needed anymore. func initYt(t *testing.T, cypressPath string) (testEnv *yttest.Env, testCfg yt_provider.YtDestinationModel, testTeardown func()) { env, cancel := yttest.NewEnv(t) cfg := yt_helpers.RecipeYtTarget(cypressPath) diff --git a/tests/e2e/pg2yt/sql_transformer/check_db_test.go b/tests/e2e/pg2yt/sql_transformer/check_db_test.go index f49610d7..97e6bd59 100644 --- a/tests/e2e/pg2yt/sql_transformer/check_db_test.go +++ b/tests/e2e/pg2yt/sql_transformer/check_db_test.go @@ -56,7 +56,7 @@ func TestGroup(t *testing.T) { } func Load(t *testing.T) { - _ = os.Setenv("CH_LOCAL_PATH", os.Getenv("RECIPE_CLICKHOUSE_BIN")) + t.Setenv("CH_LOCAL_PATH", os.Getenv("RECIPE_CLICKHOUSE_BIN")) transfer := helpers.MakeTransfer(helpers.TransferID, &Source, Target, abstract.TransferTypeSnapshotOnly) require.NoError(t, transfer.TransformationFromJSON(` diff --git a/tests/e2e/ydb2mock/incremental/check_db_test.go b/tests/e2e/ydb2mock/incremental/check_db_test.go index b8f5067c..4af5c411 100644 --- a/tests/e2e/ydb2mock/incremental/check_db_test.go +++ b/tests/e2e/ydb2mock/incremental/check_db_test.go @@ -122,7 +122,7 @@ func TestYDBIncrementalSnapshot(t *testing.T) { } } -// checkRows checks whether rows contain unique rows numbered from expectedFrom to expectedTo +// checkRows checks whether rows contain unique rows numbered from expectedFrom to expectedTo. func checkRows(t *testing.T, rows []abstract.ChangeItem, expectedFrom, expectedTo int64) { require.Len(t, rows, int(expectedTo-expectedFrom)) diff --git a/tests/e2e/ydb2ydb/debezium/snapshot_serde_via_debezium_embedded_olap/check_db_test.go b/tests/e2e/ydb2ydb/debezium/snapshot_serde_via_debezium_embedded_olap/check_db_test.go index 9fa9ff20..e3769ff9 100644 --- a/tests/e2e/ydb2ydb/debezium/snapshot_serde_via_debezium_embedded_olap/check_db_test.go +++ b/tests/e2e/ydb2ydb/debezium/snapshot_serde_via_debezium_embedded_olap/check_db_test.go @@ -21,9 +21,11 @@ import ( "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" ) -var path = "dectest/timmyb32r-test" -var pathOut = "dectest/timmyb32r-test-out" -var sourceChangeItem abstract.ChangeItem +var ( + path = "dectest/timmyb32r-test" + pathOut = "dectest/timmyb32r-test-out" + sourceChangeItem abstract.ChangeItem +) func TestSnapshotSerDeViaDebeziumEmbeddedOLAP(t *testing.T) { src := &ydb.YdbSource{ @@ -97,7 +99,7 @@ func TestSnapshotSerDeViaDebeziumEmbeddedOLAP(t *testing.T) { } var count uint64 for res.NextRow() { - err = res.ScanNamed(named.Required("co", &count), named.Required("bo", &foundInOlap)) + _ = res.ScanNamed(named.Required("co", &count), named.Required("bo", &foundInOlap)) } require.Equal(t, uint64(1), count) return res.Err() diff --git a/tests/helpers/activate_delivery_wrapper.go b/tests/helpers/activate_delivery_wrapper.go index d5d9af28..87988183 100644 --- a/tests/helpers/activate_delivery_wrapper.go +++ b/tests/helpers/activate_delivery_wrapper.go @@ -21,7 +21,7 @@ type Worker struct { cp coordinator.Coordinator } -// controlplane that catches replication failure +// controlplane that catches replication failure. type fakeCpErrRepl struct { coordinator.Coordinator onErrorCallback []func(err error) @@ -44,7 +44,7 @@ func (q *Worker) Close(t *testing.T) { } } -// Restart replication worker with updated transfer +// Restart replication worker with updated transfer. func (q *Worker) Restart(t *testing.T, transfer *model.Transfer) { q.Close(t) q.initLocalWorker(transfer) diff --git a/tests/helpers/confluent_schema_registry_mock/endpoint_matcher.go b/tests/helpers/confluent_schema_registry_mock/endpoint_matcher.go index d9235b22..1dd06dbc 100644 --- a/tests/helpers/confluent_schema_registry_mock/endpoint_matcher.go +++ b/tests/helpers/confluent_schema_registry_mock/endpoint_matcher.go @@ -29,5 +29,5 @@ type schema struct { var reSchema = regexp.MustCompile(`^/subjects/(.*)/versions/([0-9]+)$`) func (s schema) IsMatched(url string) bool { - return reSchema.Match([]byte(url)) + return reSchema.MatchString(url) } diff --git a/tests/helpers/debezium_pg_array_comparator.go b/tests/helpers/debezium_pg_array_comparator.go index f983336a..2fb8b5d3 100644 --- a/tests/helpers/debezium_pg_array_comparator.go +++ b/tests/helpers/debezium_pg_array_comparator.go @@ -40,7 +40,7 @@ func PgDebeziumIgnoreTemporalAccuracyForArraysComparator(lVal interface{}, lSche return false, false, nil } -// TimeWithPrecision takes the time in format `01:02:03[.123456]` and returns it with the given precision +// TimeWithPrecision takes the time in format `01:02:03[.123456]` and returns it with the given precision. func TimeWithPrecision(t string, precision int) string { withoutFractions := t[:8] fractions := t[8:] diff --git a/tests/helpers/http_proxy/proxy.go b/tests/helpers/http_proxy/proxy.go index f1e34390..6c5821a3 100644 --- a/tests/helpers/http_proxy/proxy.go +++ b/tests/helpers/http_proxy/proxy.go @@ -65,6 +65,7 @@ func (p *HTTPProxy) handleHTTP(w http.ResponseWriter, req *http.Request) { http.Error(w, err.Error(), http.StatusServiceUnavailable) return } + defer resp.Body.Close() // save 'resp' respBody, err := io.ReadAll(resp.Body) diff --git a/tests/helpers/testsflag/testsflag.go b/tests/helpers/testsflag/testsflag.go index 1cc21462..b6714fce 100644 --- a/tests/helpers/testsflag/testsflag.go +++ b/tests/helpers/testsflag/testsflag.go @@ -11,7 +11,7 @@ func IsTest() bool { return isTest } -// for cases, when in tests want to turn-off extra validations - for example, in benchmarks +// for cases, when in tests want to turn-off extra validations - for example, in benchmarks. func TurnOff() { isTest = false } diff --git a/tests/helpers/utils.go b/tests/helpers/utils.go index 13a1e880..d66b7254 100644 --- a/tests/helpers/utils.go +++ b/tests/helpers/utils.go @@ -39,7 +39,7 @@ func GetIntFromEnv(varName string) int { return val } -// StrictEquality - default callback for checksum - just compare typeNames +// StrictEquality - default callback for checksum - just compare typeNames. func StrictEquality(l, r string) bool { return l == r } @@ -121,7 +121,7 @@ func MakeTransferForIncrementalSnapshot(transferID string, src model.Source, dst return transfer } -// GetPortFromStr - works when the port is in the end of the string, preceded by a colon +// GetPortFromStr - works when the port is in the end of the string, preceded by a colon. func GetPortFromStr(s string) (int, error) { tokens := strings.Split(s, ":") if tokens[0] == s { diff --git a/tests/helpers/ydb.go b/tests/helpers/ydb.go index 6c61041f..01338ebe 100644 --- a/tests/helpers/ydb.go +++ b/tests/helpers/ydb.go @@ -47,15 +47,15 @@ func YDBInitChangeItem(tablePath string) *abstract.ChangeItem { {PrimaryKey: false, Required: false, ColumnName: "Utf8_", DataType: string(schema.TypeString), OriginalType: "ydb:Utf8"}, {PrimaryKey: false, Required: false, ColumnName: "Json_", DataType: string(schema.TypeAny), OriginalType: "ydb:Json"}, {PrimaryKey: false, Required: false, ColumnName: "JsonDocument_", DataType: string(schema.TypeAny), OriginalType: "ydb:JsonDocument"}, - //{PrimaryKey: false, Required: false, ColumnName: "Yson_", DataType: "", OriginalType: "ydb:Yson"}, // can't find any acceptable value - //{PrimaryKey: false, Required: false, ColumnName: "Uuid_", DataType: "", OriginalType: "ydb:Uuid"}, // Не поддержан для столбцов таблиц + // {PrimaryKey: false, Required: false, ColumnName: "Yson_", DataType: "", OriginalType: "ydb:Yson"}, // can't find any acceptable value + // {PrimaryKey: false, Required: false, ColumnName: "Uuid_", DataType: "", OriginalType: "ydb:Uuid"}, // Не поддержан для столбцов таблиц {PrimaryKey: false, Required: false, ColumnName: "Date_", DataType: string(schema.TypeDate), OriginalType: "ydb:Date"}, {PrimaryKey: false, Required: false, ColumnName: "Datetime_", DataType: string(schema.TypeDatetime), OriginalType: "ydb:Datetime"}, {PrimaryKey: false, Required: false, ColumnName: "Timestamp_", DataType: string(schema.TypeTimestamp), OriginalType: "ydb:Timestamp"}, {PrimaryKey: false, Required: false, ColumnName: "Interval_", DataType: string(schema.TypeInterval), OriginalType: "ydb:Interval"}, - //{PrimaryKey: false, Required: false, ColumnName: "TzDate_", DataType: "", OriginalType: "ydb:TzDate"}, // Не поддержан для столбцов таблиц - //{PrimaryKey: false, Required: false, ColumnName: "TzDateTime_", DataType: "", OriginalType: "ydb:TzDateTime"}, // Не поддержан для столбцов таблиц + // {PrimaryKey: false, Required: false, ColumnName: "TzDate_", DataType: "", OriginalType: "ydb:TzDate"}, // Не поддержан для столбцов таблиц + // {PrimaryKey: false, Required: false, ColumnName: "TzDateTime_", DataType: "", OriginalType: "ydb:TzDateTime"}, // Не поддержан для столбцов таблиц //{PrimaryKey: false, Required: false, ColumnName: "TzTimestamp_", DataType: "", OriginalType: "ydb:TzTimestamp"}, // Не поддержан для столбцов таблиц }), ColumnNames: []string{ @@ -77,43 +77,43 @@ func YDBInitChangeItem(tablePath string) *abstract.ChangeItem { "Utf8_", "Json_", "JsonDocument_", - //"Yson_", // can't find any acceptable value - //"Uuid_", // Не поддержан для столбцов таблиц + // "Yson_", // can't find any acceptable value + // "Uuid_", // Не поддержан для столбцов таблиц "Date_", "Datetime_", "Timestamp_", "Interval_", - //"TzDate_", // Не поддержан для столбцов таблиц - //"TzDateTime_", // Не поддержан для столбцов таблиц + // "TzDate_", // Не поддержан для столбцов таблиц + // "TzDateTime_", // Не поддержан для столбцов таблиц //"TzTimestamp_", // Не поддержан для столбцов таблиц }, ColumnValues: []interface{}{ - 1, //"id", - true, //"Bool_", - int8(1), //"Int8_", - int16(2), //"Int16_", - int32(3), //"Int32_", - int64(4), //"Int64_", - uint8(5), //"Uint8_", - uint16(6), //"Uint16_", - uint32(7), //"Uint32_", - uint64(8), //"Uint64_", - float32(1.1), //"Float_", - 2.2, //"Double_", - "234.000000000", //"Decimal_", - ".123e3", //"DyNumber_", - []byte{1}, //"String_", - "my_utf8_string", //"Utf8_", - "{}", //"Json_", - "{}", //"JsonDocument_", - //"Yson_", // can't find any acceptable value - //"Uuid_", // Не поддержан для столбцов таблиц - time.Date(2020, 2, 2, 0, 0, 0, 0, time.UTC), //"Date_", - time.Date(2020, 2, 2, 10, 2, 22, 0, time.UTC), //"Datetime_", - time.Date(2020, 2, 2, 10, 2, 22, 0, time.UTC), //"Timestamp_", - time.Duration(123000), //"Interval_", - //"TzDate_", // Не поддержан для столбцов таблиц - //"TzDateTime_", // Не поддержан для столбцов таблиц + 1, // "id", + true, // "Bool_", + int8(1), // "Int8_", + int16(2), // "Int16_", + int32(3), // "Int32_", + int64(4), // "Int64_", + uint8(5), // "Uint8_", + uint16(6), // "Uint16_", + uint32(7), // "Uint32_", + uint64(8), // "Uint64_", + float32(1.1), // "Float_", + 2.2, // "Double_", + "234.000000000", // "Decimal_", + ".123e3", // "DyNumber_", + []byte{1}, // "String_", + "my_utf8_string", // "Utf8_", + "{}", // "Json_", + "{}", // "JsonDocument_", + // "Yson_", // can't find any acceptable value + // "Uuid_", // Не поддержан для столбцов таблиц + time.Date(2020, 2, 2, 0, 0, 0, 0, time.UTC), // "Date_", + time.Date(2020, 2, 2, 10, 2, 22, 0, time.UTC), // "Datetime_", + time.Date(2020, 2, 2, 10, 2, 22, 0, time.UTC), // "Timestamp_", + time.Duration(123000), // "Interval_", + // "TzDate_", // Не поддержан для столбцов таблиц + // "TzDateTime_", // Не поддержан для столбцов таблиц //"TzTimestamp_", // Не поддержан для столбцов таблиц }, } @@ -290,7 +290,7 @@ func YDBPullDataFromTable(t *testing.T, token, database, instance, table string) return extracted } -// Test values +// Test values. func YDBStmtInsertValues(t *testing.T, tablePath string, values []interface{}, id int) *abstract.ChangeItem { result := YDBInitChangeItem(tablePath) result.ColumnValues = values diff --git a/tests/large/docker-compose/elasticsearch2pg_test.go b/tests/large/docker-compose/elasticsearch2pg_test.go index b7480055..1500945e 100644 --- a/tests/large/docker-compose/elasticsearch2pg_test.go +++ b/tests/large/docker-compose/elasticsearch2pg_test.go @@ -160,6 +160,9 @@ func deleteAllElasticIndexes(t *testing.T) { require.NoError(t, err) res, err := http.DefaultClient.Do(req) + if err == nil { + defer res.Body.Close() + } require.NoError(t, err) require.Equal(t, http.StatusOK, res.StatusCode) } diff --git a/tests/storage/pg/permissions/permissions_test.go b/tests/storage/pg/permissions/permissions_test.go index ac127077..8ca30f10 100644 --- a/tests/storage/pg/permissions/permissions_test.go +++ b/tests/storage/pg/permissions/permissions_test.go @@ -40,7 +40,7 @@ func TestTableListFilter(t *testing.T) { require.Contains(t, tl, abstract.TableID{Namespace: "public", Name: "t_accessible"}) } -// TestTableListFilterIncludeTables checks include directives do not affect `TableList` output +// TestTableListFilterIncludeTables checks include directives do not affect `TableList` output. func TestTableListFilterIncludeTables(t *testing.T) { source := prepareSource() source.DBTables = []string{"\"public\".\"does_not_exist\""} diff --git a/tests/tcrecipes/azure/azurite.go b/tests/tcrecipes/azure/azurite.go index 387ebf44..345ba6d0 100644 --- a/tests/tcrecipes/azure/azurite.go +++ b/tests/tcrecipes/azure/azurite.go @@ -3,6 +3,7 @@ package azure import ( "context" "fmt" + "net" "github.com/docker/go-connections/nat" "github.com/testcontainers/testcontainers-go" @@ -10,13 +11,13 @@ import ( ) const ( - // Default ports used by Azurite + // Default ports used by Azurite. BlobPort = "10000/tcp" QueuePort = "10001/tcp" TablePort = "10002/tcp" ) -// AzuriteContainer represents the Azurite container type used in the module +// AzuriteContainer represents the Azurite container type used in the module. type AzuriteContainer struct { testcontainers.Container Settings options @@ -45,7 +46,7 @@ func (c *AzuriteContainer) ServiceURL(ctx context.Context, srv Service) (string, return "", err } - return fmt.Sprintf("http://%s:%d", hostname, mappedPort.Int()), nil + return fmt.Sprintf("http://%s", net.JoinHostPort(hostname, mappedPort.Port())), nil } func (c *AzuriteContainer) MustServiceURL(ctx context.Context, srv Service) string { @@ -57,7 +58,7 @@ func (c *AzuriteContainer) MustServiceURL(ctx context.Context, srv Service) stri return url } -// Run creates an instance of the Azurite container type +// Run creates an instance of the Azurite container type. func RunAzurite(ctx context.Context, img string, opts ...testcontainers.ContainerCustomizer) (*AzuriteContainer, error) { req := testcontainers.ContainerRequest{ Image: img, diff --git a/tests/tcrecipes/azure/credentials.go b/tests/tcrecipes/azure/credentials.go index 3755fa5a..abd6c427 100644 --- a/tests/tcrecipes/azure/credentials.go +++ b/tests/tcrecipes/azure/credentials.go @@ -1,8 +1,8 @@ package azure const ( - // AccountName is the default testing account name used by Azurite + // AccountName is the default testing account name used by Azurite. AccountName string = "devstoreaccount1" - // AccountKey is the default testing account key used by Azurite + // AccountKey is the default testing account key used by Azurite. AccountKey string = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" ) diff --git a/tests/tcrecipes/azure/eventhub.go b/tests/tcrecipes/azure/eventhub.go index ddc7bf2a..167e2ea0 100644 --- a/tests/tcrecipes/azure/eventhub.go +++ b/tests/tcrecipes/azure/eventhub.go @@ -3,6 +3,7 @@ package azure import ( "context" "fmt" + "net" "os" "time" @@ -14,7 +15,7 @@ const ( EventHubPort = "5672/tcp" ) -// EventHubContainer represents the Eventhub container type used in the module +// EventHubContainer represents the Eventhub container type used in the module. type EventHubContainer struct { testcontainers.Container Settings options @@ -31,7 +32,7 @@ func (c *EventHubContainer) ServiceURL(ctx context.Context) (string, error) { return "", fmt.Errorf("unable to get mapped port: %s: %w", EventHubPort, err) } - return fmt.Sprintf("http://%s:%d", hostname, mappedPort.Int()), nil + return fmt.Sprintf("http://%s", net.JoinHostPort(hostname, mappedPort.Port())), nil } func (c *EventHubContainer) MustServiceURL(ctx context.Context) string { @@ -43,9 +44,8 @@ func (c *EventHubContainer) MustServiceURL(ctx context.Context) string { return url } -// Run creates an instance of the Eventhub container type +// Run creates an instance of the Eventhub container type. func RunEventHub(ctx context.Context, img string, blobAddress string, opts ...testcontainers.ContainerCustomizer) (*EventHubContainer, error) { - req := testcontainers.ContainerRequest{ Image: img, WaitingFor: wait.ForListeningPort(EventHubPort).WithStartupTimeout(3 * time.Second), diff --git a/tests/tcrecipes/clickhouse/clickhouse.go b/tests/tcrecipes/clickhouse/clickhouse.go index 42f44f4b..d510742c 100644 --- a/tests/tcrecipes/clickhouse/clickhouse.go +++ b/tests/tcrecipes/clickhouse/clickhouse.go @@ -22,7 +22,7 @@ const defaultImage = "clickhouse/clickhouse-server:23.3.8.21-alpine" const HTTPPort = nat.Port("8123/tcp") const NativePort = nat.Port("9000/tcp") -// ClickHouseContainer represents the ClickHouse container type used in the module +// ClickHouseContainer represents the ClickHouse container type used in the module. type ClickHouseContainer struct { testcontainers.Container dbName string @@ -47,7 +47,7 @@ func (c *ClickHouseContainer) ConnectionHost(ctx context.Context) (string, error // ConnectionString returns the dsn string for the clickhouse container, using the default 9000 port, and // obtaining the host and exposed port from the container. It also accepts a variadic list of extra arguments // which will be appended to the dsn string. The format of the extra arguments is the same as the -// connection string format, e.g. "dial_timeout=300ms" or "skip_verify=false" +// connection string format, e.g. "dial_timeout=300ms" or "skip_verify=false". func (c *ClickHouseContainer) ConnectionString(ctx context.Context, args ...string) (string, error) { host, err := c.ConnectionHost(ctx) if err != nil { @@ -66,7 +66,7 @@ func (c *ClickHouseContainer) ConnectionString(ctx context.Context, args ...stri return connectionString, nil } -// WithInitScripts sets the init scripts to be run when the container starts +// WithInitScripts sets the init scripts to be run when the container starts. func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { initScripts := []testcontainers.ContainerFile{} @@ -109,7 +109,7 @@ func WithConfigData(data string) testcontainers.CustomizeRequestOption { defer f.Close() // write data to the temporary file - if _, err := f.Write([]byte(data)); err != nil { + if _, err := f.WriteString(data); err != nil { panic(err) } @@ -219,7 +219,7 @@ func WithZookeeper(container *ZookeeperContainer) testcontainers.CustomizeReques `, container.IP())) } -// Prepare creates an instance of the ClickHouse container type +// Prepare creates an instance of the ClickHouse container type. func Prepare(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*ClickHouseContainer, error) { req := testcontainers.ContainerRequest{ Image: defaultImage, diff --git a/tests/tcrecipes/clickhouse/zookeeper.go b/tests/tcrecipes/clickhouse/zookeeper.go index c4c95b6a..c1ee94ba 100644 --- a/tests/tcrecipes/clickhouse/zookeeper.go +++ b/tests/tcrecipes/clickhouse/zookeeper.go @@ -12,7 +12,7 @@ const defaultZKImage = "zookeeper:3.7" const zkPort = nat.Port("2181/tcp") -// ClickHouseContainer represents the ClickHouse container type used in the module +// ClickHouseContainer represents the ClickHouse container type used in the module. type ZookeeperContainer struct { testcontainers.Container exposedPort nat.Port diff --git a/tests/tcrecipes/k3s/k3s.go b/tests/tcrecipes/k3s/k3s.go index dc29d3b7..6f011b9d 100644 --- a/tests/tcrecipes/k3s/k3s.go +++ b/tests/tcrecipes/k3s/k3s.go @@ -19,12 +19,12 @@ var ( defaultKubeConfigK3sPath = "/etc/rancher/k3s/k3s.yaml" ) -// K3sContainer represents the K3s container type used in the module +// K3sContainer represents the K3s container type used in the module. type K3sContainer struct { testcontainers.Container } -// Prepare creates an instance of the K3s container type +// Prepare creates an instance of the K3s container type. func Prepare(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*K3sContainer, error) { host, err := getContainerHost(ctx, opts...) if err != nil { @@ -98,7 +98,7 @@ func getContainerHost(ctx context.Context, opts ...testcontainers.ContainerCusto return "localhost", nil } -// GetKubeConfig returns the modified kubeconfig with server url +// GetKubeConfig returns the modified kubeconfig with server url. func (c *K3sContainer) GetKubeConfig(ctx context.Context) ([]byte, error) { hostIP, err := c.Host(ctx) if err != nil { diff --git a/tests/tcrecipes/k3s/types.go b/tests/tcrecipes/k3s/types.go index 559af2be..6232a557 100644 --- a/tests/tcrecipes/k3s/types.go +++ b/tests/tcrecipes/k3s/types.go @@ -11,45 +11,45 @@ type KubeConfigValue struct { Preferences struct{} `yaml:"preferences"` } -// KubeconfigUser is a struct used to create a kubectl configuration YAML file +// KubeconfigUser is a struct used to create a kubectl configuration YAML file. type KubeconfigUser struct { Name string `yaml:"name"` User KubeconfigUserKeyPair `yaml:"user"` } -// KubeconfigUserKeyPair is a struct used to create a kubectl configuration YAML file +// KubeconfigUserKeyPair is a struct used to create a kubectl configuration YAML file. type KubeconfigUserKeyPair struct { ClientCertificateData string `yaml:"client-certificate-data"` ClientKeyData string `yaml:"client-key-data"` AuthProvider KubeconfigAuthProvider `yaml:"auth-provider,omitempty"` } -// KubeconfigAuthProvider is a struct used to create a kubectl authentication provider +// KubeconfigAuthProvider is a struct used to create a kubectl authentication provider. type KubeconfigAuthProvider struct { Name string `yaml:"name"` Config map[string]string `yaml:"config"` } -// KubeconfigNamedCluster is a struct used to create a kubectl configuration YAML file +// KubeconfigNamedCluster is a struct used to create a kubectl configuration YAML file. type KubeconfigNamedCluster struct { Name string `yaml:"name"` Cluster KubeconfigCluster `yaml:"cluster"` } -// KubeconfigCluster is a struct used to create a kubectl configuration YAML file +// KubeconfigCluster is a struct used to create a kubectl configuration YAML file. type KubeconfigCluster struct { Server string `yaml:"server"` CertificateAuthorityData string `yaml:"certificate-authority-data"` CertificateAuthority string `yaml:"certificate-authority"` } -// KubeconfigNamedContext is a struct used to create a kubectl configuration YAML file +// KubeconfigNamedContext is a struct used to create a kubectl configuration YAML file. type KubeconfigNamedContext struct { Name string `yaml:"name"` Context KubeconfigContext `yaml:"context"` } -// KubeconfigContext is a struct used to create a kubectl configuration YAML file +// KubeconfigContext is a struct used to create a kubectl configuration YAML file. type KubeconfigContext struct { Cluster string `yaml:"cluster"` Namespace string `yaml:"namespace,omitempty"` diff --git a/tests/tcrecipes/kafka/kafka.go b/tests/tcrecipes/kafka/kafka.go index e500bdb2..bc54a81e 100644 --- a/tests/tcrecipes/kafka/kafka.go +++ b/tests/tcrecipes/kafka/kafka.go @@ -19,13 +19,13 @@ const ( //go:embed kafka_starter.sh var starterScriptContent []byte -// KafkaContainer represents the Kafka container type used in the module +// KafkaContainer represents the Kafka container type used in the module. type KafkaContainer struct { testcontainers.Container ClusterID string } -// RunContainer creates an instance of the Kafka container type +// RunContainer creates an instance of the Kafka container type. func RunContainer(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*KafkaContainer, error) { req := testcontainers.ContainerRequest{ Image: "confluentinc/confluent-local:7.5.0", diff --git a/tests/tcrecipes/localstack/types.go b/tests/tcrecipes/localstack/types.go index 5a1830f6..26ee4025 100644 --- a/tests/tcrecipes/localstack/types.go +++ b/tests/tcrecipes/localstack/types.go @@ -4,14 +4,14 @@ import ( "github.com/testcontainers/testcontainers-go" ) -// Container represents the LocalStack container type used in the module +// Container represents the LocalStack container type used in the module. type Container struct { testcontainers.Container host string } // LocalStackContainerRequest represents the LocalStack container request type used in the module -// to configure the container +// to configure the container. type LocalStackContainerRequest struct { testcontainers.GenericContainerRequest } @@ -21,12 +21,12 @@ type LocalStackContainerRequest struct { type OverrideContainerRequestOption func(req testcontainers.ContainerRequest) (testcontainers.ContainerRequest, error) // Deprecated: use testcontainers.ContainerCustomizer instead -// NoopOverrideContainerRequest returns a helper function that does not override the container request +// NoopOverrideContainerRequest returns a helper function that does not override the container request. var NoopOverrideContainerRequest = func(req testcontainers.ContainerRequest) (testcontainers.ContainerRequest, error) { return req, nil } -// Deprecated: use testcontainers.ContainerCustomizer instead +// Deprecated: use testcontainers.ContainerCustomizer instead. func (opt OverrideContainerRequestOption) Customize(req *testcontainers.GenericContainerRequest) error { r, err := opt(req.ContainerRequest) if err != nil { @@ -39,7 +39,7 @@ func (opt OverrideContainerRequestOption) Customize(req *testcontainers.GenericC } // Deprecated: use testcontainers.CustomizeRequest instead -// OverrideContainerRequest returns a function that can be used to merge the passed container request with one that is created by the LocalStack container +// OverrideContainerRequest returns a function that can be used to merge the passed container request with one that is created by the LocalStack container. func OverrideContainerRequest(r testcontainers.ContainerRequest) func(req testcontainers.ContainerRequest) (testcontainers.ContainerRequest, error) { destContainerReq := testcontainers.GenericContainerRequest{ ContainerRequest: r, diff --git a/tests/tcrecipes/postgres/postrges.go b/tests/tcrecipes/postgres/postrges.go index a2acfecf..9d25c9d7 100644 --- a/tests/tcrecipes/postgres/postrges.go +++ b/tests/tcrecipes/postgres/postrges.go @@ -20,7 +20,7 @@ const defaultPassword = "postgres" const defaultPostgresImage = "docker.io/postgres:16-alpine" const defaultPort = nat.Port("5432/tcp") -// PostgresContainer represents the postgres container type used in the module +// PostgresContainer represents the postgres container type used in the module. type PostgresContainer struct { testcontainers.Container dbName string @@ -32,7 +32,7 @@ type PostgresContainer struct { // ConnectionString returns the connection string for the postgres container, using the default 5432 port, and // obtaining the host and exposed port from the container. It also accepts a variadic list of extra arguments // which will be appended to the connection string. The format of the extra arguments is the same as the -// connection string format, e.g. "connect_timeout=10" or "application_name=myapp" +// connection string format, e.g. "connect_timeout=10" or "application_name=myapp". func (c *PostgresContainer) ConnectionString(ctx context.Context, args ...string) (string, error) { containerPort, err := c.MappedPort(ctx, "5432/tcp") if err != nil { @@ -63,7 +63,7 @@ func WithConfigFile(cfg string) testcontainers.CustomizeRequestOption { defer f.Close() - if _, err := f.Write([]byte(cfg)); err != nil { + if _, err := f.WriteString(cfg); err != nil { panic(err) } cfgFile := testcontainers.ContainerFile{ @@ -100,7 +100,7 @@ func WithInitDir(dir string) testcontainers.CustomizeRequestOption { })...) } -// WithInitScripts sets the init scripts to be run when the container starts +// WithInitScripts sets the init scripts to be run when the container starts. func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { initScripts := []testcontainers.ContainerFile{} @@ -118,7 +118,7 @@ func WithInitScripts(scripts ...string) testcontainers.CustomizeRequestOption { } } -// WithInitScripts sets the init scripts to be run when the container starts +// WithInitScripts sets the init scripts to be run when the container starts. func WithInitScript(script string, containerName string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { initScripts := []testcontainers.ContainerFile{} @@ -185,7 +185,7 @@ func WithImage(image string) testcontainers.CustomizeRequestOption { } } -// Prepare creates an instance of the postgres container type +// Prepare creates an instance of the postgres container type. func Prepare(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*PostgresContainer, error) { req := testcontainers.ContainerRequest{ Image: defaultPostgresImage, From d9a5e9d1f307273b92b4b8f3195847de71d7d548 Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 18:34:28 -0300 Subject: [PATCH 3/7] add golangci rules Signed-off-by: Henrique Goncalves --- .golangci.yml | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..cc29e7b1 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,70 @@ +linters: + disable-all: true + enable: + #- goimports + #- gosec + - unused + #- asasalint + - asciicheck + - bidichk + - bodyclose + - decorder + #- dogsled + #- durationcheck + #- errcheck + #- errname + #- errorlint + #- exhaustive + #- exportloopref + #- funlen + #- goconst + #- gocritic + - godot + #- gofmt + - gosimple + - govet + - ineffassign + - mirror + #- musttag + #- nestif + - nosprintfhostport + #- prealloc + #- predeclared + - staticcheck + - tenv + - tparallel + - typecheck + - unconvert + - usestdlibvars + - wastedassign + +issues: + exclude-rules: + - linters: + - govet + text: "printf: non-constant format string in call" + - linters: + - staticcheck + text: "SA1019" + - path: '(.+)_test\.go' + linters: + - gosec + - funlen + - goconst + - gocritic + - staticcheck + - ineffassign + - linters: + - gosimple + text: "S1000" + - path: "tests/.*" + linters: + - gosec + - funlen + - goconst + - gocritic + - staticcheck + - ineffassign + - linters: + - gosimple + text: "S1000" From 1b2cacc28f25a3d6079ef8c3c2f9566438c1f625 Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 18:46:10 -0300 Subject: [PATCH 4/7] linting checkpoint Signed-off-by: Henrique Goncalves --- go.mod | 4 +- pkg/base/adapter/legacy_table_adapter.go | 2 +- pkg/debezium/receiver_engine.go | 2 +- pkg/debezium/validator.go | 4 +- .../greenplum/pg_sink_params_regulated.go | 2 +- pkg/providers/kinesis/source.go | 4 +- pkg/providers/kinesis/stream_writer.go | 5 +- pkg/providers/mongo/client.go | 6 +- .../check_fault_tolerance_test.go | 2 +- pkg/providers/postgres/slot_monitor.go | 3 +- pkg/providers/s3/reader/reader_json_line.go | 2 +- pkg/providers/ydb/sink.go | 6 +- pkg/worker/tasks/load_sharded_snapshot.go | 4 +- pkg/worker/tasks/load_snapshot_incremental.go | 2 +- .../document_key_updates/rps_test.go | 346 ++++++++++++++++++ .../nested_shard_key/nested_shard_key_test.go | 272 ++++++++++++++ .../s32ch/replication/sqs/check_db_test.go | 2 +- .../thousands_csv_sqs/check_db_test.go | 2 +- 18 files changed, 644 insertions(+), 26 deletions(-) create mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go create mode 100644 tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go diff --git a/go.mod b/go.mod index 74e4cecd..d05e0cdc 100644 --- a/go.mod +++ b/go.mod @@ -75,11 +75,11 @@ require ( go.uber.org/zap v1.27.0 go.ytsaurus.tech/library/go/core/log v0.0.4 go.ytsaurus.tech/yt/go v0.0.24 - golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mod v0.20.0 golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 + golang.org/x/term v0.27.0 golang.org/x/text v0.21.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 golang.yandex/hasql v1.1.1 @@ -233,9 +233,9 @@ require ( go.ytsaurus.tech/library/go/ptr v0.0.2 // indirect go.ytsaurus.tech/library/go/x/xreflect v0.0.3 // indirect go.ytsaurus.tech/library/go/x/xruntime v0.0.4 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.22.1-0.20240829175637-39126e24d653 // indirect gonum.org/v1/gonum v0.14.0 // indirect diff --git a/pkg/base/adapter/legacy_table_adapter.go b/pkg/base/adapter/legacy_table_adapter.go index 71ca4419..cdee3c9d 100644 --- a/pkg/base/adapter/legacy_table_adapter.go +++ b/pkg/base/adapter/legacy_table_adapter.go @@ -195,7 +195,7 @@ func (l legacyColumnAdapter) Value(val interface{}) (base.Value, error) { return types.NewDefaultIntervalValue(&v, l), nil case int64: // value must be in microseconds - duration := time.Duration(int64(v) * 1000) + duration := time.Duration(v * 1000) return types.NewDefaultIntervalValue(&duration, l), nil } case *types.StringType: diff --git a/pkg/debezium/receiver_engine.go b/pkg/debezium/receiver_engine.go index c6f57db1..0cce3771 100644 --- a/pkg/debezium/receiver_engine.go +++ b/pkg/debezium/receiver_engine.go @@ -347,7 +347,7 @@ func arrayReceive(s *debeziumcommon.Schema, v interface{}, originalType *debeziu } resultVal := make([]interface{}, 0) vArr := v.([]interface{}) - items := debeziumcommon.Schema(*s.Items) + items := *s.Items for _, el := range vArr { elVal, isAbsent, err := receiveField(&items, el, originalType.GetArrElemTypeDescr(), true) if err != nil { diff --git a/pkg/debezium/validator.go b/pkg/debezium/validator.go index 0be8530b..3a187e61 100644 --- a/pkg/debezium/validator.go +++ b/pkg/debezium/validator.go @@ -95,9 +95,9 @@ func emitAndValidate(connectorParameters map[string]string, changeItem *abstract check := func(inStr string) { in := []byte(inStr) - schemaID := binary.BigEndian.Uint32([]byte(in)[1:5]) + schemaID := binary.BigEndian.Uint32(in[1:5]) schema := idToSchemaBody[int(schemaID)] - docBytes := []byte(in)[5:] + docBytes := in[5:] fmt.Println("EXTRA_VALIDATION_ON_TESTS__DEBEZIUM:schema:", schema) fmt.Println("EXTRA_VALIDATION_ON_TESTS__DEBEZIUM:doc:", string(docBytes)) diff --git a/pkg/providers/greenplum/pg_sink_params_regulated.go b/pkg/providers/greenplum/pg_sink_params_regulated.go index f464920a..b5dc7201 100644 --- a/pkg/providers/greenplum/pg_sink_params_regulated.go +++ b/pkg/providers/greenplum/pg_sink_params_regulated.go @@ -46,7 +46,7 @@ func (p PgSinkParamsRegulated) User() string { } func (p PgSinkParamsRegulated) Password() string { - return string(p.FPassword) + return p.FPassword } func (p PgSinkParamsRegulated) HasTLS() bool { diff --git a/pkg/providers/kinesis/source.go b/pkg/providers/kinesis/source.go index 8e0a2798..743e9e6f 100644 --- a/pkg/providers/kinesis/source.go +++ b/pkg/providers/kinesis/source.go @@ -71,7 +71,7 @@ func (s *Source) Fetch() ([]abstract.ChangeItem, error) { func (s *Source) inLimits() bool { s.inflightMutex.Lock() defer s.inflightMutex.Unlock() - return s.config.BufferSize == 0 || int(s.config.BufferSize) > s.inflightBytes + return s.config.BufferSize == 0 || s.config.BufferSize > s.inflightBytes } func (s *Source) addInflight(size int) { @@ -245,7 +245,7 @@ func (s *Source) waitLimits() { "reader throttled for %v, limits: %v / %v", backoffTimer.GetElapsedTime(), format.SizeInt(s.inflightBytes), - format.SizeInt(int(s.config.BufferSize)), + format.SizeInt(s.config.BufferSize), ) } } diff --git a/pkg/providers/kinesis/stream_writer.go b/pkg/providers/kinesis/stream_writer.go index 1e36f224..7d233559 100644 --- a/pkg/providers/kinesis/stream_writer.go +++ b/pkg/providers/kinesis/stream_writer.go @@ -15,12 +15,13 @@ func PutRecord(src *KinesisSource, data []byte, key string) error { if _, err = client. DescribeStream( &kinesis.DescribeStreamInput{ - StreamName: &src.Stream}); err != nil { + StreamName: &src.Stream, + }); err != nil { return xerrors.Errorf("No stream exists with the provided name: %w", err) } // put data to stream _, err = client.PutRecord(&kinesis.PutRecordInput{ - Data: []byte(data), + Data: data, StreamName: &src.Stream, PartitionKey: aws.String(key), }) diff --git a/pkg/providers/mongo/client.go b/pkg/providers/mongo/client.go index 10dc511f..783c9292 100644 --- a/pkg/providers/mongo/client.go +++ b/pkg/providers/mongo/client.go @@ -235,7 +235,7 @@ func DriverConnectionSrvOptions(mongoConnectionOptions *MongoConnectionOptions) clientOptions.SetAuth(options.Credential{ AuthSource: authSource, Username: mongoConnectionOptions.User, - Password: string(mongoConnectionOptions.Password), + Password: mongoConnectionOptions.Password, }) if len(mongoConnectionOptions.ReplicaSet) > 0 { @@ -270,7 +270,7 @@ func DriverConnectionOptions(mongoConnectionOptions *MongoConnectionOptions) (*o AuthSource: authSource, // AuthMechanism: "SCRAM-SHA-256", Username: mongoConnectionOptions.User, - Password: string(mongoConnectionOptions.Password), + Password: mongoConnectionOptions.Password, }) if mongoConnectionOptions.ClusterID != "" && !sharded { opts.SetReplicaSet("rs01") @@ -304,7 +304,7 @@ func newTLSConfig(caCert TrustedCACertificate) (*tls.Config, error) { case CACertificatePEMFilePaths: caFilePaths := downcasted for _, cert := range caFilePaths { - pemFileContent, err := os.ReadFile(string(cert)) + pemFileContent, err := os.ReadFile(cert) if err != nil { return nil, xerrors.Errorf("Cannot read file %s: %w", cert, err) } diff --git a/pkg/providers/postgres/dblog/tests/fault_tolerance/check_fault_tolerance_test.go b/pkg/providers/postgres/dblog/tests/fault_tolerance/check_fault_tolerance_test.go index b02452a7..db64bef2 100644 --- a/pkg/providers/postgres/dblog/tests/fault_tolerance/check_fault_tolerance_test.go +++ b/pkg/providers/postgres/dblog/tests/fault_tolerance/check_fault_tolerance_test.go @@ -49,7 +49,7 @@ func TestIncrementalSnapshotFaultTolerance(t *testing.T) { }) changeItemBuilder := helpers.NewChangeItemsBuilder("public", testTableName, arrColSchema) - helpers.CheckRowsCount(t, Source, "public", testTableName, uint64(initialRowsNumber)) + helpers.CheckRowsCount(t, Source, "public", testTableName, initialRowsNumber) pgStorage, err := pgsink.NewStorage(Source.ToStorageParams(nil)) require.NoError(t, err) diff --git a/pkg/providers/postgres/slot_monitor.go b/pkg/providers/postgres/slot_monitor.go index c7134641..68621c22 100644 --- a/pkg/providers/postgres/slot_monitor.go +++ b/pkg/providers/postgres/slot_monitor.go @@ -170,7 +170,6 @@ func (m *SlotMonitor) validateSlot(ctx context.Context) error { return err } return nil - } func (m *SlotMonitor) getLag(monitorQ string) (int64, error) { @@ -225,7 +224,7 @@ func RunSlotMonitor(ctx context.Context, pgSrc *PgSource, registry metrics.Regis return abstract.MakeStubSlotKiller(), nil, nil } - errChan := slotMonitor.StartSlotMonitoring(int64(pgSrc.SlotByteLagLimit)) + errChan := slotMonitor.StartSlotMonitoring(pgSrc.SlotByteLagLimit) pool, ok := slotMonitor.conn.(*pgxpool.Pool) if !ok { diff --git a/pkg/providers/s3/reader/reader_json_line.go b/pkg/providers/s3/reader/reader_json_line.go index 27addd4c..8f92aa49 100644 --- a/pkg/providers/s3/reader/reader_json_line.go +++ b/pkg/providers/s3/reader/reader_json_line.go @@ -419,7 +419,7 @@ func readAllLines(content []byte) ([]string, int, error) { break } } - lines = append(lines, string(line)) + lines = append(lines, line) bytesRead += (len(line) + len("\n")) } return lines, bytesRead, nil diff --git a/pkg/providers/ydb/sink.go b/pkg/providers/ydb/sink.go index 78d8a9eb..940cf0fb 100644 --- a/pkg/providers/ydb/sink.go +++ b/pkg/providers/ydb/sink.go @@ -950,21 +950,21 @@ func (s *sinker) ydbVal(dataType, originalType string, val interface{}) (types.V case "ydb:Int8": switch vv := val.(type) { case int8: - return types.Int8Value(int8(vv)), false, nil + return types.Int8Value(vv), false, nil default: return nil, true, xerrors.Errorf("Unable to convert %s value: %v with type: %T", originalType, vv, vv) } case "ydb:Int16": switch vv := val.(type) { case int16: - return types.Int16Value(int16(vv)), false, nil + return types.Int16Value(vv), false, nil default: return nil, true, xerrors.Errorf("Unable to convert %s value: %v with type: %T", originalType, vv, vv) } case "ydb:Uint16": switch vv := val.(type) { case uint16: - return types.Uint16Value(uint16(vv)), false, nil + return types.Uint16Value(vv), false, nil default: return nil, true, xerrors.Errorf("Unable to convert %s value: %v with type: %T", originalType, vv, vv) } diff --git a/pkg/worker/tasks/load_sharded_snapshot.go b/pkg/worker/tasks/load_sharded_snapshot.go index a50a1b08..b092329f 100644 --- a/pkg/worker/tasks/load_sharded_snapshot.go +++ b/pkg/worker/tasks/load_sharded_snapshot.go @@ -182,7 +182,7 @@ func (l *SnapshotLoader) SetShardedStateToCP(logger log.Logger) error { if err := l.cp.SetOperationState(l.operationID, l.shardedState); err != nil { return errors.CategorizedErrorf(categories.Internal, "unable to store upload shards: %w", err) } - logger.Info("sharded state uploaded", log.Any("state", string(l.shardedState))) + logger.Info("sharded state uploaded", log.Any("state", l.shardedState)) return nil } @@ -234,7 +234,7 @@ func (l *SnapshotLoader) WaitWorkersCompleted(ctx context.Context, workersCount completedWorkersCountPercent = (float64(completedWorkersCount) / float64(workersCount)) * 100 } - completed := (int(completedWorkersCount) == workersCount) + completed := (completedWorkersCount == workersCount) status := "running" if completed { diff --git a/pkg/worker/tasks/load_snapshot_incremental.go b/pkg/worker/tasks/load_snapshot_incremental.go index b9152018..6d921ede 100644 --- a/pkg/worker/tasks/load_snapshot_incremental.go +++ b/pkg/worker/tasks/load_snapshot_incremental.go @@ -58,7 +58,7 @@ func (l *SnapshotLoader) mergeWithNextIncrement(currentState []abstract.TableDes } nextFilters := map[abstract.TableID]abstract.WhereStatement{} for _, nextTbl := range nextState { - nextFilters[abstract.TableID{Namespace: nextTbl.Schema, Name: nextTbl.Name}] = abstract.WhereStatement(nextTbl.Filter) + nextFilters[abstract.TableID{Namespace: nextTbl.Schema, Name: nextTbl.Name}] = nextTbl.Filter } for i, table := range currentState { if filter, ok := nextFilters[table.ID()]; ok && filter != abstract.NoFilter { diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go b/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go new file mode 100644 index 00000000..ffe1afb0 --- /dev/null +++ b/tests/e2e/mongo2mongo/sharding/to_sharded/document_key_updates/rps_test.go @@ -0,0 +1,346 @@ +package shmongo + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/doublecloud/transfer/internal/logger" + "github.com/doublecloud/transfer/pkg/abstract" + cpclient "github.com/doublecloud/transfer/pkg/abstract/coordinator" + "github.com/doublecloud/transfer/pkg/abstract/model" + mongostorage "github.com/doublecloud/transfer/pkg/providers/mongo" + "github.com/doublecloud/transfer/pkg/randutil" + "github.com/doublecloud/transfer/pkg/runtime/local" + "github.com/doublecloud/transfer/pkg/worker/tasks" + mongoshardedcluster "github.com/doublecloud/transfer/recipe/mongo/pkg/cluster" + "github.com/doublecloud/transfer/tests/e2e/mongo2mongo/rps" + "github.com/doublecloud/transfer/tests/helpers" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.ytsaurus.tech/library/go/core/log" +) + +func TestGroup(t *testing.T) { + t.Skip("TM-5255 temporary skip tests") + + defer func() { + require.NoError(t, helpers.CheckConnections( + helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, + helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, + )) + }() + + t.Run("Group after port check", func(t *testing.T) { + t.Run("Ping", Ping) + t.Run("RpsTest", RpsTest) + }) +} + +const ( + slotIDAkaTransferID = "dtt_shard_to_shard" + DB = "db1" + Collection = "coll1" +) + +var ( + TransferType = abstract.TransferTypeSnapshotAndIncrement + Source = &mongostorage.MongoSource{ + Hosts: []string{os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, + Port: helpers.GetIntFromEnv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPort), + User: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterUsername), + Password: model.SecretString(os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), + AuthSource: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), + Collections: []mongostorage.MongoCollection{ + {DatabaseName: DB, CollectionName: Collection}, + }, + SlotID: slotIDAkaTransferID, + } + Target = mongostorage.MongoDestination{ + Hosts: []string{os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, + Port: helpers.GetIntFromEnv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPort), + User: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterUsername), + Password: model.SecretString(os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), + AuthSource: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), + Cleanup: model.DisabledCleanup, + } +) + +//--------------------------------------------------------------------------------------------------------------------- +// utils + +func LogMongoSource(s *mongostorage.MongoSource) { + fmt.Printf("Source.Hosts: %v\n", s.Hosts) + fmt.Printf("Source.Port: %v\n", s.Port) + fmt.Printf("Source.User: %v\n", s.User) + fmt.Printf("Source.Password: %v\n", s.Password) +} + +func LogMongoDestination(s *mongostorage.MongoDestination) { + fmt.Printf("Target.Hosts: %v\n", s.Hosts) + fmt.Printf("Target.Port: %v\n", s.Port) + fmt.Printf("Target.User: %v\n", s.User) + fmt.Printf("Target.Password: %v\n", s.Password) +} + +func MakeDstClient(t *mongostorage.MongoDestination) (*mongostorage.MongoClientWrapper, error) { + return mongostorage.Connect(context.Background(), t.ConnectionOptions([]string{}), nil) +} + +func ShardSourceCollection(t *testing.T, client *mongostorage.MongoClientWrapper) { + adminDB := client.Database("admin") + + res := adminDB.RunCommand(context.TODO(), + bson.D{ + {Key: "enableSharding", Value: DB}, + }) + require.NoError(t, res.Err()) + + var runCmdResult bson.M + require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ + {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection)}, + {Key: "key", Value: bson.D{ + {Key: "document.a", Value: "hashed"}, + {Key: "document.b", Value: 1}, + {Key: "document.c", Value: 1}, + }}, + {Key: "unique", Value: false}, + }).Decode(&runCmdResult)) +} + +func ShardTargetCollection(t *testing.T, client *mongostorage.MongoClientWrapper) { + adminDB := client.Database("admin") + + res := adminDB.RunCommand(context.TODO(), + bson.D{ + {Key: "enableSharding", Value: DB}, + }) + require.NoError(t, res.Err()) + + key := bson.D{ + {Key: "document.x", Value: "hashed"}, + {Key: "document.y", Value: 1}, + {Key: "document.z", Value: 1}, + } + + var runCmdResult bson.M + require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ + {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection)}, + {Key: "key", Value: key}, + {Key: "unique", Value: false}, + }).Decode(&runCmdResult)) +} + +func Ping(t *testing.T) { + // ping src + LogMongoSource(Source) + client, err := mongostorage.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) + defer func() { _ = client.Close(context.Background()) }() + require.NoError(t, err) + err = client.Ping(context.TODO(), nil) + require.NoError(t, err) + + // ping dst + LogMongoDestination(&Target) + client2, err := MakeDstClient(&Target) + defer func() { _ = client2.Close(context.Background()) }() + require.NoError(t, err) + err = client2.Ping(context.TODO(), nil) + require.NoError(t, err) +} + +func clearStorage(t *testing.T, client *mongostorage.MongoClientWrapper) { + t.Helper() + var err error + + db := client.Database(DB) + _ = db.Collection(Collection).Drop(context.Background()) + err = db.CreateCollection(context.Background(), Collection) + require.NoError(t, err) +} + +func RpsTest(t *testing.T) { + for _, rsName := range []mongostorage.MongoReplicationSource{ + mongostorage.MongoReplicationSourcePerDatabaseFullDocument, + mongostorage.MongoReplicationSourcePerDatabaseUpdateDocument, + } { + t.Run(string(rsName), func(t *testing.T) { + RpsTestForRS(t, rsName) + }) + } +} + +func RpsTestForRS(t *testing.T, rs mongostorage.MongoReplicationSource) { + ctx := context.Background() + + clientSource, err := mongostorage.Connect(ctx, Source.ConnectionOptions([]string{}), nil) + require.NoError(t, err) + defer func() { _ = clientSource.Close(context.Background()) }() + + // recreate collections on source + clearStorage(t, clientSource) + dbSource := clientSource.Database(DB) + collectionSource := dbSource.Collection(Collection) + + // make connection to the target + clientTarget, err := mongostorage.Connect(ctx, Target.ConnectionOptions([]string{}), nil) + require.NoError(t, err) + // drop collection on target before sharding + clearStorage(t, clientTarget) + + // shard source + //ShardSourceCollection(t, clientSource) //IS it recipe limitation? + // shard target + ShardTargetCollection(t, clientTarget) + + mongoSource := Source + mongoSource.ReplicationSource = rs + transfer := helpers.MakeTransfer(helpers.TransferID, mongoSource, &Target, TransferType) + + // activate transfer + err = tasks.ActivateDelivery(ctx, nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) + require.NoError(t, err) + + // start local worker for activation + localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) + errChan := make(chan error, 1) + go func() { + errChan <- localWorker.Run() // like .Start(), but we in control for processing error in test + }() + + dstStorage, err := mongostorage.NewStorage(Target.ToStorageParams()) + require.NoError(t, err) + + // configure desired RPS + rpsContext, rpsCancel := context.WithCancel(ctx) + defer rpsCancel() + rpsModel := rps.NewRpsModel(rpsContext, &rps.RpsCallbacks{ + OnCreate: func(ctx context.Context, entity rps.KV) { + _, err := collectionSource.InsertOne(ctx, entity) + require.NoError(t, err) + }, + OnUpdate: func(ctx context.Context, previous rps.KV, actual rps.KV) { + opts := options.Update() + doc, ok := previous.Document.(bson.D) + require.True(t, ok) + filter := bson.D{ + {Key: "_id", Value: previous.Key}, + {Key: "document.a", Value: doc.Map()["a"]}, + {Key: "document.b", Value: doc.Map()["b"]}, + {Key: "document.c", Value: doc.Map()["c"]}, + } + update := bson.D{{Key: "$set", Value: bson.D{{Key: "document", Value: actual.Document}}}} + result, err := collectionSource.UpdateOne(ctx, filter, update, opts) + require.NoError(t, err) + require.Equal(t, int64(1), result.ModifiedCount) + }, + OnReplace: func(ctx context.Context, previous rps.KV, actual rps.KV) { + opts := options.Replace() + filter := bson.D{{Key: "_id", Value: previous.Key}} + result, err := collectionSource.ReplaceOne(ctx, filter, actual, opts) + require.NoError(t, err) + require.Equal(t, int64(1), result.ModifiedCount) + }, + OnDelete: func(ctx context.Context, key string) { + filter := bson.D{{Key: "_id", Value: key}} + result, err := collectionSource.DeleteOne(ctx, filter, nil) + require.NoError(t, err) + require.Equal(t, int64(1), result.DeletedCount) + }, + Tick: func(ctx context.Context, tickId int, model *rps.RpsModel) bool { + if tickId > 12 { + // stop generation on last Delay + logger.Log.Info("RPS stopping", log.Int("tickId", tickId)) + return false + } + return true + }, + }) + + rpsModel.SetSpec(&rps.RpsSpec{ + DeleteCount: 100, + CreateCount: 100, + UpdateCount: 100, + ReplaceCount: 100, + KVConstructor: func() rps.KV { + return rps.KV{ + Key: randutil.GenerateAlphanumericString(16), + Document: bson.D{ + {Key: "a", Value: randutil.GenerateAlphanumericString(8)}, + {Key: "b", Value: randutil.GenerateAlphanumericString(8)}, + {Key: "c", Value: randutil.GenerateAlphanumericString(8)}, + {Key: "x", Value: randutil.GenerateAlphanumericString(8)}, + {Key: "y", Value: randutil.GenerateAlphanumericString(8)}, + {Key: "z", Value: randutil.GenerateAlphanumericString(8)}, + }, + } + }, + Delay: 0, + }) + + logger.Log.Info("Start RPS generator") + rpsModelDone := make(chan struct{}) + go func() { + defer rpsModel.Close() + defer close(rpsModelDone) + rpsModel.Start() + }() + select { + case <-rpsModelDone: + break + case <-ctx.Done(): + t.Fatal("Couldn't wait for RPS to close") + } + + // wait for replication to catch up lag + rowCount := uint64(len(rpsModel.Persistent)) + tryingsCount := 30 + tries := 0 + for tries = 0; tries < tryingsCount; tries++ { + td := abstract.TableID{Namespace: DB, Name: Collection} + dstTableSize, err := dstStorage.ExactTableRowsCount(td) // TODO(@kry127;@timmyb32r) TM2409 change on GetRowsCount() + require.NoError(t, err) + + t.Logf("Table: %s, count rows. Expected: %d, actual: %d", td.Fqtn(), rowCount, dstTableSize) + if dstTableSize == rowCount { + break + } + time.Sleep(time.Second) + } + if tries == tryingsCount { + // nevermind, further test is unpassable + t.Logf("Tries are over: %d out of %d", tries, tryingsCount) + } + + // wait a little bit (push batch delay is recomended) + time.Sleep(3 * mongostorage.DefaultBatchFlushInterval) + + // stop worker + logger.Log.Info("Stop local worker") + err = localWorker.Stop() + require.NoError(t, err) + + // wait for appropriate error from replication + select { + case err := <-errChan: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatalf("Couldn't wait until replication ended: %v", ctx.Err()) + } + + dbTarget := clientTarget.Database(DB) + collectionTarget := dbTarget.Collection(Collection) + + // check that 'persistent' is present in source and target, and they values are equal + // and check that 'not persistent' neither on source nor target + logger.Log.Info("Validation of source and target databases") + for fromWhere, coll := range map[string]*mongo.Collection{"source": collectionSource, "target": collectionTarget} { + rpsModel.CheckValid(t, ctx, fromWhere, coll) + } + + logger.Log.Info("All values validated, tear down") +} diff --git a/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go b/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go new file mode 100644 index 00000000..ea00e7f8 --- /dev/null +++ b/tests/e2e/mongo2mongo/sharding/to_sharded/nested_shard_key/nested_shard_key_test.go @@ -0,0 +1,272 @@ +package shmongo + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/doublecloud/transfer/internal/logger" + "github.com/doublecloud/transfer/pkg/abstract" + cpclient "github.com/doublecloud/transfer/pkg/abstract/coordinator" + "github.com/doublecloud/transfer/pkg/abstract/model" + mongodataagent "github.com/doublecloud/transfer/pkg/providers/mongo" + "github.com/doublecloud/transfer/pkg/runtime/local" + "github.com/doublecloud/transfer/pkg/worker/tasks" + mongoshardedcluster "github.com/doublecloud/transfer/recipe/mongo/pkg/cluster" + "github.com/doublecloud/transfer/tests/helpers" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +func init() { + _ = os.Setenv("YC", "1") // to not go to vanga +} + +func TestGroup(t *testing.T) { + defer func() { + require.NoError(t, helpers.CheckConnections( + helpers.LabeledPort{Label: "Mongo source", Port: Source.Port}, + helpers.LabeledPort{Label: "Mongo target", Port: Target.Port}, + )) + }() + + t.Run("Group after port check", func(t *testing.T) { + t.Run("Ping", Ping) + t.Run("Load", Load) + }) +} + +const ( + slotIDAkaTransferID = "dtt_shard_to_shard" + DB = "db1" + Collection1 = "coll1" + Collection2 = "coll2" + Collection3 = "coll3" +) + +var ( + TransferType = abstract.TransferTypeSnapshotAndIncrement + Source = &mongodataagent.MongoSource{ + Hosts: []string{os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, + Port: helpers.GetIntFromEnv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPort), + User: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterUsername), + Password: model.SecretString(os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), + AuthSource: os.Getenv("DB1_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), + Collections: []mongodataagent.MongoCollection{ + {DatabaseName: DB, CollectionName: Collection1}, + {DatabaseName: DB, CollectionName: Collection2}, + {DatabaseName: DB, CollectionName: Collection3}, + }, + SlotID: slotIDAkaTransferID, + } + Target = mongodataagent.MongoDestination{ + Hosts: []string{os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterHost)}, + Port: helpers.GetIntFromEnv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPort), + User: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterUsername), + Password: model.SecretString(os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterPassword)), + AuthSource: os.Getenv("DB2_" + mongoshardedcluster.EnvMongoShardedClusterAuthSource), + Cleanup: model.DisabledCleanup, + } +) + +func init() { + Source.WithDefaults() +} + +//--------------------------------------------------------------------------------------------------------------------- +// utils + +func LogMongoSource(s *mongodataagent.MongoSource) { + fmt.Printf("Source.Hosts: %v\n", s.Hosts) + fmt.Printf("Source.Port: %v\n", s.Port) + fmt.Printf("Source.User: %v\n", s.User) + fmt.Printf("Source.Password: %v\n", s.Password) +} + +func LogMongoDestination(s *mongodataagent.MongoDestination) { + fmt.Printf("Target.Hosts: %v\n", s.Hosts) + fmt.Printf("Target.Port: %v\n", s.Port) + fmt.Printf("Target.User: %v\n", s.User) + fmt.Printf("Target.Password: %v\n", s.Password) +} + +func ShardTargetCollections(t *testing.T, client *mongodataagent.MongoClientWrapper) { + adminDB := client.Database("admin") + + res := adminDB.RunCommand(context.TODO(), + bson.D{ + {Key: "enableSharding", Value: DB}, + }) + require.NoError(t, res.Err()) + + key1 := bson.D{ + {Key: "_id.x", Value: "hashed"}, + } + + key2 := bson.D{ + {Key: "_id.x", Value: "hashed"}, + {Key: "city", Value: 1}, + } + + key3 := bson.D{ + {Key: "_id.x", Value: "hashed"}, + {Key: "_id.y", Value: 1}, + } + + var runCmdResult bson.M + require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ + {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection1)}, + {Key: "key", Value: key1}, + {Key: "unique", Value: false}, + }).Decode(&runCmdResult)) + + require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ + {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection2)}, + {Key: "key", Value: key2}, + {Key: "unique", Value: false}, + }).Decode(&runCmdResult)) + + require.NoError(t, adminDB.RunCommand(context.Background(), bson.D{ + {Key: "shardCollection", Value: fmt.Sprintf("%s.%s", DB, Collection3)}, + {Key: "key", Value: key3}, + {Key: "unique", Value: false}, + }).Decode(&runCmdResult)) +} + +func Ping(t *testing.T) { + // ping src + LogMongoSource(Source) + client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) + defer func() { _ = client.Close(context.Background()) }() + require.NoError(t, err) + err = client.Ping(context.TODO(), nil) + require.NoError(t, err) + + // ping dst + LogMongoDestination(&Target) + client2, err := mongodataagent.Connect(context.Background(), Target.ConnectionOptions([]string{}), nil) + defer func() { _ = client2.Close(context.Background()) }() + require.NoError(t, err) + err = client2.Ping(context.TODO(), nil) + require.NoError(t, err) +} + +func insertOne(t *testing.T, coll *mongo.Collection, row any) { + _, err := coll.InsertOne(context.Background(), row) + require.NoError(t, err) +} + +func updateOne(t *testing.T, coll *mongo.Collection, filter, update bson.D) { + _, err := coll.UpdateOne(context.Background(), filter, update) + require.NoError(t, err) +} + +func Load(t *testing.T) { + client, err := mongodataagent.Connect(context.Background(), Source.ConnectionOptions([]string{}), nil) + require.NoError(t, err) + + //------------------------------------------------------------------------------------ + // create source collections + + db := client.Database(DB) + err = db.CreateCollection(context.Background(), Collection1) + require.NoError(t, err) + err = db.CreateCollection(context.Background(), Collection2) + require.NoError(t, err) + err = db.CreateCollection(context.Background(), Collection3) + require.NoError(t, err) + + coll1 := db.Collection(Collection1) + coll2 := db.Collection(Collection2) + coll3 := db.Collection(Collection3) + + type CompositeID struct { + X string `bson:"x,omitempty"` + Y string `bson:"y,omitempty"` + } + + type Citizen struct { + ID CompositeID `bson:"_id"` + Name string `bson:"name,omitempty"` + Age int `bson:"age,omitempty"` + City string `bson:"city,omitempty"` + } + + jamesGordon := Citizen{ + ID: CompositeID{"x1", "y1"}, + Name: "James Gordon", + Age: 33, + City: "Gotham", + } + + insertOne(t, coll1, jamesGordon) + insertOne(t, coll2, jamesGordon) + insertOne(t, coll3, jamesGordon) + + //------------------------------------------------------------------------------------ + // shard target collections + + targetClient, err := mongodataagent.Connect(context.Background(), Target.ConnectionOptions([]string{}), nil) + require.NoError(t, err) + + targetDB := targetClient.Database(DB) + err = targetDB.CreateCollection(context.Background(), Collection1) + require.NoError(t, err) + err = targetDB.CreateCollection(context.Background(), Collection2) + require.NoError(t, err) + err = targetDB.CreateCollection(context.Background(), Collection3) + require.NoError(t, err) + + ShardTargetCollections(t, targetClient) + + //------------------------------------------------------------------------------------ + // activate + + transfer := helpers.MakeTransfer(helpers.TransferID, Source, &Target, abstract.TransferTypeSnapshotAndIncrement) + + err = tasks.ActivateDelivery(context.TODO(), nil, cpclient.NewFakeClient(), *transfer, helpers.EmptyRegistry()) + require.NoError(t, err) + + localWorker := local.NewLocalWorker(cpclient.NewFakeClient(), transfer, helpers.EmptyRegistry(), logger.Log) + localWorker.Start() + defer localWorker.Stop() //nolint + + //------------------------------------------------------------------------------------ + // replicate update one record and insert one + + leslieThompkins := Citizen{ + ID: CompositeID{"x2", "y2"}, + Name: "Leslie Thompkins", + Age: 29, + City: "Gotham", + } + + insertOne(t, coll1, leslieThompkins) + insertOne(t, coll2, leslieThompkins) + insertOne(t, coll3, leslieThompkins) + + leslieFilter := bson.D{{Key: "_id", Value: leslieThompkins.ID}} + leslieUpdate := bson.D{{Key: "$set", Value: bson.D{{Key: "city", Value: "Atlanta"}}}} + + updateOne(t, coll1, leslieFilter, leslieUpdate) + updateOne(t, coll2, leslieFilter, leslieUpdate) + updateOne(t, coll3, leslieFilter, leslieUpdate) + + jamesFilter := bson.D{{Key: "_id", Value: jamesGordon.ID}} + jamesUpdate := bson.D{{Key: "$set", Value: bson.D{{Key: "age", Value: 34}}}} + + updateOne(t, coll1, jamesFilter, jamesUpdate) + updateOne(t, coll2, jamesFilter, jamesUpdate) + updateOne(t, coll3, jamesFilter, jamesUpdate) + + //------------------------------------------------------------------------------------ + // check results + + require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection1, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) + require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection2, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) + require.NoError(t, helpers.WaitEqualRowsCount(t, DB, Collection3, helpers.GetSampleableStorageByModel(t, Source), helpers.GetSampleableStorageByModel(t, Target), 60*time.Second)) + require.NoError(t, helpers.CompareStorages(t, Source, Target, helpers.NewCompareStorageParams())) +} diff --git a/tests/e2e/s32ch/replication/sqs/check_db_test.go b/tests/e2e/s32ch/replication/sqs/check_db_test.go index 81f5e01e..ebebf895 100644 --- a/tests/e2e/s32ch/replication/sqs/check_db_test.go +++ b/tests/e2e/s32ch/replication/sqs/check_db_test.go @@ -87,7 +87,7 @@ func TestNativeS3PathsAreUnescaped(t *testing.T) { Region: aws.String(sqsRegion), S3ForcePathStyle: aws.Bool(src.ConnectionConfig.S3ForcePathStyle), Credentials: credentials.NewStaticCredentials( - sqsUser, string(sqsQueueName), "", + sqsUser, sqsQueueName, "", ), }) require.NoError(t, err) diff --git a/tests/e2e/s32ch/replication/thousands_csv_sqs/check_db_test.go b/tests/e2e/s32ch/replication/thousands_csv_sqs/check_db_test.go index 0b4da34a..ede07156 100644 --- a/tests/e2e/s32ch/replication/thousands_csv_sqs/check_db_test.go +++ b/tests/e2e/s32ch/replication/thousands_csv_sqs/check_db_test.go @@ -82,7 +82,7 @@ func TestNativeS3PathsAreUnescaped(t *testing.T) { Region: aws.String(sqsRegion), S3ForcePathStyle: aws.Bool(src.ConnectionConfig.S3ForcePathStyle), Credentials: credentials.NewStaticCredentials( - sqsUser, string(sqsQueueName), "", + sqsUser, sqsQueueName, "", ), }) require.NoError(t, err) From 5a54bdf33be25ad9af87a546868b70d2eaf72001 Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 18:57:24 -0300 Subject: [PATCH 5/7] add SAST / SCA workflows, bump runtime to 1.22.11 --- .github/workflows/build_and_test.yml | 125 ++++++++++++++++----------- .github/workflows/security.yml | 77 +++++++++++++++++ go.mod | 2 +- 3 files changed, 151 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/security.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a2b3c1fb..da42f05c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -3,7 +3,7 @@ name: Run Build and Test on: push: branches: - - 'main' + - "main" pull_request: branches: - "*" @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: "1.22.0" + go-version: "1.22.11" - shell: bash run: | make build @@ -31,26 +31,30 @@ jobs: fail-fast: false matrix: suite: [ - # CLI test suites - { group: "cmd", name: "cmd", path: "" }, - # providers suites, some of the providers are too heavy to run as single test - { group: "pkg/providers", name: "yt", path: "yt", yt: true }, - { group: "pkg/providers", name: "providers-postgres", path: "postgres" }, - # e2e test suites - { group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" }, - { group: "tests/e2e", name: "pg2pg", path: "pg2pg" }, - { group: "tests/e2e", name: "pg2ch", path: "pg2ch" }, - { group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" }, - { group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" }, - { group: "tests/e2e", name: "ch2s3", path: "ch2s3" }, - ] + # CLI test suites + { group: "cmd", name: "cmd", path: "" }, + # providers suites, some of the providers are too heavy to run as single test + { group: "pkg/providers", name: "yt", path: "yt", yt: true }, + { + group: "pkg/providers", + name: "providers-postgres", + path: "postgres", + }, + # e2e test suites + { group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" }, + { group: "tests/e2e", name: "pg2pg", path: "pg2pg" }, + { group: "tests/e2e", name: "pg2ch", path: "pg2ch" }, + { group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" }, + { group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" }, + { group: "tests/e2e", name: "ch2s3", path: "ch2s3" }, + ] steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Go uses: actions/setup-go@v5 with: - go-version: "1.22.0" + go-version: "1.22.11" - shell: bash run: | go install gotest.tools/gotestsum@latest @@ -94,44 +98,64 @@ jobs: fail-fast: false matrix: suite: [ - # canon test suites - { group: "tests/canon", name: "canon-parser", path: "parser" }, - { group: "tests/storage", name: "storage-pg", path: "pg" }, - # internal test suites - { group: "internal", name: "internal", path: "..." }, - # provider test suites - { group: "pkg/providers", name: "providers-mongo", path: "mongo" }, - { group: "pkg/providers", name: "providers-mysql", path: "mysql" }, - { group: "pkg/providers", name: "providers-sample", path: "sample" }, - { group: "pkg/providers", name: "providers-kafka", path: "kafka" }, - { group: "pkg/providers", name: "providers-kinesis", path: "kinesis" }, - { group: "pkg/providers", name: "providers-greenplum", path: "greenplum" }, - { group: "pkg/providers", name: "providers-clickhouse", path: "clickhouse" }, - { group: "pkg/providers", name: "providers-elastic", path: "elastic" }, - # pkg test suites - { group: "pkg", name: "abstract", path: "abstract" }, - { group: "pkg", name: "transformer", path: "transformer" }, - { group: "pkg", name: "predicate", path: "predicate" }, - { group: "pkg", name: "dblog", path: "dblog" }, - { group: "pkg", name: "functions", path: "functions" }, - { group: "pkg", name: "maplock", path: "maplock" }, - { group: "pkg", name: "middlewares", path: "middlewares" }, - { group: "pkg", name: "parsequeue", path: "parsequeue" }, - { group: "pkg", name: "util", path: "util" }, - { group: "pkg", name: "stringutil", path: "stringutil" }, - { group: "pkg", name: "serializer", path: "serializer" }, - { group: "pkg", name: "worker", path: "worker" }, - { group: "pkg", name: "schemaregistry", path: "schemaregistry" }, - { group: "pkg", name: "parsers-generic", path: "parsers/generic" }, - { group: "pkg", name: "parsers-scanner", path: "parsers/scanner" } - ] + # canon test suites + { group: "tests/canon", name: "canon-parser", path: "parser" }, + { group: "tests/storage", name: "storage-pg", path: "pg" }, + # internal test suites + { group: "internal", name: "internal", path: "..." }, + # provider test suites + { group: "pkg/providers", name: "providers-mongo", path: "mongo" }, + { group: "pkg/providers", name: "providers-mysql", path: "mysql" }, + { + group: "pkg/providers", + name: "providers-sample", + path: "sample", + }, + { group: "pkg/providers", name: "providers-kafka", path: "kafka" }, + { + group: "pkg/providers", + name: "providers-kinesis", + path: "kinesis", + }, + { + group: "pkg/providers", + name: "providers-greenplum", + path: "greenplum", + }, + { + group: "pkg/providers", + name: "providers-clickhouse", + path: "clickhouse", + }, + { + group: "pkg/providers", + name: "providers-elastic", + path: "elastic", + }, + # pkg test suites + { group: "pkg", name: "abstract", path: "abstract" }, + { group: "pkg", name: "transformer", path: "transformer" }, + { group: "pkg", name: "predicate", path: "predicate" }, + { group: "pkg", name: "dblog", path: "dblog" }, + { group: "pkg", name: "functions", path: "functions" }, + { group: "pkg", name: "maplock", path: "maplock" }, + { group: "pkg", name: "middlewares", path: "middlewares" }, + { group: "pkg", name: "parsequeue", path: "parsequeue" }, + { group: "pkg", name: "util", path: "util" }, + { group: "pkg", name: "stringutil", path: "stringutil" }, + { group: "pkg", name: "serializer", path: "serializer" }, + { group: "pkg", name: "worker", path: "worker" }, + { group: "pkg", name: "schemaregistry", path: "schemaregistry" }, + { group: "pkg", name: "parsers-generic", path: "parsers/generic" }, + { group: "pkg", name: "parsers-scanner", path: "parsers/scanner" }, + ] steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Go uses: actions/setup-go@v5 with: - go-version: "1.22.0" + go-version: "1.22.11" - shell: bash run: | go install gotest.tools/gotestsum@latest @@ -167,10 +191,7 @@ jobs: run: exit 1 test-report: - needs: [ - generic-tests, - e2e-tests, - ] + needs: [generic-tests, e2e-tests] name: test-report if: always() && !contains(needs.*.result, 'skipped') runs-on: ubuntu-latest diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..3bd828b0 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,77 @@ +name: Security Testing + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened] + +env: + GO_VERSION: "1.22.5" + +jobs: + SAST: + name: Perform SAST analysis (golangci-lint) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install Protoc + uses: arduino/setup-protoc@v3 + + - name: Install protoc-gen-go + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + + - name: Install protoc-gen-go-grpc + run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 + + - name: Generate + run: go generate ./... + + - name: Run SAST (golangci-lint) + uses: golangci/golangci-lint-action@v4 + with: + version: "latest" + only-new-issues: true + args: --timeout=10m + skip-pkg-cache: true + skip-build-cache: true + + SCA: + name: Perform SCA analysis (govulncheck) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install Protoc + uses: arduino/setup-protoc@v3 + + - name: Install protoc-gen-go + run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + + - name: Install protoc-gen-go-grpc + run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 + + - name: Generate + run: go generate ./... + + - name: Run SCA (govulncheck) + uses: golang/govulncheck-action@v1 + with: + go-version-input: ${{ env.GO_VERSION }} + go-package: ./... + cache: false diff --git a/go.mod b/go.mod index d05e0cdc..388bfb4e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/doublecloud/transfer -go 1.22.5 +go 1.22.11 require ( cloud.google.com/go v0.112.2 From c800c4b2e216c802100c4f31807664b59e87a089 Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 18:59:05 -0300 Subject: [PATCH 6/7] remove unnecessary workflow steps --- .github/workflows/security.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 3bd828b0..3828bdef 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -23,18 +23,6 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - - name: Install Protoc - uses: arduino/setup-protoc@v3 - - - name: Install protoc-gen-go - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 - - - name: Install protoc-gen-go-grpc - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - - - name: Generate - run: go generate ./... - - name: Run SAST (golangci-lint) uses: golangci/golangci-lint-action@v4 with: @@ -57,18 +45,6 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - - name: Install Protoc - uses: arduino/setup-protoc@v3 - - - name: Install protoc-gen-go - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 - - - name: Install protoc-gen-go-grpc - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - - - name: Generate - run: go generate ./... - - name: Run SCA (govulncheck) uses: golang/govulncheck-action@v1 with: From 0f05378b22ac9c1c31a06f7571ee861d83b40bc8 Mon Sep 17 00:00:00 2001 From: Henrique Goncalves Date: Thu, 23 Jan 2025 19:20:55 -0300 Subject: [PATCH 7/7] fix abstract test Signed-off-by: Henrique Goncalves --- pkg/abstract/model/endpoint_rotator_config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/abstract/model/endpoint_rotator_config_test.go b/pkg/abstract/model/endpoint_rotator_config_test.go index 67bc8e05..47d8b279 100644 --- a/pkg/abstract/model/endpoint_rotator_config_test.go +++ b/pkg/abstract/model/endpoint_rotator_config_test.go @@ -10,6 +10,7 @@ import ( // test runner. func TestRotatorConfig(t *testing.T) { + t.Setenv("TZ", "Europe/Moscow") // this test is timezone aware t.Parallel() t.Run("ScenarioTesting", scenarioTesting) t.Run("NilWorkaround", nilWorkaround) // temporary test @@ -206,7 +207,6 @@ func offsetDateTestHours(t *testing.T) { func offsetDateTestDays(t *testing.T) { t.Parallel() - t.Setenv("TZ", "Europe/Moscow") // this test is timezone aware rcDays := RotatorConfig{KeepPartCount: 0, PartType: RotatorPartDay, PartSize: 1, TimeColumn: ""} rcDaysTimestamp := time.Now()