From 6a011bb72f5d19354a2729b2adba846d39ced559 Mon Sep 17 00:00:00 2001
From: wudi <676366545@qq.com>
Date: Thu, 13 Feb 2025 15:54:16 +0800
Subject: [PATCH] add lookup join retry
---
.github/workflows/build-connector.yml | 30 +-
.github/workflows/run-itcase.yml | 8 +-
.../flink-doris-connector-1.15/pom.xml | 52 +++
.../lookup/LookupRuntimeProviderFactory.java | 35 ++
.../DorisRowDataAsyncLookupFunction.java | 14 +-
.../table/DorisRowDataJdbcLookupFunction.java | 16 +-
.../flink/lookup/DorisLookupTableITCase.java | 116 ++++++-
.../flink-doris-connector-1.16/pom.xml | 43 +++
.../flink-doris-connector-1.17/pom.xml | 43 +++
.../flink-doris-connector-1.18/pom.xml | 43 +++
.../flink-doris-connector-1.19/pom.xml | 43 +++
.../flink-doris-connector-1.20/pom.xml | 44 +++
.../flink-doris-connector-common/pom.xml | 47 +++
.../doris/flink/backend/BackendClient.java | 0
.../doris/flink/catalog/DorisCatalog.java | 0
.../flink/catalog/DorisCatalogFactory.java | 0
.../flink/catalog/DorisCatalogOptions.java | 0
.../doris/flink/catalog/DorisTypeMapper.java | 0
.../doris/flink/catalog/doris/DataModel.java | 0
.../catalog/doris/DorisSchemaFactory.java | 0
.../flink/catalog/doris/DorisSystem.java | 0
.../doris/flink/catalog/doris/DorisType.java | 0
.../flink/catalog/doris/FieldSchema.java | 0
.../flink/catalog/doris/TableSchema.java | 0
.../doris/flink/cfg/ConfigurationOptions.java | 0
.../flink/cfg/DorisConnectionOptions.java | 0
.../flink/cfg/DorisExecutionOptions.java | 0
.../doris/flink/cfg/DorisLookupOptions.java | 0
.../apache/doris/flink/cfg/DorisOptions.java | 0
.../doris/flink/cfg/DorisReadOptions.java | 0
.../doris/flink/cfg/DorisStreamOptions.java | 0
.../connection/JdbcConnectionProvider.java | 0
.../SimpleJdbcConnectionProvider.java | 0
.../flink/datastream/DorisSourceFunction.java | 0
.../DorisDeserializationSchema.java | 0
.../RowDataDeserializationSchema.java | 0
.../SimpleListDeserializationSchema.java | 0
.../converter/DorisRowConverter.java | 0
.../exception/ConnectedFailedException.java | 0
.../flink/exception/CopyLoadException.java | 0
.../flink/exception/CreateTableException.java | 0
.../exception/DorisBatchLoadException.java | 0
.../doris/flink/exception/DorisException.java | 0
.../exception/DorisInternalException.java | 0
.../exception/DorisRuntimeException.java | 0
.../exception/DorisSchemaChangeException.java | 0
.../flink/exception/DorisSystemException.java | 0
.../exception/IllegalArgumentException.java | 0
.../LabelAlreadyExistsException.java | 0
.../exception/ShouldNeverHappenException.java | 0
.../flink/exception/StreamLoadException.java | 0
.../flink/lookup/DorisJdbcLookupReader.java | 7 +-
.../flink/lookup/DorisLookupFunction.java | 21 ++
.../doris/flink/lookup/DorisLookupReader.java | 7 +-
.../doris/flink/lookup/ExecutionPool.java | 0
.../org/apache/doris/flink/lookup/Get.java | 0
.../apache/doris/flink/lookup/GetAction.java | 0
.../doris/flink/lookup/LookupMetrics.java | 0
.../lookup/LookupRuntimeProviderFactory.java | 35 ++
.../doris/flink/lookup/LookupSchema.java | 0
.../org/apache/doris/flink/lookup/Record.java | 0
.../apache/doris/flink/lookup/RecordKey.java | 0
.../org/apache/doris/flink/lookup/Worker.java | 0
.../doris/flink/rest/PartitionDefinition.java | 0
.../apache/doris/flink/rest/RestService.java | 0
.../apache/doris/flink/rest/SchemaUtils.java | 0
.../doris/flink/rest/models/BackendV2.java | 0
.../apache/doris/flink/rest/models/Field.java | 0
.../doris/flink/rest/models/QueryPlan.java | 0
.../doris/flink/rest/models/RespContent.java | 0
.../doris/flink/rest/models/Schema.java | 0
.../doris/flink/rest/models/Tablet.java | 0
.../doris/flink/serialization/Routing.java | 0
.../doris/flink/serialization/RowBatch.java | 0
.../apache/doris/flink/sink/BackendUtil.java | 0
.../flink/sink/DorisAbstractCommittable.java | 0
.../doris/flink/sink/DorisCommittable.java | 0
.../sink/DorisCommittableSerializer.java | 0
.../apache/doris/flink/sink/DorisSink.java | 0
.../doris/flink/sink/EscapeHandler.java | 0
.../doris/flink/sink/HttpGetWithEntity.java | 0
.../doris/flink/sink/HttpPutBuilder.java | 0
.../org/apache/doris/flink/sink/HttpUtil.java | 0
.../apache/doris/flink/sink/LoadStatus.java | 0
.../apache/doris/flink/sink/ResponseUtil.java | 0
.../sink/batch/BatchBufferHttpEntity.java | 0
.../flink/sink/batch/BatchBufferStream.java | 0
.../flink/sink/batch/BatchRecordBuffer.java | 0
.../flink/sink/batch/DorisBatchSink.java | 0
.../sink/batch/DorisBatchStreamLoad.java | 0
.../flink/sink/batch/DorisBatchWriter.java | 0
.../flink/sink/batch/RecordWithMeta.java | 0
.../flink/sink/committer/DorisCommitter.java | 0
.../flink/sink/copy/BackoffAndRetryUtils.java | 0
.../flink/sink/copy/BatchRecordBuffer.java | 0
.../doris/flink/sink/copy/BatchStageLoad.java | 0
.../sink/copy/CopyCommittableSerializer.java | 0
.../doris/flink/sink/copy/CopySQLBuilder.java | 0
.../flink/sink/copy/DorisCopyCommittable.java | 0
.../flink/sink/copy/DorisCopyCommitter.java | 0
.../flink/sink/copy/DorisCopyWriter.java | 0
.../flink/sink/copy/HttpPostBuilder.java | 0
.../flink/sink/copy/models/BaseResponse.java | 0
.../flink/sink/copy/models/CopyIntoResp.java | 0
.../sink/schema/SQLParserSchemaManager.java | 0
.../flink/sink/schema/SchemaChangeHelper.java | 0
.../sink/schema/SchemaChangeManager.java | 0
.../flink/sink/schema/SchemaChangeMode.java | 0
.../flink/sink/util/DeleteOperation.java | 0
.../flink/sink/writer/CacheRecordBuffer.java | 0
.../doris/flink/sink/writer/ChangeEvent.java | 0
.../sink/writer/DorisAbstractWriter.java | 0
.../flink/sink/writer/DorisStreamLoad.java | 0
.../flink/sink/writer/DorisWriteMetrics.java | 0
.../doris/flink/sink/writer/DorisWriter.java | 0
.../flink/sink/writer/DorisWriterState.java | 0
.../writer/DorisWriterStateSerializer.java | 0
.../doris/flink/sink/writer/EventType.java | 0
.../flink/sink/writer/LabelGenerator.java | 0
.../flink/sink/writer/LoadConstants.java | 0
.../doris/flink/sink/writer/RecordBuffer.java | 0
.../doris/flink/sink/writer/RecordStream.java | 0
.../doris/flink/sink/writer/WriteMode.java | 0
.../sink/writer/serializer/DorisRecord.java | 0
.../serializer/DorisRecordSerializer.java | 0
.../JsonDebeziumSchemaSerializer.java | 0
.../serializer/RecordWithMetaSerializer.java | 0
.../writer/serializer/RowDataSerializer.java | 0
.../sink/writer/serializer/RowSerializer.java | 0
.../serializer/SimpleStringSerializer.java | 0
.../jsondebezium/CdcDataChange.java | 0
.../jsondebezium/CdcSchemaChange.java | 0
.../JsonDebeziumChangeContext.java | 0
.../jsondebezium/JsonDebeziumChangeUtils.java | 0
.../jsondebezium/JsonDebeziumDataChange.java | 0
.../JsonDebeziumSchemaChange.java | 0
.../JsonDebeziumSchemaChangeImpl.java | 0
.../JsonDebeziumSchemaChangeImplV2.java | 0
.../jsondebezium/SQLParserSchemaChange.java | 0
.../doris/flink/source/DorisSource.java | 0
.../source/assigners/DorisSplitAssigner.java | 0
.../source/assigners/SimpleSplitAssigner.java | 0
.../enumerator/DorisSourceEnumerator.java | 0
.../enumerator/PendingSplitsCheckpoint.java | 0
.../PendingSplitsCheckpointSerializer.java | 0
.../source/reader/DorisFlightValueReader.java | 0
.../source/reader/DorisRecordEmitter.java | 0
.../source/reader/DorisSourceReader.java | 0
.../source/reader/DorisSourceSplitReader.java | 0
.../flink/source/reader/DorisValueReader.java | 0
.../flink/source/reader/ValueReader.java | 0
.../flink/source/split/DorisSourceSplit.java | 0
.../split/DorisSourceSplitSerializer.java | 0
.../source/split/DorisSourceSplitState.java | 0
.../flink/source/split/DorisSplitRecords.java | 0
.../doris/flink/table/DorisConfigOptions.java | 0
.../flink/table/DorisDynamicTableFactory.java | 0
.../flink/table/DorisDynamicTableSink.java | 0
.../flink/table/DorisDynamicTableSource.java | 15 +-
.../flink/table/DorisExpressionVisitor.java | 0
.../DorisRowDataAsyncLookupFunction.java | 165 +++++++++
.../flink/table/DorisRowDataInputFormat.java | 0
.../table/DorisRowDataJdbcLookupFunction.java | 133 +++++++
.../flink/table/DorisTableInputSplit.java | 0
.../doris/flink/tools/cdc/CdcTools.java | 0
.../doris/flink/tools/cdc/DatabaseSync.java | 0
.../flink/tools/cdc/DatabaseSyncConfig.java | 0
.../flink/tools/cdc/DorisTableConfig.java | 0
.../flink/tools/cdc/JdbcSourceSchema.java | 0
.../tools/cdc/ParsingProcessFunction.java | 0
.../flink/tools/cdc/SourceConnector.java | 0
.../doris/flink/tools/cdc/SourceSchema.java | 0
.../flink/tools/cdc/db2/Db2DatabaseSync.java | 0
.../flink/tools/cdc/db2/Db2DateConverter.java | 0
.../doris/flink/tools/cdc/db2/Db2Schema.java | 0
.../doris/flink/tools/cdc/db2/Db2Type.java | 0
...orisJsonDebeziumDeserializationSchema.java | 0
.../cdc/mongodb/ChangeStreamConstant.java | 0
.../cdc/mongodb/MongoDBDatabaseSync.java | 0
.../tools/cdc/mongodb/MongoDBSchema.java | 0
.../flink/tools/cdc/mongodb/MongoDBType.java | 0
.../tools/cdc/mongodb/MongoDateConverter.java | 0
.../mongodb/MongoParsingProcessFunction.java | 0
.../MongoDBJsonDebeziumSchemaSerializer.java | 0
.../MongoJsonDebeziumDataChange.java | 0
.../MongoJsonDebeziumSchemaChange.java | 0
.../cdc/mysql/DateToStringConverter.java | 0
.../tools/cdc/mysql/MysqlDatabaseSync.java | 0
.../flink/tools/cdc/mysql/MysqlSchema.java | 0
.../flink/tools/cdc/mysql/MysqlType.java | 0
.../tools/cdc/oracle/OracleDatabaseSync.java | 0
.../tools/cdc/oracle/OracleDateConverter.java | 0
.../flink/tools/cdc/oracle/OracleSchema.java | 0
.../flink/tools/cdc/oracle/OracleType.java | 0
.../cdc/postgres/PostgresDatabaseSync.java | 0
.../cdc/postgres/PostgresDateConverter.java | 0
.../tools/cdc/postgres/PostgresSchema.java | 0
.../tools/cdc/postgres/PostgresType.java | 0
.../cdc/sqlserver/SqlServerDatabaseSync.java | 0
.../cdc/sqlserver/SqlServerDateConverter.java | 0
.../tools/cdc/sqlserver/SqlServerSchema.java | 0
.../tools/cdc/sqlserver/SqlServerType.java | 0
.../doris/flink/util/ErrorMessages.java | 0
.../apache/doris/flink/util/FastDateUtil.java | 0
.../org/apache/doris/flink/util/IPUtils.java | 0
.../arrow/serializers/ArrowSerializer.java | 0
.../org.apache.flink.table.factories.Factory | 0
.../src/main/resources/log4j2.properties | 0
.../flink/backend/BackendClientTest.java | 0
.../catalog/DorisCatalogFactoryTest.java | 0
.../flink/catalog/DorisCatalogITCase.java | 0
.../catalog/DorisCatalogOptionsTest.java | 0
.../doris/flink/catalog/DorisCatalogTest.java | 0
.../flink/catalog/DorisTypeMapperTest.java | 0
.../catalog/doris/DorisSchemaFactoryTest.java | 0
.../flink/catalog/doris/DorisSystemTest.java | 0
.../flink/cfg/DorisExecutionOptionsTest.java | 0
.../flink/cfg/DorisLookupOptionsTest.java | 0
.../doris/flink/cfg/DorisOptionsTest.java | 0
.../doris/flink/cfg/DorisReadOptionsTest.java | 0
.../SimpleJdbcConnectionProviderTest.java | 0
.../container/AbstractContainerTestBase.java | 0
.../flink/container/AbstractE2EService.java | 0
.../container/AbstractITCaseService.java | 0
.../doris/flink/container/ContainerUtils.java | 0
.../container/e2e/Doris2DorisE2ECase.java | 0
.../container/e2e/Mysql2DorisE2ECase.java | 0
.../container/instance/ContainerService.java | 0
.../container/instance/DorisContainer.java | 0
.../instance/DorisCustomerContainer.java | 0
.../container/instance/MySQLContainer.java | 0
.../RowDataDeserializationSchemaTest.java | 0
.../convert/DorisRowConverterTest.java | 0
.../flink/example/CDCSchemaChangeExample.java | 20 +-
.../doris/flink/example/CatalogExample.java | 0
.../example/DorisDateAndTimestampSqlTest.java | 0
.../DorisIntranetAccessSinkExample.java | 0
.../example/DorisSinkArraySQLExample.java | 0
.../flink/example/DorisSinkBatchExample.java | 24 +-
.../doris/flink/example/DorisSinkExample.java | 51 ++-
.../example/DorisSinkExampleRowData.java | 0
.../example/DorisSinkMultiTableExample.java | 0
.../flink/example/DorisSinkSQLExample.java | 40 ++-
.../DorisSinkStreamMultiTableExample.java | 0
.../flink/example/DorisSourceDataStream.java | 0
.../flink/example/DorisSourceExample.java | 37 +-
.../flink/example/DorisSourceSinkExample.java | 0
.../flink/example/LookupJoinCdcExample.java | 117 +++++++
.../flink/example/LookupJoinExample.java | 9 +-
.../flink/lookup/DorisLookupTableITCase.java | 326 ++++++++++++++++++
.../apache/doris/flink/lookup/RecordTest.java | 0
.../doris/flink/rest/SchemaUtilsTest.java | 0
.../flink/rest/TestPartitionDefinition.java | 0
.../doris/flink/rest/TestRestService.java | 0
.../doris/flink/rest/models/TestSchema.java | 0
.../flink/serialization/TestRouting.java | 0
.../flink/serialization/TestRowBatch.java | 0
.../doris/flink/sink/DorisSinkITCase.java | 0
.../doris/flink/sink/DorisSinkTest.java | 0
.../doris/flink/sink/HttpEntityMock.java | 0
.../apache/doris/flink/sink/HttpTestUtil.java | 0
.../apache/doris/flink/sink/OptionUtils.java | 0
.../doris/flink/sink/TestBackendUtil.java | 0
.../flink/sink/TestDorisCommittable.java | 0
.../sink/TestDorisCommittableSerializer.java | 0
.../doris/flink/sink/TestEscapeHandler.java | 0
.../doris/flink/sink/TestResponseUtil.java | 0
.../org/apache/doris/flink/sink/TestUtil.java | 0
.../sink/batch/TestBatchBufferHttpEntity.java | 0
.../sink/batch/TestBatchBufferStream.java | 0
.../flink/sink/batch/TestDorisBatchSink.java | 0
.../sink/batch/TestDorisBatchStreamLoad.java | 0
.../sink/batch/TestDorisBatchWriter.java | 0
.../flink/sink/batch/TestRecordWithMeta.java | 0
.../sink/committer/MockCommitRequest.java | 0
.../sink/committer/TestDorisCommitter.java | 0
.../sink/copy/TestBatchRecordBuffer.java | 0
.../copy/TestCopyCommittableSerializer.java | 0
.../sink/copy/TestDorisCopyCommitter.java | 0
.../flink/sink/copy/TestDorisCopyWriter.java | 0
.../schema/SQLParserSchemaManagerTest.java | 0
.../sink/schema/SchemaChangeHelperTest.java | 0
.../sink/schema/SchemaManagerITCase.java | 0
.../flink/sink/schema/SchemaManagerTest.java | 0
.../sink/writer/TestCacheRecordBuffer.java | 0
.../sink/writer/TestDorisStreamLoad.java | 0
.../flink/sink/writer/TestDorisWriter.java | 0
.../TestDorisWriterStateSerializer.java | 0
.../TestJsonDebeziumSchemaSerializer.java | 0
.../flink/sink/writer/TestLabelGenerator.java | 0
.../flink/sink/writer/TestRecordBuffer.java | 0
.../writer/TestRecordWithMetaSerializer.java | 0
.../sink/writer/TestRowDataSerializer.java | 0
.../flink/sink/writer/TestRowSerializer.java | 0
.../TestJsonDebeziumChangeBase.java | 0
.../TestJsonDebeziumDataChange.java | 0
.../TestJsonDebeziumSchemaChangeImpl.java | 0
.../TestJsonDebeziumSchemaChangeImplV2.java | 0
.../TestSQLParserSchemaChange.java | 0
.../flink/source/DorisSourceExampleTest.java | 0
.../doris/flink/source/DorisSourceITCase.java | 0
.../enumerator/DorisSourceEnumeratorTest.java | 0
...PendingSplitsCheckpointSerializerTest.java | 0
.../source/reader/DorisSourceReaderTest.java | 0
.../source/reader/TestingReaderContext.java | 0
.../split/DorisSourceSplitSerializerTest.java | 0
.../source/split/DorisSourceSplitTest.java | 0
.../source/split/DorisSplitRecordsTest.java | 0
.../table/DorisDynamicTableFactoryTest.java | 0
.../table/DorisDynamicTableSourceTest.java | 0
.../DorisRowDataJdbcLookupFunctionITCase.java | 3 +-
.../tools/cdc/CdcDb2SyncDatabaseCase.java | 0
.../tools/cdc/CdcMongoSyncDatabaseCase.java | 0
.../tools/cdc/CdcMysqlSyncDatabaseCase.java | 27 +-
.../tools/cdc/CdcOraclelSyncDatabaseCase.java | 26 +-
.../cdc/CdcPostgresSyncDatabaseCase.java | 0
.../cdc/CdcSqlServerSyncDatabaseCase.java | 0
.../doris/flink/tools/cdc/CdcToolsTest.java | 0
.../flink/tools/cdc/DatabaseSyncTest.java | 0
.../flink/tools/cdc/DorisTableConfigTest.java | 0
.../flink/tools/cdc/MockSourceSchema.java | 0
.../flink/tools/cdc/db2/Db2TypeTest.java | 0
.../cdc/mongodb/MongoDBDatabaseSyncTest.java | 0
.../tools/cdc/mongodb/MongoDBSchemaTest.java | 0
.../tools/cdc/mongodb/MongoDBTypeTest.java | 0
.../cdc/mongodb/MongoDateConverterTest.java | 0
.../MongoParsingProcessFunctionTest.java | 0
.../flink/utils/DateToStringConverter.java | 0
.../doris/flink/utils/FactoryMocks.java | 0
.../doris/flink/utils/FastDateUtilTest.java | 0
.../apache/doris/flink/utils/MockSource.java | 0
.../test_doris2doris_sink_test_tbl.sql | 0
.../test_doris2doris_source_test_tbl.sql | 0
.../e2e/mysql2doris/testAutoAddTable.txt | 0
.../e2e/mysql2doris/testAutoAddTable_init.sql | 0
.../e2e/mysql2doris/testMySQL2Doris.txt | 0
.../mysql2doris/testMySQL2DorisByDefault.txt | 0
.../testMySQL2DorisByDefault_init.sql | 0
.../testMySQL2DorisCreateTable.txt | 0
.../testMySQL2DorisCreateTable_init.sql | 0
.../testMySQL2DorisEnableDelete.txt | 0
.../testMySQL2DorisEnableDelete_init.sql | 0
.../testMySQL2DorisMultiDb2One.txt | 0
.../testMySQL2DorisMultiDb2One_init.sql | 0
.../testMySQL2DorisMultiDbSync.txt | 0
.../testMySQL2DorisMultiDbSync_init.sql | 0
.../mysql2doris/testMySQL2DorisSQLParse.txt | 0
.../testMySQL2DorisSQLParse_init.sql | 0
.../e2e/mysql2doris/testMySQL2Doris_init.sql | 0
.../src/test/resources/log4j2-test.properties | 0
flink-doris-connector/pom.xml | 19 +-
.../flink/example/LookupJoinCdcExample.java | 83 -----
352 files changed, 1456 insertions(+), 243 deletions(-)
create mode 100644 flink-doris-connector/flink-doris-connector-1.15/pom.xml
create mode 100644 flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java
rename flink-doris-connector/{ => flink-doris-connector-1.15}/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java (92%)
rename flink-doris-connector/{ => flink-doris-connector-1.15}/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java (91%)
rename flink-doris-connector/{ => flink-doris-connector-1.15}/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java (56%)
create mode 100644 flink-doris-connector/flink-doris-connector-1.16/pom.xml
create mode 100644 flink-doris-connector/flink-doris-connector-1.17/pom.xml
create mode 100644 flink-doris-connector/flink-doris-connector-1.18/pom.xml
create mode 100644 flink-doris-connector/flink-doris-connector-1.19/pom.xml
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/pom.xml
create mode 100644 flink-doris-connector/flink-doris-connector-common/pom.xml
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/backend/BackendClient.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/CreateTableException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/DorisException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java (94%)
create mode 100644 flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java (82%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/Get.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/GetAction.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java (100%)
create mode 100644 flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/Record.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/RecordKey.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/lookup/Worker.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/RestService.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/models/Field.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/models/RespContent.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/models/Schema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/rest/models/Tablet.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/serialization/Routing.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/serialization/RowBatch.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/BackendUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/DorisSink.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/HttpUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/LoadStatus.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/EventType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/DorisSource.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java (96%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java (100%)
create mode 100644 flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java (100%)
create mode 100644 flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/util/ErrorMessages.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/util/FastDateUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/doris/flink/util/IPUtils.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/main/resources/log4j2.properties (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/ContainerUtils.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java (81%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/CatalogExample.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java (90%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java (76%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java (60%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java (60%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java (100%)
create mode 100644 flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java (88%)
create mode 100644 flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/lookup/RecordTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/rest/TestRestService.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/serialization/TestRouting.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/OptionUtils.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/TestUtil.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java (98%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java (84%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java (82%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/java/org/apache/doris/flink/utils/MockSource.java (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql (100%)
rename flink-doris-connector/{ => flink-doris-connector-common}/src/test/resources/log4j2-test.properties (100%)
delete mode 100644 flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
diff --git a/.github/workflows/build-connector.yml b/.github/workflows/build-connector.yml
index 2b2a242be..6c974b4b9 100644
--- a/.github/workflows/build-connector.yml
+++ b/.github/workflows/build-connector.yml
@@ -40,42 +40,24 @@ jobs:
- name: Build flink connector 1.15
run: |
- cd flink-doris-connector && mvn clean package \
- -Dflink.version=1.15.0 \
- -Dflink.minor.version=1.15 \
- -Dflink.python.id=flink-python_2.12
+ cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.15 -am
- name: Build flink connector 1.16
run: |
- cd flink-doris-connector && mvn clean package \
- -Dflink.version=1.16.0 \
- -Dflink.minor.version=1.16 \
- -Dflink.python.id=flink-python
+ cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.16 -am
- name: Build flink connector 1.17
run: |
- cd flink-doris-connector && mvn clean package \
- -Dflink.version=1.17.0 \
- -Dflink.minor.version=1.17 \
- -Dflink.python.id=flink-python
+ cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.17 -am
- name: Build flink connector 1.18
run: |
- cd flink-doris-connector && mvn clean package \
- -Dflink.version=1.18.0 \
- -Dflink.minor.version=1.18 \
- -Dflink.python.id=flink-python
+ cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.18 -am
- name: Build flink connector 1.19
run: |
- cd flink-doris-connector && mvn clean package \
- -Dflink.version=1.19.0 \
- -Dflink.minor.version=1.19 \
- -Dflink.python.id=flink-python
+ cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.19 -am
- name: Build flink connector 1.20
run: |
- cd flink-doris-connector && mvn clean package \
- -Dflink.version=1.20.0 \
- -Dflink.minor.version=1.20 \
- -Dflink.python.id=flink-python
+ cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.20 -am
\ No newline at end of file
diff --git a/.github/workflows/run-itcase.yml b/.github/workflows/run-itcase.yml
index 0b6706560..b036e9326 100644
--- a/.github/workflows/run-itcase.yml
+++ b/.github/workflows/run-itcase.yml
@@ -38,7 +38,11 @@ jobs:
distribution: adopt
java-version: '8'
- - name: Run ITCases
+ - name: Run ITCases For 1.15
run: |
- cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0"
+ cd flink-doris-connector && mvn test -pl flink-doris-connector-1.15 -am -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0"
+
+ - name: Run ITCases For 1.18
+ run: |
+ cd flink-doris-connector && mvn test -pl flink-doris-connector-1.18 -am -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0"
diff --git a/flink-doris-connector/flink-doris-connector-1.15/pom.xml b/flink-doris-connector/flink-doris-connector-1.15/pom.xml
new file mode 100644
index 000000000..c6072ac33
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.15/pom.xml
@@ -0,0 +1,52 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-1.15
+ Flink Doris Connector Flink 1.15
+
+
+ 1.15.3
+ 1.15
+ flink-python_2.12
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+ test-jar
+ test
+
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java
new file mode 100644
index 000000000..e074151fc
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.lookup;
+
+import org.apache.flink.table.connector.source.AsyncTableFunctionProvider;
+import org.apache.flink.table.connector.source.LookupTableSource;
+import org.apache.flink.table.connector.source.TableFunctionProvider;
+import org.apache.flink.table.functions.AsyncTableFunction;
+import org.apache.flink.table.functions.TableFunction;
+
+/** Factory to create {@link LookupTableSource.LookupRuntimeProvider}. */
+public class LookupRuntimeProviderFactory {
+
+ public static LookupTableSource.LookupRuntimeProvider create(
+ DorisLookupFunction function, boolean enableAsync) {
+ return enableAsync
+ ? AsyncTableFunctionProvider.of((AsyncTableFunction) function)
+ : TableFunctionProvider.of((TableFunction) function);
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
similarity index 92%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
rename to flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
index 2cd3c9968..5bc462642 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
+++ b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
@@ -30,6 +30,7 @@
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
+import org.apache.doris.flink.lookup.DorisLookupFunction;
import org.apache.doris.flink.lookup.DorisLookupReader;
import org.apache.doris.flink.lookup.LookupMetrics;
import org.apache.doris.flink.lookup.LookupSchema;
@@ -39,18 +40,19 @@
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
-import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
-public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction {
+/** async lookup {@link AsyncTableFunction} for 1.15. */
+public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction
+ implements DorisLookupFunction {
private static final Logger LOG =
LoggerFactory.getLogger(DorisRowDataAsyncLookupFunction.class);
private final DorisOptions options;
private final DorisLookupOptions lookupOptions;
private final long cacheMaxSize;
private final long cacheExpireMs;
- private transient Cache> cache;
+ private transient Cache> cache;
private DorisLookupReader lookupReader;
private LookupSchema lookupSchema;
private LookupMetrics lookupMetrics;
@@ -101,7 +103,7 @@ public void eval(CompletableFuture> future, Object... keys)
throws IOException {
RowData keyRow = GenericRowData.of(keys);
if (cache != null) {
- List cachedRows = cache.getIfPresent(keyRow);
+ Collection cachedRows = cache.getIfPresent(keyRow);
if (cachedRows != null) {
lookupMetrics.incHitCount();
if (LOG.isDebugEnabled()) {
@@ -114,7 +116,7 @@ public void eval(CompletableFuture> future, Object... keys)
}
}
- CompletableFuture> resultFuture = lookupReader.asyncGet(keyRow);
+ CompletableFuture> resultFuture = lookupReader.asyncGet(keyRow);
resultFuture.handleAsync(
(resultRows, throwable) -> {
try {
@@ -151,7 +153,7 @@ public void close() throws Exception {
}
@VisibleForTesting
- public Cache> getCache() {
+ public Cache> getCache() {
return cache;
}
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
similarity index 91%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
rename to flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
index a4b07677e..1dbcbeab0 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
+++ b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
@@ -30,6 +30,7 @@
import org.apache.doris.flink.cfg.DorisLookupOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
+import org.apache.doris.flink.lookup.DorisLookupFunction;
import org.apache.doris.flink.lookup.DorisLookupReader;
import org.apache.doris.flink.lookup.LookupMetrics;
import org.apache.doris.flink.lookup.LookupSchema;
@@ -37,18 +38,19 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
-import java.util.List;
import java.util.concurrent.TimeUnit;
-/** use jdbc to query. */
-public class DorisRowDataJdbcLookupFunction extends TableFunction {
+/** lookup {@link TableFunction} for 1.15. */
+public class DorisRowDataJdbcLookupFunction extends TableFunction
+ implements DorisLookupFunction {
private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataJdbcLookupFunction.class);
private final DorisOptions options;
private final DorisLookupOptions lookupOptions;
private final long cacheMaxSize;
private final long cacheExpireMs;
- private transient Cache> cache;
+ private transient Cache> cache;
private DorisLookupReader lookupReader;
private LookupSchema lookupSchema;
private LookupMetrics lookupMetrics;
@@ -97,7 +99,7 @@ public void open(FunctionContext context) throws Exception {
public void eval(Object... keys) throws IOException {
RowData keyRow = GenericRowData.of(keys);
if (cache != null) {
- List cachedRows = cache.getIfPresent(keyRow);
+ Collection cachedRows = cache.getIfPresent(keyRow);
if (cachedRows != null) {
lookupMetrics.incHitCount();
if (LOG.isDebugEnabled()) {
@@ -115,7 +117,7 @@ public void eval(Object... keys) throws IOException {
}
private void queryRecord(RowData keyRow) throws IOException {
- List rowData = lookupReader.get(keyRow);
+ Collection rowData = lookupReader.get(keyRow);
if (rowData == null) {
rowData = Collections.emptyList();
}
@@ -135,7 +137,7 @@ public void close() throws Exception {
}
@VisibleForTesting
- public Cache> getCache() {
+ public Cache> getCache() {
return cache;
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java b/flink-doris-connector/flink-doris-connector-1.15/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
similarity index 56%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
rename to flink-doris-connector/flink-doris-connector-1.15/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
index 6d569bcfc..a4383a31d 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
+++ b/flink-doris-connector/flink-doris-connector-1.15/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
@@ -30,6 +30,7 @@
import org.apache.doris.flink.container.AbstractITCaseService;
import org.apache.doris.flink.container.ContainerUtils;
import org.apache.doris.flink.table.DorisConfigOptions;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,12 +44,21 @@ public class DorisLookupTableITCase extends AbstractITCaseService {
private static final Logger LOG = LoggerFactory.getLogger(DorisLookupTableITCase.class);
private static final String DATABASE = "test_lookup";
private static final String TABLE_READ_TBL = "tbl_read_tbl";
+ private static final String TABLE_DIM_TBL = "tbl_dim_tbl";
+
+ private StreamExecutionEnvironment env;
+ private StreamTableEnvironment tEnv;
+
+ @Before
+ public void before() {
+ env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(DEFAULT_PARALLELISM);
+ tEnv = StreamTableEnvironment.create(env);
+ }
@Test
public void testLookupTable() throws Exception {
initializeTable();
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(DEFAULT_PARALLELISM);
DataStreamSource sourceStream = env.fromElements(1, 2, 3, 4);
final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
Schema schema =
@@ -138,4 +148,106 @@ private void initializeTable() {
"insert into %s.%s values (3,-106,-14878,1466614815449373200)",
DATABASE, DorisLookupTableITCase.TABLE_READ_TBL));
}
+
+ @Test
+ public void testLookup() throws Exception {
+ initFlinkTable();
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .columnByExpression("proctime", "PROCTIME()")
+ .build();
+
+ Table table = tEnv.fromDataStream(env.fromElements(1, 2, 3), schema);
+ tEnv.createTemporaryView("fact_table", table);
+
+ String query =
+ "select fact_table.f0,"
+ + "doris_lookup.score"
+ + " from fact_table"
+ + " left join doris_lookup FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id";
+ TableResult tableResult = tEnv.executeSql(query);
+ CloseableIterator collectIter = tableResult.collect();
+ List actual = collectSize(collectIter, 3);
+ String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, null]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ collectIter.close();
+
+ // mock data
+ tEnv.dropTemporaryView("fact_table");
+ tEnv.createTemporaryView(
+ "fact_table", tEnv.fromDataStream(env.fromElements(1, 2, 3, 4), schema));
+
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format(
+ "insert into %s.%s values (3,300)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL));
+
+ TableResult tableResult2 = tEnv.executeSql(query);
+ CloseableIterator collectIter2 = tableResult2.collect();
+ actual = collectSize(collectIter2, 4);
+ expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, 300]", "+I[4, null]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ collectIter2.close();
+ }
+
+ private static List collectSize(CloseableIterator iterator, int rows)
+ throws Exception {
+ List result = new ArrayList<>();
+ while (result.size() < rows && iterator.hasNext()) {
+ result.add(iterator.next().toString());
+ }
+ return result;
+ }
+
+ private void initFlinkTable() {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format(
+ "DROP TABLE IF EXISTS %s.%s",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`id` int(11),\n"
+ + "`score` int(11)\n"
+ + ") DISTRIBUTED BY HASH(`id`) BUCKETS 4\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL),
+ String.format(
+ "insert into %s.%s values (1,100)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL),
+ String.format(
+ "insert into %s.%s values (2,200)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL));
+
+ String lookupDDL =
+ String.format(
+ "CREATE TABLE `doris_lookup`("
+ + " `id` INTEGER,"
+ + " `score` INTEGER,"
+ + " PRIMARY KEY (`id`) NOT ENFORCED"
+ + ") WITH ("
+ + "'connector' = '"
+ + DorisConfigOptions.IDENTIFIER
+ + "',"
+ + "'fenodes' = '%s',"
+ + "'jdbc-url' = '%s',"
+ + "'table.identifier' = '%s',"
+ + "'username' = '%s',"
+ + "'password' = '%s'"
+ + ")",
+ getFenodes(),
+ getDorisQueryUrl(),
+ DATABASE + "." + TABLE_DIM_TBL,
+ getDorisUsername(),
+ getDorisPassword());
+
+ tEnv.executeSql(lookupDDL);
+ }
}
diff --git a/flink-doris-connector/flink-doris-connector-1.16/pom.xml b/flink-doris-connector/flink-doris-connector-1.16/pom.xml
new file mode 100644
index 000000000..b0180f7ab
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.16/pom.xml
@@ -0,0 +1,43 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-1.16
+ Flink Doris Connector Flink 1.16
+
+
+ 1.16.0
+ 1.16
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-1.17/pom.xml b/flink-doris-connector/flink-doris-connector-1.17/pom.xml
new file mode 100644
index 000000000..7e88e81a1
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.17/pom.xml
@@ -0,0 +1,43 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-1.17
+ Flink Doris Connector Flink 1.17
+
+
+ 1.17.0
+ 1.17
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-1.18/pom.xml b/flink-doris-connector/flink-doris-connector-1.18/pom.xml
new file mode 100644
index 000000000..ce75a66b0
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.18/pom.xml
@@ -0,0 +1,43 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-1.18
+ Flink Doris Connector Flink 1.18
+
+
+ 1.18.0
+ 1.18
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-1.19/pom.xml b/flink-doris-connector/flink-doris-connector-1.19/pom.xml
new file mode 100644
index 000000000..0dc9dbb1c
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.19/pom.xml
@@ -0,0 +1,43 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-1.19
+ Flink Doris Connector Flink 1.19
+
+
+ 1.19.0
+ 1.19
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-1.20/pom.xml b/flink-doris-connector/flink-doris-connector-1.20/pom.xml
new file mode 100644
index 000000000..39ab1eaef
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/pom.xml
@@ -0,0 +1,44 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-1.20
+ Flink Doris Connector Flink 1.20
+
+
+ 1.20.0
+ 1.20
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-common
+ ${project.version}
+
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-common/pom.xml b/flink-doris-connector/flink-doris-connector-common/pom.xml
new file mode 100644
index 000000000..78764d20e
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/pom.xml
@@ -0,0 +1,47 @@
+
+
+
+ 4.0.0
+
+ org.apache.doris
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+
+ flink-doris-connector-common
+ Flink Doris Connector Common
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ test-jar
+
+
+
+
+
+
+
+
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/backend/BackendClient.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/backend/BackendClient.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CreateTableException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CreateTableException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
similarity index 94%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
index 958aa9e34..12a8bc062 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
@@ -30,6 +30,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
@@ -65,8 +66,8 @@ private DorisRowConverter buildKeyConvert() {
}
@Override
- public CompletableFuture> asyncGet(RowData recordIn) throws IOException {
- CompletableFuture> result = new CompletableFuture<>();
+ public CompletableFuture> asyncGet(RowData recordIn) throws IOException {
+ CompletableFuture> result = new CompletableFuture<>();
Record record = convertRecord(recordIn);
try {
pool.get(new Get(record))
@@ -125,7 +126,7 @@ private Record convertRecord(RowData recordIn) {
}
@Override
- public List get(RowData record) throws IOException {
+ public Collection get(RowData record) throws IOException {
try {
return this.asyncGet(record).get();
} catch (InterruptedException | ExecutionException e) {
diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java
new file mode 100644
index 000000000..2c9e07102
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java
@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.lookup;
+
+/** Doris lookup function. */
+public interface DorisLookupFunction {}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
similarity index 82%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
index 7c8663e0e..3bf1c9d07 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
@@ -21,12 +21,13 @@
import java.io.Closeable;
import java.io.IOException;
-import java.util.List;
+import java.util.Collection;
import java.util.concurrent.CompletableFuture;
public abstract class DorisLookupReader implements Closeable {
- public abstract CompletableFuture> asyncGet(RowData record) throws IOException;
+ public abstract CompletableFuture> asyncGet(RowData record)
+ throws IOException;
- public abstract List get(RowData record) throws IOException;
+ public abstract Collection get(RowData record) throws IOException;
}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Get.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Get.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/GetAction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/GetAction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java
diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java
new file mode 100644
index 000000000..ea77615a6
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.lookup;
+
+import org.apache.flink.table.connector.source.LookupTableSource;
+import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider;
+import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider;
+import org.apache.flink.table.functions.AsyncLookupFunction;
+import org.apache.flink.table.functions.LookupFunction;
+
+/** Factory to create {@link LookupTableSource.LookupRuntimeProvider}. */
+public class LookupRuntimeProviderFactory {
+
+ public static LookupTableSource.LookupRuntimeProvider create(
+ DorisLookupFunction function, boolean enableAsync) {
+ return enableAsync
+ ? AsyncLookupFunctionProvider.of((AsyncLookupFunction) function)
+ : LookupFunctionProvider.of((LookupFunction) function);
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Record.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Record.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Worker.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Worker.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/RestService.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/RestService.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Field.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Field.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Schema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Schema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/Routing.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/Routing.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisSink.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisSink.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/EventType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/EventType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/DorisSource.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/DorisSource.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
similarity index 96%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
index e4f27d8bc..e7a5a95ba 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
@@ -21,13 +21,11 @@
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.connector.Projection;
-import org.apache.flink.table.connector.source.AsyncTableFunctionProvider;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.connector.source.InputFormatProvider;
import org.apache.flink.table.connector.source.LookupTableSource;
import org.apache.flink.table.connector.source.ScanTableSource;
import org.apache.flink.table.connector.source.SourceProvider;
-import org.apache.flink.table.connector.source.TableFunctionProvider;
import org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown;
import org.apache.flink.table.connector.source.abilities.SupportsLimitPushDown;
import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown;
@@ -42,6 +40,8 @@
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.deserialization.RowDataDeserializationSchema;
import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.lookup.DorisLookupFunction;
+import org.apache.doris.flink.lookup.LookupRuntimeProviderFactory;
import org.apache.doris.flink.rest.PartitionDefinition;
import org.apache.doris.flink.rest.RestService;
import org.apache.doris.flink.source.DorisSource;
@@ -151,8 +151,10 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) {
keyNames[i] = DataType.getFieldNames(physicalRowDataType).get(innerKeyArr[0]);
keyIndexs[i] = innerKeyArr[0];
}
+
+ DorisLookupFunction lookupFunction;
if (lookupOptions.isAsync()) {
- return AsyncTableFunctionProvider.of(
+ lookupFunction =
new DorisRowDataAsyncLookupFunction(
options,
lookupOptions,
@@ -160,9 +162,9 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) {
DataType.getFieldDataTypes(physicalRowDataType)
.toArray(new DataType[0]),
keyNames,
- keyIndexs));
+ keyIndexs);
} else {
- return TableFunctionProvider.of(
+ lookupFunction =
new DorisRowDataJdbcLookupFunction(
options,
lookupOptions,
@@ -170,8 +172,9 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) {
DataType.getFieldDataTypes(physicalRowDataType)
.toArray(new DataType[0]),
keyNames,
- keyIndexs));
+ keyIndexs);
}
+ return LookupRuntimeProviderFactory.create(lookupFunction, lookupOptions.isAsync());
}
@Override
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java
diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
new file mode 100644
index 000000000..4e0c87b8e
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.AsyncLookupFunction;
+import org.apache.flink.table.functions.AsyncTableFunction;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.util.Preconditions;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
+import org.apache.doris.flink.lookup.DorisLookupFunction;
+import org.apache.doris.flink.lookup.DorisLookupReader;
+import org.apache.doris.flink.lookup.LookupMetrics;
+import org.apache.doris.flink.lookup.LookupSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+/** async lookup {@link AsyncTableFunction} for after 1.16 */
+public class DorisRowDataAsyncLookupFunction extends AsyncLookupFunction
+ implements DorisLookupFunction {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DorisRowDataAsyncLookupFunction.class);
+ private final DorisOptions options;
+ private final DorisLookupOptions lookupOptions;
+ private final long cacheMaxSize;
+ private final long cacheExpireMs;
+ private transient Cache> cache;
+ private DorisLookupReader lookupReader;
+ private LookupSchema lookupSchema;
+ private LookupMetrics lookupMetrics;
+
+ public DorisRowDataAsyncLookupFunction(
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
+ Preconditions.checkNotNull(
+ options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup");
+ this.options = options;
+ this.cacheMaxSize = lookupOptions.getCacheMaxSize();
+ this.cacheExpireMs = lookupOptions.getCacheExpireMs();
+ this.lookupOptions = lookupOptions;
+ this.lookupSchema =
+ new LookupSchema(
+ options.getTableIdentifier(),
+ selectFields,
+ fieldTypes,
+ conditionFields,
+ keyIndex);
+ }
+
+ @Override
+ public void open(FunctionContext context) throws Exception {
+ super.open(context);
+ LOG.info(
+ "lookup options: threadSize {}, batchSize {}, queueSize {}",
+ lookupOptions.getJdbcReadThreadSize(),
+ lookupOptions.getJdbcReadBatchSize(),
+ lookupOptions.getJdbcReadBatchQueueSize());
+ this.cache =
+ cacheMaxSize == -1 || cacheExpireMs == -1
+ ? null
+ : CacheBuilder.newBuilder()
+ .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
+ .maximumSize(cacheMaxSize)
+ .build();
+ this.lookupReader = new DorisJdbcLookupReader(options, lookupOptions, lookupSchema);
+ this.lookupMetrics = new LookupMetrics(context.getMetricGroup());
+ }
+
+ @Override
+ public CompletableFuture> asyncLookup(RowData keyRow) {
+ CompletableFuture> resultFuture = new CompletableFuture<>();
+ if (cache != null) {
+ List cachedRows = cache.getIfPresent(keyRow);
+ if (cachedRows != null) {
+ lookupMetrics.incHitCount();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("lookup cache hit for key: {}", keyRow);
+ }
+ resultFuture.complete(cachedRows);
+ return resultFuture;
+ } else {
+ lookupMetrics.incMissCount();
+ }
+ }
+
+ CompletableFuture> future;
+ try {
+ future = lookupReader.asyncGet(keyRow);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ resultFuture.handleAsync(
+ (resultRows, throwable) -> {
+ try {
+ if (throwable != null) {
+ future.completeExceptionally(throwable);
+ } else {
+ if (resultRows == null || resultRows.isEmpty()) {
+ if (cache != null) {
+ cache.put(keyRow, Collections.emptyList());
+ lookupMetrics.incLoadCount();
+ }
+ future.complete(Collections.emptyList());
+ } else {
+ if (cache != null) {
+ cache.put(keyRow, (List) resultRows);
+ lookupMetrics.incLoadCount();
+ }
+ future.complete(resultRows);
+ }
+ }
+ } catch (Throwable t) {
+ future.completeExceptionally(t);
+ }
+ return null;
+ });
+ return future;
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ if (lookupReader != null) {
+ lookupReader.close();
+ }
+ }
+
+ @VisibleForTesting
+ public Cache> getCache() {
+ return cache;
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
new file mode 100644
index 000000000..70fefa46c
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.LookupFunction;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.util.Preconditions;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.lookup.DorisJdbcLookupReader;
+import org.apache.doris.flink.lookup.DorisLookupFunction;
+import org.apache.doris.flink.lookup.DorisLookupReader;
+import org.apache.doris.flink.lookup.LookupMetrics;
+import org.apache.doris.flink.lookup.LookupSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+/** lookup {@link LookupFunction} for after 1.16 */
+public class DorisRowDataJdbcLookupFunction extends LookupFunction implements DorisLookupFunction {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataJdbcLookupFunction.class);
+ private final DorisOptions options;
+ private final DorisLookupOptions lookupOptions;
+ private final long cacheMaxSize;
+ private final long cacheExpireMs;
+ private transient Cache> cache;
+ private DorisLookupReader lookupReader;
+ private LookupSchema lookupSchema;
+ private LookupMetrics lookupMetrics;
+
+ public DorisRowDataJdbcLookupFunction(
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
+ Preconditions.checkNotNull(
+ options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup");
+ this.options = options;
+ this.cacheMaxSize = lookupOptions.getCacheMaxSize();
+ this.cacheExpireMs = lookupOptions.getCacheExpireMs();
+ this.lookupOptions = lookupOptions;
+ this.lookupSchema =
+ new LookupSchema(
+ options.getTableIdentifier(),
+ selectFields,
+ fieldTypes,
+ conditionFields,
+ keyIndex);
+ }
+
+ @Override
+ public void open(FunctionContext context) throws Exception {
+ super.open(context);
+ this.cache =
+ cacheMaxSize == -1 || cacheExpireMs == -1
+ ? null
+ : CacheBuilder.newBuilder()
+ .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS)
+ .maximumSize(cacheMaxSize)
+ .build();
+ this.lookupReader = new DorisJdbcLookupReader(options, lookupOptions, lookupSchema);
+ this.lookupMetrics = new LookupMetrics(context.getMetricGroup());
+ }
+
+ @Override
+ public Collection lookup(RowData keyRow) throws IOException {
+ if (cache != null) {
+ Collection cachedRows = cache.getIfPresent(keyRow);
+ if (cachedRows != null) {
+ lookupMetrics.incHitCount();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("lookup cache hit for key: {}", keyRow);
+ }
+ return cachedRows;
+ } else {
+ lookupMetrics.incMissCount();
+ }
+ }
+ return queryRecord(keyRow);
+ }
+
+ private Collection queryRecord(RowData keyRow) throws IOException {
+ Collection rowData = lookupReader.get(keyRow);
+ if (rowData == null) {
+ rowData = Collections.emptyList();
+ }
+ if (cache != null) {
+ cache.put(keyRow, rowData);
+ lookupMetrics.incLoadCount();
+ }
+ return rowData;
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ if (lookupReader != null) {
+ lookupReader.close();
+ }
+ }
+
+ @VisibleForTesting
+ public Cache> getCache() {
+ return cache;
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/FastDateUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/FastDateUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/IPUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/IPUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java
diff --git a/flink-doris-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-doris-connector/flink-doris-connector-common/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
similarity index 100%
rename from flink-doris-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
rename to flink-doris-connector/flink-doris-connector-common/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
diff --git a/flink-doris-connector/src/main/resources/log4j2.properties b/flink-doris-connector/flink-doris-connector-common/src/main/resources/log4j2.properties
similarity index 100%
rename from flink-doris-connector/src/main/resources/log4j2.properties
rename to flink-doris-connector/flink-doris-connector-common/src/main/resources/log4j2.properties
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
similarity index 81%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
index 8fe017e1c..700ecd671 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
@@ -44,18 +44,24 @@ public static void main(String[] args) throws Exception {
JsonDebeziumDeserializationSchema schema =
new JsonDebeziumDeserializationSchema(false, customConverterConfigs);
+ Properties defaultProps = DateToStringConverter.DEFAULT_PROPS;
+ // defaultProps.put(
+ // "column.include.list",
+ // "test.test_column.id,test.test_column.c1,test.test_column.c2");
MySqlSource mySqlSource =
MySqlSource.builder()
.hostname("127.0.0.1")
.port(3306)
.databaseList("test") // set captured database
- .tableList("test.t1") // set captured table
+ .tableList("test.test_flink") // set captured table
.username("root")
.password("123456")
- .debeziumProperties(DateToStringConverter.DEFAULT_PROPS)
- .deserializer(schema)
- .serverTimeZone("Asia/Shanghai")
- .includeSchemaChanges(true) // converts SourceRecord to JSON String
+ .debeziumProperties(defaultProps)
+ // .deserializer(schema)
+ .deserializer(new JsonDebeziumDeserializationSchema())
+ // .startupOptions(StartupOptions.latest())
+ // .serverTimeZone("Asia/Shanghai")
+ // .includeSchemaChanges(true) // converts SourceRecord to JSON String
.build();
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
@@ -67,8 +73,8 @@ public static void main(String[] args) throws Exception {
props.setProperty("read_json_by_line", "true");
DorisOptions dorisOptions =
DorisOptions.builder()
- .setFenodes("127.0.0.1:8030")
- .setTableIdentifier("test.t1")
+ .setFenodes("10.16.10.6:28737")
+ .setTableIdentifier("test.test_flink_a")
.setUsername("root")
.setPassword("")
.build();
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CatalogExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CatalogExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
similarity index 90%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
index 7a111b258..70963e052 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
@@ -26,7 +26,6 @@
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.doris.flink.sink.batch.DorisBatchSink;
-import org.apache.doris.flink.sink.writer.WriteMode;
import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer;
import java.util.Arrays;
@@ -38,7 +37,7 @@ public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
// env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.enableCheckpointing(5000);
+ env.enableCheckpointing(30000);
// env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
// env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5,
@@ -61,20 +60,17 @@ public static void main(String[] args) throws Exception {
properties.setProperty("format", "csv");
DorisOptions.Builder dorisBuilder = DorisOptions.builder();
dorisBuilder
- .setFenodes("127.0.0.1:8030")
- .setTableIdentifier("test.test_flink")
+ .setFenodes("10.16.10.6:28737")
+ .setTableIdentifier("test.test_flink_error")
.setUsername("root")
.setPassword("");
DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
- executionBuilder
- .setLabelPrefix("label")
- .setStreamLoadProp(properties)
- .setDeletable(false)
- .setBufferFlushMaxBytes(8 * 1024)
- .setBufferFlushMaxRows(900)
- .setBufferFlushIntervalMs(1000 * 10)
- // .setBatchMode(true);
- .setWriteMode(WriteMode.STREAM_LOAD_BATCH);
+ executionBuilder.setLabelPrefix("label").setStreamLoadProp(properties).setDeletable(false);
+ // .setBufferFlushMaxBytes(8 * 102400000)
+ // .setBufferFlushMaxRows(90000)
+ // .setBufferFlushIntervalMs(1000 * 10)
+ // .setBatchMode(true);
+ // .setWriteMode(WriteMode.STREAM_LOAD_BATCH);
builder.setDorisReadOptions(readOptionBuilder.build())
.setDorisExecutionOptions(executionBuilder.build())
.setSerializer(new SimpleStringSerializer())
@@ -88,7 +84,7 @@ public static void main(String[] args) throws Exception {
public void run(SourceContext out) throws Exception {
while (true) {
id = id + 1;
- String record = id + "," + UUID.randomUUID() + "," + id + "";
+ String record = UUID.randomUUID() + "," + id + "";
out.collect(record);
Thread.sleep(500);
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
similarity index 76%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
index bcb169656..dcbd0eef3 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
@@ -17,15 +17,11 @@
package org.apache.doris.flink.example;
-import org.apache.flink.api.common.RuntimeExecutionMode;
-import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
@@ -36,11 +32,12 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import java.util.UUID;
public class DorisSinkExample {
public static void main(String[] args) throws Exception {
- JSONFormatWrite();
+ CSVFormatWrite();
}
public static void JSONFormatWrite() throws Exception {
@@ -85,12 +82,14 @@ public static void JSONFormatWrite() throws Exception {
public static void CSVFormatWrite() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setRuntimeMode(RuntimeExecutionMode.BATCH);
env.enableCheckpointing(10000);
+ env.setParallelism(1);
env.getCheckpointConfig()
.enableExternalizedCheckpoints(
CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
- env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(30000)));
+ // env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5,
+ // Time.milliseconds(30000)));
+ env.setRestartStrategy(RestartStrategies.noRestart());
DorisSink.Builder builder = DorisSink.builder();
Properties properties = new Properties();
properties.setProperty("column_separator", ",");
@@ -98,27 +97,41 @@ public static void CSVFormatWrite() throws Exception {
properties.setProperty("format", "csv");
DorisOptions.Builder dorisBuilder = DorisOptions.builder();
dorisBuilder
- .setFenodes("127.0.0.1:8030")
- .setTableIdentifier("db.table")
- .setUsername("test")
- .setPassword("test");
+ .setFenodes("10.16.10.6:48737")
+ .setTableIdentifier("test.test_flink")
+ .setUsername("root")
+ .setPassword("");
DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
executionBuilder
.setLabelPrefix("label-doris")
- .setStreamLoadProp(properties)
- .setBufferSize(8 * 1024)
- .setBufferCount(3);
+ .setDeletable(false)
+ // .setBatchMode(true)
+ .setStreamLoadProp(properties);
builder.setDorisReadOptions(DorisReadOptions.builder().build())
.setDorisExecutionOptions(executionBuilder.build())
.setSerializer(new SimpleStringSerializer())
.setDorisOptions(dorisBuilder.build());
- List> data = new ArrayList<>();
- data.add(new Tuple2<>("doris", 1));
- DataStreamSource> source = env.fromCollection(data);
- source.map((MapFunction, String>) t -> t.f0 + "," + t.f1)
+ env.addSource(
+ new SourceFunction() {
+ private Long id = 0L;
+
+ @Override
+ public void run(SourceContext out) throws Exception {
+ while (true) {
+ id = id + 1;
+ String record = UUID.randomUUID() + "," + id + "";
+ out.collect(record);
+ Thread.sleep(500);
+ }
+ }
+
+ @Override
+ public void cancel() {}
+ })
.sinkTo(builder.build());
+
env.execute("doris test");
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
similarity index 60%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
index a31ab8a55..856c8c8f6 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
@@ -18,28 +18,36 @@
package org.apache.doris.flink.example;
import org.apache.flink.api.common.RuntimeExecutionMode;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.flink.table.api.Expressions.$;
-
public class DorisSinkSQLExample {
public static void main(String[] args) {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
+ env.enableCheckpointing(10000);
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
- List> data = new ArrayList<>();
- data.add(new Tuple2<>("doris", 1));
- DataStreamSource> source = env.fromCollection(data);
- tEnv.createTemporaryView("doris_test", source, $("name"), $("age"));
+ tEnv.executeSql(
+ "CREATE TABLE doris_test_source ("
+ + "name STRING,"
+ + "age INT"
+ + ") "
+ + "WITH (\n"
+ + " 'connector' = 'doris',\n"
+ + " 'fenodes' = '10.16.10.6:28737',\n"
+ + " 'table.identifier' = 'test.test_flink_10bucket',\n"
+ + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n"
+ + " 'username' = 'root',\n"
+ + " 'password' = '',\n"
+ + " 'sink.properties.format' = 'json',\n"
+ + " 'sink.buffer-count' = '4',\n"
+ + " 'sink.buffer-size' = '4086',"
+ + " 'sink.label-prefix' = 'doris_lab1el112121',\n"
+ + " 'sink.properties.read_json_by_line' = 'true'\n"
+ + ")");
tEnv.executeSql(
"CREATE TABLE doris_test_sink ("
@@ -48,16 +56,18 @@ public static void main(String[] args) {
+ ") "
+ "WITH (\n"
+ " 'connector' = 'doris',\n"
- + " 'fenodes' = 'FE_IP:8030',\n"
- + " 'table.identifier' = 'db.table',\n"
+ + " 'fenodes' = '10.16.10.6:28737',\n"
+ + " 'table.identifier' = 'test.test_flink_a',\n"
+ + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n"
+ " 'username' = 'root',\n"
+ " 'password' = '',\n"
+ " 'sink.properties.format' = 'json',\n"
+ " 'sink.buffer-count' = '4',\n"
+ " 'sink.buffer-size' = '4086',"
- + " 'sink.label-prefix' = 'doris_label',\n"
+ + " 'sink.label-prefix' = 'doris_lab1el4',\n"
+ " 'sink.properties.read_json_by_line' = 'true'\n"
+ ")");
- tEnv.executeSql("INSERT INTO doris_test_sink select name,age from doris_test");
+
+ tEnv.executeSql("INSERT OVERWRITE doris_test_sink select 'zhangsan',1");
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
similarity index 60%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
index 90e461cfc..ba756d269 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
@@ -20,45 +20,46 @@
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.types.Row;
public class DorisSourceExample {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
+ env.setParallelism(2);
final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
// register a table in the catalog
tEnv.executeSql(
"CREATE TABLE doris_source ("
- + "bigint_1 BIGINT,"
- + "char_1 STRING,"
- + "date_1 STRING,"
- + "datetime_1 STRING,"
- + "decimal_1 DECIMAL(5,2),"
- + "double_1 DOUBLE,"
- + "float_1 FLOAT ,"
- + "int_1 INT ,"
- + "largeint_1 STRING, "
- + "smallint_1 SMALLINT, "
- + "tinyint_1 TINYINT, "
- + "varchar_1 STRING "
+ + "name STRING,"
+ + "age int"
+ ") "
+ "WITH (\n"
+ " 'connector' = 'doris',\n"
- + " 'fenodes' = 'FE_IP:8030',\n"
- + " 'table.identifier' = 'db.table',\n"
+ + " 'fenodes' = '10.16.10.6:28737',\n"
+ + " 'table.identifier' = 'test.test_flink_bucket',\n"
+ " 'username' = 'root',\n"
- + " 'password' = ''\n"
+ + " 'password' = '',\n"
+ // + " 'doris.filter.query' = ' (dt =
+ // DATE_FORMAT(TIMESTAMPADD(DAY , -7, NOW()), ''yyyy-MM-dd'')) ',\n"
+ + " 'source.use-flight-sql' = 'true',\n"
+ + " 'doris.request.tablet.size' = '1',\n"
+ + " 'source.flight-sql-port' = '29747'"
+ ")");
// define a dynamic aggregating query
- final Table result = tEnv.sqlQuery("SELECT * from doris_source ");
+ final Table result = tEnv.sqlQuery(" SELECT * from doris_source limit 1 ");
+ // TableResult tableResult = tEnv.executeSql("explain PLAN FOR SELECT * from
+ // doris_source limit 10 ");
+ // tableResult.print();
+ // final Table result = tEnv.sqlQuery("SELECT * from ( select name,age as c,dt from
+ // doris_source) tb where c >2");
// print the result to the console
- tEnv.toDataStream(result).print();
+ tEnv.toRetractStream(result, Row.class).print();
env.execute();
}
}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
diff --git a/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
new file mode 100644
index 000000000..535bcf911
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.example;
+
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.types.Row;
+
+public class LookupJoinCdcExample {
+ public static void main(String[] args) throws Exception {
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+ // env.disableOperatorChaining();
+ env.enableCheckpointing(10000);
+ final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
+
+ tEnv.executeSql(
+ "CREATE TABLE mysql_tb ("
+ + "id INT,"
+ + "name STRING,"
+ + "create_time TIMESTAMP(3),"
+ + "WATERMARK FOR create_time AS create_time - INTERVAL '60' SECOND,"
+ + "primary key(id) NOT ENFORCED"
+ + ") "
+ + "WITH (\n"
+ + " 'connector' = 'mysql-cdc',\n"
+ + " 'hostname' = '127.0.0.1',\n"
+ + " 'port' = '3306',\n"
+ + " 'username' = 'root',\n"
+ + " 'password' = '123456',\n"
+ + " 'database-name' = 'test',\n"
+ + " 'scan.startup.mode' = 'latest-offset',\n"
+ + " 'server-time-zone' = 'Asia/Shanghai',\n"
+ + " 'table-name' = 'student' "
+ + ")");
+
+ tEnv.executeSql(
+ "CREATE TABLE mysql_tb1 ("
+ + "id INT,"
+ + "name STRING,"
+ + "age STRING,"
+ + "create_time TIMESTAMP(3),"
+ + "WATERMARK FOR create_time AS create_time - INTERVAL '60' SECOND,"
+ + "primary key(id) NOT ENFORCED"
+ + ") "
+ + "WITH (\n"
+ + " 'connector' = 'mysql-cdc',\n"
+ + " 'hostname' = '127.0.0.1',\n"
+ + " 'port' = '3306',\n"
+ + " 'username' = 'root',\n"
+ + " 'password' = '123456',\n"
+ + " 'database-name' = 'test',\n"
+ + " 'scan.startup.mode' = 'latest-offset',\n"
+ + " 'server-time-zone' = 'Asia/Shanghai',\n"
+ + " 'table-name' = 'student_copy1' "
+ + ")");
+
+ // tEnv.executeSql(
+ // "CREATE TABLE doris_tb ("
+ // + "name STRING,"
+ // + "age INT,"
+ // + "primary key(name) NOT ENFORCED"
+ // + ") "
+ // + "WITH (\n"
+ // + " 'connector' = 'doris',\n"
+ // + " 'fenodes' = '10.16.10.6:28737',\n"
+ // + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n"
+ // + " 'table.identifier' = 'test.student',\n"
+ //// + " 'lookup.cache.max-rows' = '1000',"
+ //// + " 'lookup.cache.ttl' = '1 hour',"
+ // // + " 'lookup.jdbc.async' = 'true',\n"
+ // + " 'username' = 'root',\n"
+ // + " 'password' = ''\n"
+ // + ")");
+
+ // tEnv.executeSql(
+ // "CREATE TABLE doris_tb ("
+ // + "name STRING,"
+ // + "age INT,"
+ // + "primary key(name) NOT ENFORCED"
+ // + ") "
+ // + "WITH (\n"
+ // + " 'connector' = 'jdbc',\n"
+ // + " 'url' = 'jdbc:mysql://10.16.10.6:29737/test',"
+ // + " 'table-name' = 'test_flink_a',\n"
+ // + " 'username' = 'root',\n"
+ // + " 'password' = ''\n"
+ // + ")");
+
+ Table table =
+ tEnv.sqlQuery(
+ "SELECT a.id, a.name, b.age\n"
+ + "FROM mysql_tb a\n"
+ + " left join mysql_tb1 FOR SYSTEM_TIME AS OF a.create_time AS b\n"
+ + " ON a.name = b.name");
+
+ tEnv.toRetractStream(table, Row.class).print();
+
+ env.execute();
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
similarity index 88%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
index 9e82c6d41..ffca3e8a6 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
@@ -41,7 +41,7 @@ public static void main(String[] args) throws Exception {
DataStreamSource> source =
env.addSource(
new SourceFunction>() {
- private Integer id = 1;
+ private Integer id = 3000000;
@Override
public void run(SourceContext> out)
@@ -73,8 +73,8 @@ public void cancel() {}
+ ") "
+ "WITH (\n"
+ " 'connector' = 'doris',\n"
- + " 'fenodes' = '127.0.0.1:8030',\n"
- + " 'jdbc-url' = 'jdbc:mysql://127.0.0.1:9030',\n"
+ + " 'fenodes' = '10.16.10.6:28737',\n"
+ + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n"
+ " 'table.identifier' = 'ssb.customer',\n"
+ " 'lookup.jdbc.async' = 'true',\n"
+ " 'username' = 'root',\n"
@@ -83,7 +83,8 @@ public void cancel() {}
Table table =
tEnv.sqlQuery(
- "select a.id,a.uuid,b.c_name,b.c_nation,b.c_phone from doris_source a "
+ "select /*+ LOOKUP('table'='b', 'async'='true', 'output-mode' = 'allow_unordered', 'retry-predicate'='lookup_miss','retry-strategy'='fixed_delay', 'fixed-delay'='1s', 'max-attempts'='10') */"
+ + " a.id,a.uuid,b.c_name,b.c_nation,b.c_phone from doris_source a "
+ "left join lookup_dim_tbl FOR SYSTEM_TIME AS OF a.process_time b "
+ "ON a.id = b.c_custkey");
diff --git a/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
new file mode 100644
index 000000000..7cdf2608f
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.lookup;
+
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.types.Row;
+import org.apache.flink.util.CloseableIterator;
+
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
+import org.apache.doris.flink.table.DorisConfigOptions;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class DorisLookupTableITCase extends AbstractITCaseService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DorisLookupTableITCase.class);
+ private static final String DATABASE = "test_lookup";
+ private static final String TABLE_READ_TBL = "tbl_read_tbl";
+ private static final String TABLE_DIM_TBL = "tbl_dim_tbl";
+
+ private StreamExecutionEnvironment env;
+ private StreamTableEnvironment tEnv;
+
+ @Before
+ public void before() {
+ env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(DEFAULT_PARALLELISM);
+ tEnv = StreamTableEnvironment.create(env);
+ }
+
+ @Test
+ public void testLookupTable() throws Exception {
+ initializeTable();
+ DataStreamSource sourceStream = env.fromElements(1, 2, 3, 4);
+ final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .columnByExpression("proctime", "PROCTIME()")
+ .build();
+ Table table = tEnv.fromDataStream(sourceStream, schema);
+ tEnv.createTemporaryView("source", table);
+
+ String lookupDDL =
+ String.format(
+ "CREATE TABLE `doris_lookup`("
+ + " `id` INTEGER,"
+ + " `tinyintColumn` TINYINT,"
+ + " `smallintColumn` SMALLINT,"
+ + " `bigintColumn` BIGINT,"
+ + " PRIMARY KEY (`id`) NOT ENFORCED"
+ + ") WITH ("
+ + "'connector' = '"
+ + DorisConfigOptions.IDENTIFIER
+ + "',"
+ + "'fenodes' = '%s',"
+ + "'jdbc-url' = '%s',"
+ + "'table.identifier' = '%s',"
+ + "'username' = '%s',"
+ + "'password' = '%s',"
+ + "'lookup.cache.max-rows' = '100'"
+ + ")",
+ getFenodes(),
+ getDorisQueryUrl(),
+ DATABASE + "." + TABLE_READ_TBL,
+ getDorisUsername(),
+ getDorisPassword());
+ tEnv.executeSql(lookupDDL);
+ TableResult tableResult =
+ tEnv.executeSql(
+ "select source.f0,"
+ + "tinyintColumn,"
+ + "smallintColumn,"
+ + "bigintColumn"
+ + " from `source`"
+ + " inner join `doris_lookup` FOR SYSTEM_TIME AS OF source.proctime on source.f0 = doris_lookup.id");
+
+ List actual = new ArrayList<>();
+ try (CloseableIterator iterator = tableResult.collect()) {
+ while (iterator.hasNext()) {
+ actual.add(iterator.next().toString());
+ }
+ }
+
+ String[] expected =
+ new String[] {
+ "+I[1, 97, 27479, 8670353564751764000]",
+ "+I[2, 79, 17119, -4381380624467725000]",
+ "+I[3, -106, -14878, 1466614815449373200]"
+ };
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ }
+
+ private void initializeTable() {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format(
+ "DROP TABLE IF EXISTS %s.%s",
+ DATABASE, DorisLookupTableITCase.TABLE_READ_TBL),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`id` int(11),\n"
+ + "`tinyintColumn` tinyint(4),\n"
+ + "`smallintColumn` smallint(6),\n"
+ + "`bigintColumn` bigint(20),\n"
+ + ") DISTRIBUTED BY HASH(`id`) BUCKETS 10\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, DorisLookupTableITCase.TABLE_READ_TBL),
+ String.format(
+ "insert into %s.%s values (1,97,27479,8670353564751764000)",
+ DATABASE, DorisLookupTableITCase.TABLE_READ_TBL),
+ String.format(
+ "insert into %s.%s values (2,79,17119,-4381380624467725000)",
+ DATABASE, DorisLookupTableITCase.TABLE_READ_TBL),
+ String.format(
+ "insert into %s.%s values (3,-106,-14878,1466614815449373200)",
+ DATABASE, DorisLookupTableITCase.TABLE_READ_TBL));
+ }
+
+ @Test
+ public void testLookup() throws Exception {
+ initFlinkTable();
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .columnByExpression("proctime", "PROCTIME()")
+ .build();
+
+ Table table = tEnv.fromDataStream(env.fromElements(1, 2, 3), schema);
+ tEnv.createTemporaryView("fact_table", table);
+
+ String query =
+ "select fact_table.f0,"
+ + "doris_lookup.score"
+ + " from fact_table"
+ + " left join doris_lookup FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id";
+ TableResult tableResult = tEnv.executeSql(query);
+ CloseableIterator collectIter = tableResult.collect();
+ List actual = collectSize(collectIter, 3);
+ String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, null]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ collectIter.close();
+
+ // mock data
+ tEnv.dropTemporaryView("fact_table");
+ tEnv.createTemporaryView(
+ "fact_table", tEnv.fromDataStream(env.fromElements(1, 2, 3, 4), schema));
+
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format(
+ "insert into %s.%s values (3,300)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL));
+
+ TableResult tableResult2 = tEnv.executeSql(query);
+ CloseableIterator collectIter2 = tableResult2.collect();
+ actual = collectSize(collectIter2, 4);
+ expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, 300]", "+I[4, null]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ collectIter2.close();
+ }
+
+ @Test
+ public void testRetryLookup() throws Exception {
+ initFlinkTable();
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .columnByExpression("proctime", "PROCTIME()")
+ .build();
+
+ Table table = tEnv.fromDataStream(env.fromElements(1, 2, 3), schema);
+ tEnv.createTemporaryView("fact_table", table);
+
+ String query =
+ "select /*+ LOOKUP('table'='doris_lookup', 'retry-predicate'='lookup_miss', "
+ + " 'retry-strategy'='fixed_delay', 'fixed-delay'='1s','max-attempts'='60') */"
+ + " fact_table.f0,"
+ + "doris_lookup.score"
+ + " from fact_table"
+ + " left join doris_lookup FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id";
+ TableResult tableResult = tEnv.executeSql(query);
+ CloseableIterator collectIter = tableResult.collect();
+ // wait
+ Thread.sleep(10000);
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format(
+ "insert into %s.%s values (3,300)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL));
+
+ List actual = collectSize(collectIter, 3);
+ String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, 300]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ collectIter.close();
+ }
+
+ @Test
+ public void testAsyncRetryLookup() throws Exception {
+ initFlinkTable();
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .columnByExpression("proctime", "PROCTIME()")
+ .build();
+
+ Table table = tEnv.fromDataStream(env.fromElements(3, 2, 1), schema);
+ tEnv.createTemporaryView("fact_table", table);
+
+ String query =
+ "select /*+ LOOKUP('table'='doris_lookup', 'retry-predicate'='lookup_miss', "
+ + " 'retry-strategy'='fixed_delay', 'output-mode'='allow_unordered', 'fixed-delay'='1s','max-attempts'='60') */"
+ + " fact_table.f0,"
+ + "doris_lookup.score"
+ + " from fact_table"
+ + " left join doris_lookup /*+ OPTIONS('lookup.jdbc.async'='true') */ FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id";
+ TableResult tableResult = tEnv.executeSql(query);
+ CloseableIterator collectIter = tableResult.collect();
+ List actual = collectSize(collectIter, 2);
+ String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format(
+ "insert into %s.%s values (3,300)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL));
+ actual = collectSize(collectIter, 1);
+ expected = new String[] {"+I[3, 300]"};
+ assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray()));
+ collectIter.close();
+ }
+
+ private static List collectSize(CloseableIterator iterator, int rows)
+ throws Exception {
+ List result = new ArrayList<>();
+ while (result.size() < rows && iterator.hasNext()) {
+ result.add(iterator.next().toString());
+ }
+ return result;
+ }
+
+ private void initFlinkTable() {
+ ContainerUtils.executeSQLStatement(
+ getDorisQueryConnection(),
+ LOG,
+ String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE),
+ String.format(
+ "DROP TABLE IF EXISTS %s.%s",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL),
+ String.format(
+ "CREATE TABLE %s.%s ( \n"
+ + "`id` int(11),\n"
+ + "`score` int(11)\n"
+ + ") DISTRIBUTED BY HASH(`id`) BUCKETS 4\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ")\n",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL),
+ String.format(
+ "insert into %s.%s values (1,100)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL),
+ String.format(
+ "insert into %s.%s values (2,200)",
+ DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL));
+
+ String lookupDDL =
+ String.format(
+ "CREATE TABLE `doris_lookup`("
+ + " `id` INTEGER,"
+ + " `score` INTEGER,"
+ + " PRIMARY KEY (`id`) NOT ENFORCED"
+ + ") WITH ("
+ + "'connector' = '"
+ + DorisConfigOptions.IDENTIFIER
+ + "',"
+ + "'fenodes' = '%s',"
+ + "'jdbc-url' = '%s',"
+ + "'table.identifier' = '%s',"
+ + "'username' = '%s',"
+ + "'password' = '%s'"
+ + ")",
+ getFenodes(),
+ getDorisQueryUrl(),
+ DATABASE + "." + TABLE_DIM_TBL,
+ getDorisUsername(),
+ getDorisPassword());
+
+ tEnv.executeSql(lookupDDL);
+ }
+}
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/RecordTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/RecordTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestRestService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestRestService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRouting.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRouting.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/OptionUtils.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/OptionUtils.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
similarity index 98%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
index 7f9021f81..5fd313574 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
@@ -37,6 +37,7 @@
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@@ -142,7 +143,7 @@ public void testEvalWithCache() throws Exception {
lookupFunction.eval(5, StringData.fromString("5"));
RowData keyRow = GenericRowData.of(4, StringData.fromString("D"));
RowData keyRowNoExist = GenericRowData.of(5, StringData.fromString("5"));
- Cache> cache = lookupFunction.getCache();
+ Cache> cache = lookupFunction.getCache();
// empty data should cache
assertEquals(
cache.getIfPresent(keyRow),
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java
similarity index 84%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java
index e85e888fc..d2afbb167 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java
@@ -48,32 +48,35 @@ public static void main(String[] args) throws Exception {
String tablePrefix = "";
String tableSuffix = "";
Map mysqlConfig = new HashMap<>();
- mysqlConfig.put(MySqlSourceOptions.DATABASE_NAME.key(), "test");
+ mysqlConfig.put(MySqlSourceOptions.DATABASE_NAME.key(), "aaaaa");
mysqlConfig.put(MySqlSourceOptions.HOSTNAME.key(), "127.0.0.1");
mysqlConfig.put(MySqlSourceOptions.PORT.key(), "3306");
mysqlConfig.put(MySqlSourceOptions.USERNAME.key(), "root");
- mysqlConfig.put(MySqlSourceOptions.PASSWORD.key(), "12345678");
+ mysqlConfig.put(MySqlSourceOptions.PASSWORD.key(), "123456");
// add jdbc properties for MySQL
mysqlConfig.put("jdbc.properties.use_ssl", "false");
Configuration config = Configuration.fromMap(mysqlConfig);
Map sinkConfig = new HashMap<>();
- sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.20.30.1:8030");
+ sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.16.10.6:28737");
sinkConfig.put(DorisConfigOptions.USERNAME.key(), "root");
sinkConfig.put(DorisConfigOptions.PASSWORD.key(), "");
- sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.20.30.1:9030");
+ sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.16.10.6:29737");
sinkConfig.put(DorisConfigOptions.SINK_LABEL_PREFIX.key(), UUID.randomUUID().toString());
- sinkConfig.put("sink.enable-delete", "false");
+ // sinkConfig.put("sink.enable-delete", "false");
Configuration sinkConf = Configuration.fromMap(sinkConfig);
Map tableConfig = new HashMap<>();
tableConfig.put(DorisTableConfig.REPLICATION_NUM, "1");
- tableConfig.put(DorisTableConfig.TABLE_BUCKETS, "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50");
+ // tableConfig.put(DorisTableConfig.TABLE_BUCKETS,
+ // "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50");
// String includingTables = "tbl1|tbl2|tbl3";
- String includingTables = "a_.*|b_.*|c";
+ // tableConfig.put(DorisTableConfig.TABLE_PARTITIONS,
+ // "test_partition:create_dtime:day");
+ String includingTables = "test_partition";
String excludingTables = "";
- String multiToOneOrigin = "a_.*|b_.*";
- String multiToOneTarget = "a|b";
+ // String multiToOneOrigin = "a_.*|b_.*";
+ // String multiToOneTarget = "a|b";
boolean ignoreDefaultValue = false;
boolean useNewSchemaChange = true;
String schemaChangeMode = SchemaChangeMode.DEBEZIUM_STRUCTURE.getName();
@@ -88,12 +91,12 @@ public static void main(String[] args) throws Exception {
.setTableSuffix(tableSuffix)
.setIncludingTables(includingTables)
.setExcludingTables(excludingTables)
- .setMultiToOneOrigin(multiToOneOrigin)
- .setMultiToOneTarget(multiToOneTarget)
+ // .setMultiToOneOrigin(multiToOneOrigin)
+ // .setMultiToOneTarget(multiToOneTarget)
.setIgnoreDefaultValue(ignoreDefaultValue)
.setSinkConfig(sinkConf)
.setTableConfig(tableConfig)
- .setCreateTableOnly(false)
+ .setCreateTableOnly(true)
.setNewSchemaChange(useNewSchemaChange)
.setSchemaChangeMode(schemaChangeMode)
.setSingleSink(singleSink)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java
similarity index 82%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java
index 92600ffd6..d3f637a81 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java
+++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java
@@ -41,12 +41,12 @@ public static void main(String[] args) throws Exception {
String tablePrefix = "";
String tableSuffix = "";
Map sourceConfig = new HashMap<>();
- sourceConfig.put(OracleSourceOptions.DATABASE_NAME.key(), "XE");
+ sourceConfig.put(OracleSourceOptions.DATABASE_NAME.key(), "HELOWIN");
sourceConfig.put(OracleSourceOptions.SCHEMA_NAME.key(), "ADMIN");
- sourceConfig.put(OracleSourceOptions.HOSTNAME.key(), "127.0.0.1");
- sourceConfig.put(OracleSourceOptions.PORT.key(), "1521");
+ sourceConfig.put(OracleSourceOptions.HOSTNAME.key(), "10.16.10.6");
+ sourceConfig.put(OracleSourceOptions.PORT.key(), "1523");
sourceConfig.put(OracleSourceOptions.USERNAME.key(), "admin");
- sourceConfig.put(OracleSourceOptions.PASSWORD.key(), "");
+ sourceConfig.put(OracleSourceOptions.PASSWORD.key(), "admin123");
// sourceConfig.put("debezium.database.tablename.case.insensitive","false");
sourceConfig.put("debezium.log.mining.strategy", "online_catalog");
sourceConfig.put("debezium.log.mining.continuous.mine", "true");
@@ -54,18 +54,19 @@ public static void main(String[] args) throws Exception {
Configuration config = Configuration.fromMap(sourceConfig);
Map sinkConfig = new HashMap<>();
- sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.20.30.1:8030");
+ sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.16.10.6:28737");
sinkConfig.put(DorisConfigOptions.USERNAME.key(), "root");
sinkConfig.put(DorisConfigOptions.PASSWORD.key(), "");
- sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.20.30.1:9030");
+ sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.16.10.6:29737");
sinkConfig.put(DorisConfigOptions.SINK_LABEL_PREFIX.key(), UUID.randomUUID().toString());
Configuration sinkConf = Configuration.fromMap(sinkConfig);
Map tableConfig = new HashMap<>();
tableConfig.put(DorisTableConfig.REPLICATION_NUM, "1");
- tableConfig.put(DorisTableConfig.TABLE_BUCKETS, "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50");
- String includingTables = "a_.*|b_.*|c";
- String excludingTables = "";
+ // tableConfig.put(DorisTableConfig.TABLE_BUCKETS,
+ // "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50");
+ String includingTables = "PER.*|PERSONS";
+ String excludingTables = "PER123";
String multiToOneOrigin = "a_.*|b_.*";
String multiToOneTarget = "a|b";
boolean ignoreDefaultValue = false;
@@ -79,11 +80,12 @@ public static void main(String[] args) throws Exception {
.setTablePrefix(tablePrefix)
.setTableSuffix(tableSuffix)
.setIncludingTables(includingTables)
- .setExcludingTables(excludingTables)
- .setMultiToOneOrigin(multiToOneOrigin)
- .setMultiToOneTarget(multiToOneTarget)
+ // .setExcludingTables(excludingTables)
+ // .setMultiToOneOrigin(multiToOneOrigin)
+ // .setMultiToOneTarget(multiToOneTarget)
.setIgnoreDefaultValue(ignoreDefaultValue)
.setSinkConfig(sinkConf)
+ .setSingleSink(false)
.setTableConfig(tableConfig)
.setCreateTableOnly(false)
.setNewSchemaChange(useNewSchemaChange)
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/MockSource.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java
rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/MockSource.java
diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql
diff --git a/flink-doris-connector/src/test/resources/log4j2-test.properties b/flink-doris-connector/flink-doris-connector-common/src/test/resources/log4j2-test.properties
similarity index 100%
rename from flink-doris-connector/src/test/resources/log4j2-test.properties
rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/log4j2-test.properties
diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml
index 883964ecb..9d0dce5c0 100644
--- a/flink-doris-connector/pom.xml
+++ b/flink-doris-connector/pom.xml
@@ -26,8 +26,9 @@ under the License.
23
org.apache.doris
- flink-doris-connector-${flink.major.version}
- ${revision}
+ flink-doris-connector
+ 25.0.0-SNAPSHOT
+ pom
Flink Doris Connector
https://doris.apache.org/
@@ -38,6 +39,15 @@ under the License.
repo
+
+ flink-doris-connector-1.15
+ flink-doris-connector-1.16
+ flink-doris-connector-1.17
+ flink-doris-connector-1.18
+ flink-doris-connector-1.19
+ flink-doris-connector-1.20
+ flink-doris-connector-common
+
scm:git:https://git@github.com/apache/doris-flink-connector.git
@@ -67,9 +77,8 @@ under the License.
- 24.1.0-SNAPSHOT
- 1.18.0
- 1.18
+ 1.20.0
+ 1.20
3.2.1
flink-python
0.16.0
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
deleted file mode 100644
index f972d6e27..000000000
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.flink.example;
-
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.types.Row;
-
-public class LookupJoinCdcExample {
- public static void main(String[] args) throws Exception {
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(1);
- // env.disableOperatorChaining();
- env.enableCheckpointing(30000);
- final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
-
- tEnv.executeSql(
- "CREATE TABLE mysql_tb ("
- + "id INT,"
- + "name STRING,"
- + "process_time as proctime(),"
- + "primary key(id) NOT ENFORCED"
- + ") "
- + "WITH (\n"
- + " 'connector' = 'mysql-cdc',\n"
- + " 'hostname' = '127.0.0.1',\n"
- + " 'port' = '3306',\n"
- + " 'username' = 'root',\n"
- + " 'password' = '123456',\n"
- + " 'database-name' = 'test',\n"
- + " 'scan.startup.mode' = 'latest-offset',\n"
- + " 'server-time-zone' = 'Asia/Shanghai',\n"
- + " 'table-name' = 'fact_table' "
- + ")");
-
- tEnv.executeSql(
- "CREATE TABLE doris_tb ("
- + "id INT,"
- + "age INT,"
- + "dt DATE,"
- + "dtime TIMESTAMP,"
- + "primary key(id) NOT ENFORCED"
- + ") "
- + "WITH (\n"
- + " 'connector' = 'doris',\n"
- + " 'fenodes' = '127.0.0.1:8030',\n"
- + " 'jdbc-url' = 'jdbc:mysql://127.0.0.1:9030',\n"
- + " 'table.identifier' = 'test.dim_table_dt',\n"
- + " 'lookup.cache.max-rows' = '1000',"
- + " 'lookup.cache.ttl' = '1 hour',"
- + " 'lookup.jdbc.async' = 'true',\n"
- + " 'username' = 'root',\n"
- + " 'password' = ''\n"
- + ")");
-
- Table table =
- tEnv.sqlQuery(
- "SELECT a.id, a.name, b.age, b.dt, b.dtime\n"
- + "FROM mysql_tb a\n"
- + " left join doris_tb FOR SYSTEM_TIME AS OF a.process_time AS b\n"
- + " ON a.id = b.id");
-
- tEnv.toRetractStream(table, Row.class).print();
-
- env.execute();
- }
-}