diff --git a/.github/workflows/build-connector.yml b/.github/workflows/build-connector.yml index 2b2a242be..6c974b4b9 100644 --- a/.github/workflows/build-connector.yml +++ b/.github/workflows/build-connector.yml @@ -40,42 +40,24 @@ jobs: - name: Build flink connector 1.15 run: | - cd flink-doris-connector && mvn clean package \ - -Dflink.version=1.15.0 \ - -Dflink.minor.version=1.15 \ - -Dflink.python.id=flink-python_2.12 + cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.15 -am - name: Build flink connector 1.16 run: | - cd flink-doris-connector && mvn clean package \ - -Dflink.version=1.16.0 \ - -Dflink.minor.version=1.16 \ - -Dflink.python.id=flink-python + cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.16 -am - name: Build flink connector 1.17 run: | - cd flink-doris-connector && mvn clean package \ - -Dflink.version=1.17.0 \ - -Dflink.minor.version=1.17 \ - -Dflink.python.id=flink-python + cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.17 -am - name: Build flink connector 1.18 run: | - cd flink-doris-connector && mvn clean package \ - -Dflink.version=1.18.0 \ - -Dflink.minor.version=1.18 \ - -Dflink.python.id=flink-python + cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.18 -am - name: Build flink connector 1.19 run: | - cd flink-doris-connector && mvn clean package \ - -Dflink.version=1.19.0 \ - -Dflink.minor.version=1.19 \ - -Dflink.python.id=flink-python + cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.19 -am - name: Build flink connector 1.20 run: | - cd flink-doris-connector && mvn clean package \ - -Dflink.version=1.20.0 \ - -Dflink.minor.version=1.20 \ - -Dflink.python.id=flink-python + cd flink-doris-connector && mvn clean install -pl flink-doris-connector-1.20 -am \ No newline at end of file diff --git a/.github/workflows/checkstyle.yaml b/.github/workflows/checkstyle.yaml index 325ed71a6..3bbb11f66 100644 --- a/.github/workflows/checkstyle.yaml +++ b/.github/workflows/checkstyle.yaml @@ -40,4 +40,4 @@ jobs: - name: Run java checkstyle run: - cd flink-doris-connector && mvn clean compile checkstyle:checkstyle \ No newline at end of file + cd flink-doris-connector && mvn clean install -DskipTests && mvn checkstyle:checkstyle \ No newline at end of file diff --git a/.github/workflows/run-e2ecase.yml b/.github/workflows/run-e2ecase.yml index e6318f8f9..06134187f 100644 --- a/.github/workflows/run-e2ecase.yml +++ b/.github/workflows/run-e2ecase.yml @@ -40,5 +40,5 @@ jobs: - name: Run E2ECases run: | - cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="apache/doris:doris-all-in-one-2.1.0" + cd flink-doris-connector && mvn clean test -DfailIfNoTests=false -Dtest="*E2ECase" -Dimage="apache/doris:doris-all-in-one-2.1.0" diff --git a/.github/workflows/run-itcase.yml b/.github/workflows/run-itcase.yml index 0b6706560..4ecc05966 100644 --- a/.github/workflows/run-itcase.yml +++ b/.github/workflows/run-itcase.yml @@ -38,7 +38,11 @@ jobs: distribution: adopt java-version: '8' - - name: Run ITCases + - name: Run ITCases For 1.15 run: | - cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0" + cd flink-doris-connector && mvn clean test -pl flink-doris-connector-1.15 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0" + + - name: Run ITCases For 1.18 + run: | + cd flink-doris-connector && mvn clean test -pl flink-doris-connector-1.18 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0" diff --git a/.licenserc.yaml b/.licenserc.yaml index 27e1080ed..6d51f876c 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -12,6 +12,6 @@ header: - '.github/PULL_REQUEST_TEMPLATE.md' - '.licenserc.yaml' - 'custom_env.sh.tpl' - - 'flink-doris-connector/src/test/resources/container/' + - 'flink-doris-connector/flink-doris-connector-common/src/test/resources/container/' comment: on-failure diff --git a/flink-doris-connector/build.sh b/flink-doris-connector/build.sh index 4852553b5..c4d070dba 100755 --- a/flink-doris-connector/build.sh +++ b/flink-doris-connector/build.sh @@ -151,7 +151,6 @@ flinkVer=$? FLINK_PYTHON_ID="flink-python" if [ ${flinkVer} -eq 1 ]; then FLINK_VERSION="1.15.0" - FLINK_PYTHON_ID="flink-python_2.12" elif [ ${flinkVer} -eq 2 ]; then FLINK_VERSION="1.16.0" elif [ ${flinkVer} -eq 3 ]; then @@ -172,7 +171,7 @@ FLINK_MAJOR_VERSION=0 echo_g " flink version: ${FLINK_VERSION}, major version: ${FLINK_MAJOR_VERSION}" echo_g " build starting..." -${MVN_BIN} clean package -Dflink.version=${FLINK_VERSION} -Dflink.major.version=${FLINK_MAJOR_VERSION} -Dflink.python.id=${FLINK_PYTHON_ID} "$@" +${MVN_BIN} clean install -pl flink-doris-connector-${FLINK_MAJOR_VERSION} -am "$@" EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then diff --git a/flink-doris-connector/flink-doris-connector-1.15/pom.xml b/flink-doris-connector/flink-doris-connector-1.15/pom.xml new file mode 100644 index 000000000..c6072ac33 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.15/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-1.15 + Flink Doris Connector Flink 1.15 + + + 1.15.3 + 1.15 + flink-python_2.12 + + + + + org.apache.doris + flink-doris-connector-common + ${project.version} + + + org.apache.doris + flink-doris-connector-common + ${project.version} + test-jar + test + + + + diff --git a/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java new file mode 100644 index 000000000..e074151fc --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.lookup; + +import org.apache.flink.table.connector.source.AsyncTableFunctionProvider; +import org.apache.flink.table.connector.source.LookupTableSource; +import org.apache.flink.table.connector.source.TableFunctionProvider; +import org.apache.flink.table.functions.AsyncTableFunction; +import org.apache.flink.table.functions.TableFunction; + +/** Factory to create {@link LookupTableSource.LookupRuntimeProvider}. */ +public class LookupRuntimeProviderFactory { + + public static LookupTableSource.LookupRuntimeProvider create( + DorisLookupFunction function, boolean enableAsync) { + return enableAsync + ? AsyncTableFunctionProvider.of((AsyncTableFunction) function) + : TableFunctionProvider.of((TableFunction) function); + } +} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java similarity index 92% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java rename to flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java index 2cd3c9968..5bc462642 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java +++ b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java @@ -30,6 +30,7 @@ import org.apache.doris.flink.cfg.DorisLookupOptions; import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.lookup.DorisJdbcLookupReader; +import org.apache.doris.flink.lookup.DorisLookupFunction; import org.apache.doris.flink.lookup.DorisLookupReader; import org.apache.doris.flink.lookup.LookupMetrics; import org.apache.doris.flink.lookup.LookupSchema; @@ -39,18 +40,19 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction { +/** async lookup {@link AsyncTableFunction} for 1.15. */ +public class DorisRowDataAsyncLookupFunction extends AsyncTableFunction + implements DorisLookupFunction { private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataAsyncLookupFunction.class); private final DorisOptions options; private final DorisLookupOptions lookupOptions; private final long cacheMaxSize; private final long cacheExpireMs; - private transient Cache> cache; + private transient Cache> cache; private DorisLookupReader lookupReader; private LookupSchema lookupSchema; private LookupMetrics lookupMetrics; @@ -101,7 +103,7 @@ public void eval(CompletableFuture> future, Object... keys) throws IOException { RowData keyRow = GenericRowData.of(keys); if (cache != null) { - List cachedRows = cache.getIfPresent(keyRow); + Collection cachedRows = cache.getIfPresent(keyRow); if (cachedRows != null) { lookupMetrics.incHitCount(); if (LOG.isDebugEnabled()) { @@ -114,7 +116,7 @@ public void eval(CompletableFuture> future, Object... keys) } } - CompletableFuture> resultFuture = lookupReader.asyncGet(keyRow); + CompletableFuture> resultFuture = lookupReader.asyncGet(keyRow); resultFuture.handleAsync( (resultRows, throwable) -> { try { @@ -151,7 +153,7 @@ public void close() throws Exception { } @VisibleForTesting - public Cache> getCache() { + public Cache> getCache() { return cache; } } diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java similarity index 91% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java rename to flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java index a4b07677e..1dbcbeab0 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java +++ b/flink-doris-connector/flink-doris-connector-1.15/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java @@ -30,6 +30,7 @@ import org.apache.doris.flink.cfg.DorisLookupOptions; import org.apache.doris.flink.cfg.DorisOptions; import org.apache.doris.flink.lookup.DorisJdbcLookupReader; +import org.apache.doris.flink.lookup.DorisLookupFunction; import org.apache.doris.flink.lookup.DorisLookupReader; import org.apache.doris.flink.lookup.LookupMetrics; import org.apache.doris.flink.lookup.LookupSchema; @@ -37,18 +38,19 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.concurrent.TimeUnit; -/** use jdbc to query. */ -public class DorisRowDataJdbcLookupFunction extends TableFunction { +/** lookup {@link TableFunction} for 1.15. */ +public class DorisRowDataJdbcLookupFunction extends TableFunction + implements DorisLookupFunction { private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataJdbcLookupFunction.class); private final DorisOptions options; private final DorisLookupOptions lookupOptions; private final long cacheMaxSize; private final long cacheExpireMs; - private transient Cache> cache; + private transient Cache> cache; private DorisLookupReader lookupReader; private LookupSchema lookupSchema; private LookupMetrics lookupMetrics; @@ -97,7 +99,7 @@ public void open(FunctionContext context) throws Exception { public void eval(Object... keys) throws IOException { RowData keyRow = GenericRowData.of(keys); if (cache != null) { - List cachedRows = cache.getIfPresent(keyRow); + Collection cachedRows = cache.getIfPresent(keyRow); if (cachedRows != null) { lookupMetrics.incHitCount(); if (LOG.isDebugEnabled()) { @@ -115,7 +117,7 @@ public void eval(Object... keys) throws IOException { } private void queryRecord(RowData keyRow) throws IOException { - List rowData = lookupReader.get(keyRow); + Collection rowData = lookupReader.get(keyRow); if (rowData == null) { rowData = Collections.emptyList(); } @@ -135,7 +137,7 @@ public void close() throws Exception { } @VisibleForTesting - public Cache> getCache() { + public Cache> getCache() { return cache; } } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java b/flink-doris-connector/flink-doris-connector-1.15/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java similarity index 56% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java rename to flink-doris-connector/flink-doris-connector-1.15/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java index 6d569bcfc..a4383a31d 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java +++ b/flink-doris-connector/flink-doris-connector-1.15/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java @@ -30,6 +30,7 @@ import org.apache.doris.flink.container.AbstractITCaseService; import org.apache.doris.flink.container.ContainerUtils; import org.apache.doris.flink.table.DorisConfigOptions; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,12 +44,21 @@ public class DorisLookupTableITCase extends AbstractITCaseService { private static final Logger LOG = LoggerFactory.getLogger(DorisLookupTableITCase.class); private static final String DATABASE = "test_lookup"; private static final String TABLE_READ_TBL = "tbl_read_tbl"; + private static final String TABLE_DIM_TBL = "tbl_dim_tbl"; + + private StreamExecutionEnvironment env; + private StreamTableEnvironment tEnv; + + @Before + public void before() { + env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(DEFAULT_PARALLELISM); + tEnv = StreamTableEnvironment.create(env); + } @Test public void testLookupTable() throws Exception { initializeTable(); - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(DEFAULT_PARALLELISM); DataStreamSource sourceStream = env.fromElements(1, 2, 3, 4); final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); Schema schema = @@ -138,4 +148,106 @@ private void initializeTable() { "insert into %s.%s values (3,-106,-14878,1466614815449373200)", DATABASE, DorisLookupTableITCase.TABLE_READ_TBL)); } + + @Test + public void testLookup() throws Exception { + initFlinkTable(); + Schema schema = + Schema.newBuilder() + .column("f0", DataTypes.INT()) + .columnByExpression("proctime", "PROCTIME()") + .build(); + + Table table = tEnv.fromDataStream(env.fromElements(1, 2, 3), schema); + tEnv.createTemporaryView("fact_table", table); + + String query = + "select fact_table.f0," + + "doris_lookup.score" + + " from fact_table" + + " left join doris_lookup FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id"; + TableResult tableResult = tEnv.executeSql(query); + CloseableIterator collectIter = tableResult.collect(); + List actual = collectSize(collectIter, 3); + String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, null]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + collectIter.close(); + + // mock data + tEnv.dropTemporaryView("fact_table"); + tEnv.createTemporaryView( + "fact_table", tEnv.fromDataStream(env.fromElements(1, 2, 3, 4), schema)); + + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format( + "insert into %s.%s values (3,300)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL)); + + TableResult tableResult2 = tEnv.executeSql(query); + CloseableIterator collectIter2 = tableResult2.collect(); + actual = collectSize(collectIter2, 4); + expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, 300]", "+I[4, null]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + collectIter2.close(); + } + + private static List collectSize(CloseableIterator iterator, int rows) + throws Exception { + List result = new ArrayList<>(); + while (result.size() < rows && iterator.hasNext()) { + result.add(iterator.next().toString()); + } + return result; + } + + private void initFlinkTable() { + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE), + String.format( + "DROP TABLE IF EXISTS %s.%s", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL), + String.format( + "CREATE TABLE %s.%s ( \n" + + "`id` int(11),\n" + + "`score` int(11)\n" + + ") DISTRIBUTED BY HASH(`id`) BUCKETS 4\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ")\n", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL), + String.format( + "insert into %s.%s values (1,100)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL), + String.format( + "insert into %s.%s values (2,200)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL)); + + String lookupDDL = + String.format( + "CREATE TABLE `doris_lookup`(" + + " `id` INTEGER," + + " `score` INTEGER," + + " PRIMARY KEY (`id`) NOT ENFORCED" + + ") WITH (" + + "'connector' = '" + + DorisConfigOptions.IDENTIFIER + + "'," + + "'fenodes' = '%s'," + + "'jdbc-url' = '%s'," + + "'table.identifier' = '%s'," + + "'username' = '%s'," + + "'password' = '%s'" + + ")", + getFenodes(), + getDorisQueryUrl(), + DATABASE + "." + TABLE_DIM_TBL, + getDorisUsername(), + getDorisPassword()); + + tEnv.executeSql(lookupDDL); + } } diff --git a/flink-doris-connector/flink-doris-connector-1.16/pom.xml b/flink-doris-connector/flink-doris-connector-1.16/pom.xml new file mode 100644 index 000000000..b0180f7ab --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.16/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-1.16 + Flink Doris Connector Flink 1.16 + + + 1.16.0 + 1.16 + + + + + org.apache.doris + flink-doris-connector-common + ${project.version} + + + diff --git a/flink-doris-connector/flink-doris-connector-1.17/pom.xml b/flink-doris-connector/flink-doris-connector-1.17/pom.xml new file mode 100644 index 000000000..7e88e81a1 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.17/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-1.17 + Flink Doris Connector Flink 1.17 + + + 1.17.0 + 1.17 + + + + + org.apache.doris + flink-doris-connector-common + ${project.version} + + + diff --git a/flink-doris-connector/flink-doris-connector-1.18/pom.xml b/flink-doris-connector/flink-doris-connector-1.18/pom.xml new file mode 100644 index 000000000..ce75a66b0 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.18/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-1.18 + Flink Doris Connector Flink 1.18 + + + 1.18.0 + 1.18 + + + + + org.apache.doris + flink-doris-connector-common + ${project.version} + + + diff --git a/flink-doris-connector/flink-doris-connector-1.19/pom.xml b/flink-doris-connector/flink-doris-connector-1.19/pom.xml new file mode 100644 index 000000000..0dc9dbb1c --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.19/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-1.19 + Flink Doris Connector Flink 1.19 + + + 1.19.0 + 1.19 + + + + + org.apache.doris + flink-doris-connector-common + ${project.version} + + + diff --git a/flink-doris-connector/flink-doris-connector-1.20/pom.xml b/flink-doris-connector/flink-doris-connector-1.20/pom.xml new file mode 100644 index 000000000..39ab1eaef --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-1.20/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-1.20 + Flink Doris Connector Flink 1.20 + + + 1.20.0 + 1.20 + + + + + org.apache.doris + flink-doris-connector-common + ${project.version} + + + + diff --git a/flink-doris-connector/flink-doris-connector-common/pom.xml b/flink-doris-connector/flink-doris-connector-common/pom.xml new file mode 100644 index 000000000..78764d20e --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + + org.apache.doris + flink-doris-connector + 25.0.0-SNAPSHOT + + flink-doris-connector-common + Flink Doris Connector Common + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/backend/BackendClient.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/backend/BackendClient.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisCatalogOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DataModel.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactory.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CreateTableException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/CreateTableException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java similarity index 94% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java index 958aa9e34..12a8bc062 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -65,8 +66,8 @@ private DorisRowConverter buildKeyConvert() { } @Override - public CompletableFuture> asyncGet(RowData recordIn) throws IOException { - CompletableFuture> result = new CompletableFuture<>(); + public CompletableFuture> asyncGet(RowData recordIn) throws IOException { + CompletableFuture> result = new CompletableFuture<>(); Record record = convertRecord(recordIn); try { pool.get(new Get(record)) @@ -125,7 +126,7 @@ private Record convertRecord(RowData recordIn) { } @Override - public List get(RowData record) throws IOException { + public Collection get(RowData record) throws IOException { try { return this.asyncGet(record).get(); } catch (InterruptedException | ExecutionException e) { diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java new file mode 100644 index 000000000..2c9e07102 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupFunction.java @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.lookup; + +/** Doris lookup function. */ +public interface DorisLookupFunction {} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java similarity index 82% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java index 7c8663e0e..3bf1c9d07 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java @@ -21,12 +21,13 @@ import java.io.Closeable; import java.io.IOException; -import java.util.List; +import java.util.Collection; import java.util.concurrent.CompletableFuture; public abstract class DorisLookupReader implements Closeable { - public abstract CompletableFuture> asyncGet(RowData record) throws IOException; + public abstract CompletableFuture> asyncGet(RowData record) + throws IOException; - public abstract List get(RowData record) throws IOException; + public abstract Collection get(RowData record) throws IOException; } diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Get.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Get.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/GetAction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/GetAction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java new file mode 100644 index 000000000..ea77615a6 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupRuntimeProviderFactory.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.lookup; + +import org.apache.flink.table.connector.source.LookupTableSource; +import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider; +import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider; +import org.apache.flink.table.functions.AsyncLookupFunction; +import org.apache.flink.table.functions.LookupFunction; + +/** Factory to create {@link LookupTableSource.LookupRuntimeProvider}. */ +public class LookupRuntimeProviderFactory { + + public static LookupTableSource.LookupRuntimeProvider create( + DorisLookupFunction function, boolean enableAsync) { + return enableAsync + ? AsyncLookupFunctionProvider.of((AsyncLookupFunction) function) + : LookupFunctionProvider.of((LookupFunction) function); + } +} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Record.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Record.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/RecordKey.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/RecordKey.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Worker.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/lookup/Worker.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/RestService.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/RestService.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Field.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Field.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/RespContent.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/RespContent.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Schema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Schema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Tablet.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/rest/models/Tablet.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/Routing.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/Routing.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/RowBatch.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/serialization/RowBatch.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/BackendUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/BackendUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisSink.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/DorisSink.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/HttpUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/LoadStatus.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/LoadStatus.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/EventType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/EventType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/DorisSource.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/DorisSource.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java similarity index 96% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java index e4f27d8bc..e7a5a95ba 100644 --- a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java @@ -21,13 +21,11 @@ import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.Projection; -import org.apache.flink.table.connector.source.AsyncTableFunctionProvider; import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.connector.source.InputFormatProvider; import org.apache.flink.table.connector.source.LookupTableSource; import org.apache.flink.table.connector.source.ScanTableSource; import org.apache.flink.table.connector.source.SourceProvider; -import org.apache.flink.table.connector.source.TableFunctionProvider; import org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown; import org.apache.flink.table.connector.source.abilities.SupportsLimitPushDown; import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; @@ -42,6 +40,8 @@ import org.apache.doris.flink.cfg.DorisReadOptions; import org.apache.doris.flink.deserialization.RowDataDeserializationSchema; import org.apache.doris.flink.exception.DorisException; +import org.apache.doris.flink.lookup.DorisLookupFunction; +import org.apache.doris.flink.lookup.LookupRuntimeProviderFactory; import org.apache.doris.flink.rest.PartitionDefinition; import org.apache.doris.flink.rest.RestService; import org.apache.doris.flink.source.DorisSource; @@ -151,8 +151,10 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { keyNames[i] = DataType.getFieldNames(physicalRowDataType).get(innerKeyArr[0]); keyIndexs[i] = innerKeyArr[0]; } + + DorisLookupFunction lookupFunction; if (lookupOptions.isAsync()) { - return AsyncTableFunctionProvider.of( + lookupFunction = new DorisRowDataAsyncLookupFunction( options, lookupOptions, @@ -160,9 +162,9 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { DataType.getFieldDataTypes(physicalRowDataType) .toArray(new DataType[0]), keyNames, - keyIndexs)); + keyIndexs); } else { - return TableFunctionProvider.of( + lookupFunction = new DorisRowDataJdbcLookupFunction( options, lookupOptions, @@ -170,8 +172,9 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { DataType.getFieldDataTypes(physicalRowDataType) .toArray(new DataType[0]), keyNames, - keyIndexs)); + keyIndexs); } + return LookupRuntimeProviderFactory.create(lookupFunction, lookupOptions.isAsync()); } @Override diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java new file mode 100644 index 000000000..4e0c87b8e --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.AsyncLookupFunction; +import org.apache.flink.table.functions.AsyncTableFunction; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.types.DataType; +import org.apache.flink.util.Preconditions; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.lookup.DorisJdbcLookupReader; +import org.apache.doris.flink.lookup.DorisLookupFunction; +import org.apache.doris.flink.lookup.DorisLookupReader; +import org.apache.doris.flink.lookup.LookupMetrics; +import org.apache.doris.flink.lookup.LookupSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** async lookup {@link AsyncTableFunction} for after 1.16 */ +public class DorisRowDataAsyncLookupFunction extends AsyncLookupFunction + implements DorisLookupFunction { + private static final Logger LOG = + LoggerFactory.getLogger(DorisRowDataAsyncLookupFunction.class); + private final DorisOptions options; + private final DorisLookupOptions lookupOptions; + private final long cacheMaxSize; + private final long cacheExpireMs; + private transient Cache> cache; + private DorisLookupReader lookupReader; + private LookupSchema lookupSchema; + private LookupMetrics lookupMetrics; + + public DorisRowDataAsyncLookupFunction( + DorisOptions options, + DorisLookupOptions lookupOptions, + String[] selectFields, + DataType[] fieldTypes, + String[] conditionFields, + int[] keyIndex) { + Preconditions.checkNotNull( + options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup"); + this.options = options; + this.cacheMaxSize = lookupOptions.getCacheMaxSize(); + this.cacheExpireMs = lookupOptions.getCacheExpireMs(); + this.lookupOptions = lookupOptions; + this.lookupSchema = + new LookupSchema( + options.getTableIdentifier(), + selectFields, + fieldTypes, + conditionFields, + keyIndex); + } + + @Override + public void open(FunctionContext context) throws Exception { + super.open(context); + LOG.info( + "lookup options: threadSize {}, batchSize {}, queueSize {}", + lookupOptions.getJdbcReadThreadSize(), + lookupOptions.getJdbcReadBatchSize(), + lookupOptions.getJdbcReadBatchQueueSize()); + this.cache = + cacheMaxSize == -1 || cacheExpireMs == -1 + ? null + : CacheBuilder.newBuilder() + .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS) + .maximumSize(cacheMaxSize) + .build(); + this.lookupReader = new DorisJdbcLookupReader(options, lookupOptions, lookupSchema); + this.lookupMetrics = new LookupMetrics(context.getMetricGroup()); + } + + @Override + public CompletableFuture> asyncLookup(RowData keyRow) { + CompletableFuture> resultFuture = new CompletableFuture<>(); + if (cache != null) { + List cachedRows = cache.getIfPresent(keyRow); + if (cachedRows != null) { + lookupMetrics.incHitCount(); + if (LOG.isDebugEnabled()) { + LOG.debug("lookup cache hit for key: {}", keyRow); + } + resultFuture.complete(cachedRows); + return resultFuture; + } else { + lookupMetrics.incMissCount(); + } + } + + CompletableFuture> future; + try { + future = lookupReader.asyncGet(keyRow); + } catch (IOException e) { + throw new RuntimeException(e); + } + resultFuture.handleAsync( + (resultRows, throwable) -> { + try { + if (throwable != null) { + future.completeExceptionally(throwable); + } else { + if (resultRows == null || resultRows.isEmpty()) { + if (cache != null) { + cache.put(keyRow, Collections.emptyList()); + lookupMetrics.incLoadCount(); + } + future.complete(Collections.emptyList()); + } else { + if (cache != null) { + cache.put(keyRow, (List) resultRows); + lookupMetrics.incLoadCount(); + } + future.complete(resultRows); + } + } + } catch (Throwable t) { + future.completeExceptionally(t); + } + return null; + }); + return future; + } + + @Override + public void close() throws Exception { + super.close(); + if (lookupReader != null) { + lookupReader.close(); + } + } + + @VisibleForTesting + public Cache> getCache() { + return cache; + } +} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java diff --git a/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java new file mode 100644 index 000000000..70fefa46c --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.table; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.FunctionContext; +import org.apache.flink.table.functions.LookupFunction; +import org.apache.flink.table.types.DataType; +import org.apache.flink.util.Preconditions; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.doris.flink.cfg.DorisLookupOptions; +import org.apache.doris.flink.cfg.DorisOptions; +import org.apache.doris.flink.lookup.DorisJdbcLookupReader; +import org.apache.doris.flink.lookup.DorisLookupFunction; +import org.apache.doris.flink.lookup.DorisLookupReader; +import org.apache.doris.flink.lookup.LookupMetrics; +import org.apache.doris.flink.lookup.LookupSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +/** lookup {@link LookupFunction} for after 1.16 */ +public class DorisRowDataJdbcLookupFunction extends LookupFunction implements DorisLookupFunction { + private static final Logger LOG = LoggerFactory.getLogger(DorisRowDataJdbcLookupFunction.class); + private final DorisOptions options; + private final DorisLookupOptions lookupOptions; + private final long cacheMaxSize; + private final long cacheExpireMs; + private transient Cache> cache; + private DorisLookupReader lookupReader; + private LookupSchema lookupSchema; + private LookupMetrics lookupMetrics; + + public DorisRowDataJdbcLookupFunction( + DorisOptions options, + DorisLookupOptions lookupOptions, + String[] selectFields, + DataType[] fieldTypes, + String[] conditionFields, + int[] keyIndex) { + Preconditions.checkNotNull( + options.getJdbcUrl(), "jdbc-url is required in jdbc mode lookup"); + this.options = options; + this.cacheMaxSize = lookupOptions.getCacheMaxSize(); + this.cacheExpireMs = lookupOptions.getCacheExpireMs(); + this.lookupOptions = lookupOptions; + this.lookupSchema = + new LookupSchema( + options.getTableIdentifier(), + selectFields, + fieldTypes, + conditionFields, + keyIndex); + } + + @Override + public void open(FunctionContext context) throws Exception { + super.open(context); + this.cache = + cacheMaxSize == -1 || cacheExpireMs == -1 + ? null + : CacheBuilder.newBuilder() + .expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS) + .maximumSize(cacheMaxSize) + .build(); + this.lookupReader = new DorisJdbcLookupReader(options, lookupOptions, lookupSchema); + this.lookupMetrics = new LookupMetrics(context.getMetricGroup()); + } + + @Override + public Collection lookup(RowData keyRow) throws IOException { + if (cache != null) { + Collection cachedRows = cache.getIfPresent(keyRow); + if (cachedRows != null) { + lookupMetrics.incHitCount(); + if (LOG.isDebugEnabled()) { + LOG.debug("lookup cache hit for key: {}", keyRow); + } + return cachedRows; + } else { + lookupMetrics.incMissCount(); + } + } + return queryRecord(keyRow); + } + + private Collection queryRecord(RowData keyRow) throws IOException { + Collection rowData = lookupReader.get(keyRow); + if (rowData == null) { + rowData = Collections.emptyList(); + } + if (cache != null) { + cache.put(keyRow, rowData); + lookupMetrics.incLoadCount(); + } + return rowData; + } + + @Override + public void close() throws Exception { + super.close(); + if (lookupReader != null) { + lookupReader.close(); + } + } + + @VisibleForTesting + public Cache> getCache() { + return cache; + } +} diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/CdcTools.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DatabaseSyncConfig.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/JdbcSourceSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/ParsingProcessFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2DateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Schema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/deserialize/DorisJsonDebeziumDeserializationSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/ChangeStreamConstant.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunction.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoDBJsonDebeziumSchemaSerializer.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumDataChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mongodb/serializer/MongoJsonDebeziumSchemaChange.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/DateToStringConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDatabaseSync.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerDateConverter.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerSchema.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/ErrorMessages.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/ErrorMessages.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/FastDateUtil.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/FastDateUtil.java diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/IPUtils.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/doris/flink/util/IPUtils.java diff --git a/flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java similarity index 100% rename from flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java diff --git a/flink-doris-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-doris-connector/flink-doris-connector-common/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory similarity index 100% rename from flink-doris-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory rename to flink-doris-connector/flink-doris-connector-common/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory diff --git a/flink-doris-connector/src/main/resources/log4j2.properties b/flink-doris-connector/flink-doris-connector-common/src/main/resources/log4j2.properties similarity index 100% rename from flink-doris-connector/src/main/resources/log4j2.properties rename to flink-doris-connector/flink-doris-connector-common/src/main/resources/log4j2.properties diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java index 5c7c151ec..21e203553 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java @@ -20,6 +20,7 @@ import org.apache.doris.flink.container.instance.ContainerService; import org.apache.doris.flink.container.instance.DorisContainer; import org.apache.doris.flink.container.instance.DorisCustomerContainer; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,6 +84,7 @@ protected String getDorisInstanceHost() { return dorisContainerService.getInstanceHost(); } + @AfterClass public static void closeContainers() { LOG.info("Starting to close containers."); closeDorisContainer(); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractE2EService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/ContainerUtils.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/ContainerUtils.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/e2e/Mysql2DorisE2ECase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java similarity index 81% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java index 8fe017e1c..700ecd671 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java @@ -44,18 +44,24 @@ public static void main(String[] args) throws Exception { JsonDebeziumDeserializationSchema schema = new JsonDebeziumDeserializationSchema(false, customConverterConfigs); + Properties defaultProps = DateToStringConverter.DEFAULT_PROPS; + // defaultProps.put( + // "column.include.list", + // "test.test_column.id,test.test_column.c1,test.test_column.c2"); MySqlSource mySqlSource = MySqlSource.builder() .hostname("127.0.0.1") .port(3306) .databaseList("test") // set captured database - .tableList("test.t1") // set captured table + .tableList("test.test_flink") // set captured table .username("root") .password("123456") - .debeziumProperties(DateToStringConverter.DEFAULT_PROPS) - .deserializer(schema) - .serverTimeZone("Asia/Shanghai") - .includeSchemaChanges(true) // converts SourceRecord to JSON String + .debeziumProperties(defaultProps) + // .deserializer(schema) + .deserializer(new JsonDebeziumDeserializationSchema()) + // .startupOptions(StartupOptions.latest()) + // .serverTimeZone("Asia/Shanghai") + // .includeSchemaChanges(true) // converts SourceRecord to JSON String .build(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); @@ -67,8 +73,8 @@ public static void main(String[] args) throws Exception { props.setProperty("read_json_by_line", "true"); DorisOptions dorisOptions = DorisOptions.builder() - .setFenodes("127.0.0.1:8030") - .setTableIdentifier("test.t1") + .setFenodes("10.16.10.6:28737") + .setTableIdentifier("test.test_flink_a") .setUsername("root") .setPassword("") .build(); diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CatalogExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/CatalogExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java similarity index 90% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java index 7a111b258..70963e052 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java @@ -26,7 +26,6 @@ import org.apache.doris.flink.cfg.DorisReadOptions; import org.apache.doris.flink.sink.DorisSink; import org.apache.doris.flink.sink.batch.DorisBatchSink; -import org.apache.doris.flink.sink.writer.WriteMode; import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer; import java.util.Arrays; @@ -38,7 +37,7 @@ public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); // env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); - env.enableCheckpointing(5000); + env.enableCheckpointing(30000); // env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); // env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, @@ -61,20 +60,17 @@ public static void main(String[] args) throws Exception { properties.setProperty("format", "csv"); DorisOptions.Builder dorisBuilder = DorisOptions.builder(); dorisBuilder - .setFenodes("127.0.0.1:8030") - .setTableIdentifier("test.test_flink") + .setFenodes("10.16.10.6:28737") + .setTableIdentifier("test.test_flink_error") .setUsername("root") .setPassword(""); DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder(); - executionBuilder - .setLabelPrefix("label") - .setStreamLoadProp(properties) - .setDeletable(false) - .setBufferFlushMaxBytes(8 * 1024) - .setBufferFlushMaxRows(900) - .setBufferFlushIntervalMs(1000 * 10) - // .setBatchMode(true); - .setWriteMode(WriteMode.STREAM_LOAD_BATCH); + executionBuilder.setLabelPrefix("label").setStreamLoadProp(properties).setDeletable(false); + // .setBufferFlushMaxBytes(8 * 102400000) + // .setBufferFlushMaxRows(90000) + // .setBufferFlushIntervalMs(1000 * 10) + // .setBatchMode(true); + // .setWriteMode(WriteMode.STREAM_LOAD_BATCH); builder.setDorisReadOptions(readOptionBuilder.build()) .setDorisExecutionOptions(executionBuilder.build()) .setSerializer(new SimpleStringSerializer()) @@ -88,7 +84,7 @@ public static void main(String[] args) throws Exception { public void run(SourceContext out) throws Exception { while (true) { id = id + 1; - String record = id + "," + UUID.randomUUID() + "," + id + ""; + String record = UUID.randomUUID() + "," + id + ""; out.collect(record); Thread.sleep(500); } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java similarity index 76% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java index bcb169656..dcbd0eef3 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java @@ -17,15 +17,11 @@ package org.apache.doris.flink.example; -import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.time.Time; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.doris.flink.cfg.DorisExecutionOptions; import org.apache.doris.flink.cfg.DorisOptions; @@ -36,11 +32,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; +import java.util.UUID; public class DorisSinkExample { public static void main(String[] args) throws Exception { - JSONFormatWrite(); + CSVFormatWrite(); } public static void JSONFormatWrite() throws Exception { @@ -85,12 +82,14 @@ public static void JSONFormatWrite() throws Exception { public static void CSVFormatWrite() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); - env.setRuntimeMode(RuntimeExecutionMode.BATCH); env.enableCheckpointing(10000); + env.setParallelism(1); env.getCheckpointConfig() .enableExternalizedCheckpoints( CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); - env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(30000))); + // env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, + // Time.milliseconds(30000))); + env.setRestartStrategy(RestartStrategies.noRestart()); DorisSink.Builder builder = DorisSink.builder(); Properties properties = new Properties(); properties.setProperty("column_separator", ","); @@ -98,27 +97,41 @@ public static void CSVFormatWrite() throws Exception { properties.setProperty("format", "csv"); DorisOptions.Builder dorisBuilder = DorisOptions.builder(); dorisBuilder - .setFenodes("127.0.0.1:8030") - .setTableIdentifier("db.table") - .setUsername("test") - .setPassword("test"); + .setFenodes("10.16.10.6:48737") + .setTableIdentifier("test.test_flink") + .setUsername("root") + .setPassword(""); DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder(); executionBuilder .setLabelPrefix("label-doris") - .setStreamLoadProp(properties) - .setBufferSize(8 * 1024) - .setBufferCount(3); + .setDeletable(false) + // .setBatchMode(true) + .setStreamLoadProp(properties); builder.setDorisReadOptions(DorisReadOptions.builder().build()) .setDorisExecutionOptions(executionBuilder.build()) .setSerializer(new SimpleStringSerializer()) .setDorisOptions(dorisBuilder.build()); - List> data = new ArrayList<>(); - data.add(new Tuple2<>("doris", 1)); - DataStreamSource> source = env.fromCollection(data); - source.map((MapFunction, String>) t -> t.f0 + "," + t.f1) + env.addSource( + new SourceFunction() { + private Long id = 0L; + + @Override + public void run(SourceContext out) throws Exception { + while (true) { + id = id + 1; + String record = UUID.randomUUID() + "," + id + ""; + out.collect(record); + Thread.sleep(500); + } + } + + @Override + public void cancel() {} + }) .sinkTo(builder.build()); + env.execute("doris test"); } } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java similarity index 60% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java index a31ab8a55..856c8c8f6 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java @@ -18,28 +18,36 @@ package org.apache.doris.flink.example; import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.flink.table.api.Expressions.$; - public class DorisSinkSQLExample { public static void main(String[] args) { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); + env.enableCheckpointing(10000); env.setRuntimeMode(RuntimeExecutionMode.BATCH); final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); - List> data = new ArrayList<>(); - data.add(new Tuple2<>("doris", 1)); - DataStreamSource> source = env.fromCollection(data); - tEnv.createTemporaryView("doris_test", source, $("name"), $("age")); + tEnv.executeSql( + "CREATE TABLE doris_test_source (" + + "name STRING," + + "age INT" + + ") " + + "WITH (\n" + + " 'connector' = 'doris',\n" + + " 'fenodes' = '10.16.10.6:28737',\n" + + " 'table.identifier' = 'test.test_flink_10bucket',\n" + + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n" + + " 'username' = 'root',\n" + + " 'password' = '',\n" + + " 'sink.properties.format' = 'json',\n" + + " 'sink.buffer-count' = '4',\n" + + " 'sink.buffer-size' = '4086'," + + " 'sink.label-prefix' = 'doris_lab1el112121',\n" + + " 'sink.properties.read_json_by_line' = 'true'\n" + + ")"); tEnv.executeSql( "CREATE TABLE doris_test_sink (" @@ -48,16 +56,18 @@ public static void main(String[] args) { + ") " + "WITH (\n" + " 'connector' = 'doris',\n" - + " 'fenodes' = 'FE_IP:8030',\n" - + " 'table.identifier' = 'db.table',\n" + + " 'fenodes' = '10.16.10.6:28737',\n" + + " 'table.identifier' = 'test.test_flink_a',\n" + + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n" + " 'username' = 'root',\n" + " 'password' = '',\n" + " 'sink.properties.format' = 'json',\n" + " 'sink.buffer-count' = '4',\n" + " 'sink.buffer-size' = '4086'," - + " 'sink.label-prefix' = 'doris_label',\n" + + " 'sink.label-prefix' = 'doris_lab1el4',\n" + " 'sink.properties.read_json_by_line' = 'true'\n" + ")"); - tEnv.executeSql("INSERT INTO doris_test_sink select name,age from doris_test"); + + tEnv.executeSql("INSERT OVERWRITE doris_test_sink select 'zhangsan',1"); } } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java similarity index 60% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java index 90e461cfc..ba756d269 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java @@ -20,45 +20,46 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; public class DorisSourceExample { public static void main(String[] args) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(1); + env.setParallelism(2); final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); // register a table in the catalog tEnv.executeSql( "CREATE TABLE doris_source (" - + "bigint_1 BIGINT," - + "char_1 STRING," - + "date_1 STRING," - + "datetime_1 STRING," - + "decimal_1 DECIMAL(5,2)," - + "double_1 DOUBLE," - + "float_1 FLOAT ," - + "int_1 INT ," - + "largeint_1 STRING, " - + "smallint_1 SMALLINT, " - + "tinyint_1 TINYINT, " - + "varchar_1 STRING " + + "name STRING," + + "age int" + ") " + "WITH (\n" + " 'connector' = 'doris',\n" - + " 'fenodes' = 'FE_IP:8030',\n" - + " 'table.identifier' = 'db.table',\n" + + " 'fenodes' = '10.16.10.6:28737',\n" + + " 'table.identifier' = 'test.test_flink_bucket',\n" + " 'username' = 'root',\n" - + " 'password' = ''\n" + + " 'password' = '',\n" + // + " 'doris.filter.query' = ' (dt = + // DATE_FORMAT(TIMESTAMPADD(DAY , -7, NOW()), ''yyyy-MM-dd'')) ',\n" + + " 'source.use-flight-sql' = 'true',\n" + + " 'doris.request.tablet.size' = '1',\n" + + " 'source.flight-sql-port' = '29747'" + ")"); // define a dynamic aggregating query - final Table result = tEnv.sqlQuery("SELECT * from doris_source "); + final Table result = tEnv.sqlQuery(" SELECT * from doris_source limit 1 "); + // TableResult tableResult = tEnv.executeSql("explain PLAN FOR SELECT * from + // doris_source limit 10 "); + // tableResult.print(); + // final Table result = tEnv.sqlQuery("SELECT * from ( select name,age as c,dt from + // doris_source) tb where c >2"); // print the result to the console - tEnv.toDataStream(result).print(); + tEnv.toRetractStream(result, Row.class).print(); env.execute(); } } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java diff --git a/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java new file mode 100644 index 000000000..535bcf911 --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.example; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; + +public class LookupJoinCdcExample { + public static void main(String[] args) throws Exception { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + // env.disableOperatorChaining(); + env.enableCheckpointing(10000); + final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); + + tEnv.executeSql( + "CREATE TABLE mysql_tb (" + + "id INT," + + "name STRING," + + "create_time TIMESTAMP(3)," + + "WATERMARK FOR create_time AS create_time - INTERVAL '60' SECOND," + + "primary key(id) NOT ENFORCED" + + ") " + + "WITH (\n" + + " 'connector' = 'mysql-cdc',\n" + + " 'hostname' = '127.0.0.1',\n" + + " 'port' = '3306',\n" + + " 'username' = 'root',\n" + + " 'password' = '123456',\n" + + " 'database-name' = 'test',\n" + + " 'scan.startup.mode' = 'latest-offset',\n" + + " 'server-time-zone' = 'Asia/Shanghai',\n" + + " 'table-name' = 'student' " + + ")"); + + tEnv.executeSql( + "CREATE TABLE mysql_tb1 (" + + "id INT," + + "name STRING," + + "age STRING," + + "create_time TIMESTAMP(3)," + + "WATERMARK FOR create_time AS create_time - INTERVAL '60' SECOND," + + "primary key(id) NOT ENFORCED" + + ") " + + "WITH (\n" + + " 'connector' = 'mysql-cdc',\n" + + " 'hostname' = '127.0.0.1',\n" + + " 'port' = '3306',\n" + + " 'username' = 'root',\n" + + " 'password' = '123456',\n" + + " 'database-name' = 'test',\n" + + " 'scan.startup.mode' = 'latest-offset',\n" + + " 'server-time-zone' = 'Asia/Shanghai',\n" + + " 'table-name' = 'student_copy1' " + + ")"); + + // tEnv.executeSql( + // "CREATE TABLE doris_tb (" + // + "name STRING," + // + "age INT," + // + "primary key(name) NOT ENFORCED" + // + ") " + // + "WITH (\n" + // + " 'connector' = 'doris',\n" + // + " 'fenodes' = '10.16.10.6:28737',\n" + // + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n" + // + " 'table.identifier' = 'test.student',\n" + //// + " 'lookup.cache.max-rows' = '1000'," + //// + " 'lookup.cache.ttl' = '1 hour'," + // // + " 'lookup.jdbc.async' = 'true',\n" + // + " 'username' = 'root',\n" + // + " 'password' = ''\n" + // + ")"); + + // tEnv.executeSql( + // "CREATE TABLE doris_tb (" + // + "name STRING," + // + "age INT," + // + "primary key(name) NOT ENFORCED" + // + ") " + // + "WITH (\n" + // + " 'connector' = 'jdbc',\n" + // + " 'url' = 'jdbc:mysql://10.16.10.6:29737/test'," + // + " 'table-name' = 'test_flink_a',\n" + // + " 'username' = 'root',\n" + // + " 'password' = ''\n" + // + ")"); + + Table table = + tEnv.sqlQuery( + "SELECT a.id, a.name, b.age\n" + + "FROM mysql_tb a\n" + + " left join mysql_tb1 FOR SYSTEM_TIME AS OF a.create_time AS b\n" + + " ON a.name = b.name"); + + tEnv.toRetractStream(table, Row.class).print(); + + env.execute(); + } +} diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java similarity index 88% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java index 9e82c6d41..ffca3e8a6 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java @@ -41,7 +41,7 @@ public static void main(String[] args) throws Exception { DataStreamSource> source = env.addSource( new SourceFunction>() { - private Integer id = 1; + private Integer id = 3000000; @Override public void run(SourceContext> out) @@ -73,8 +73,8 @@ public void cancel() {} + ") " + "WITH (\n" + " 'connector' = 'doris',\n" - + " 'fenodes' = '127.0.0.1:8030',\n" - + " 'jdbc-url' = 'jdbc:mysql://127.0.0.1:9030',\n" + + " 'fenodes' = '10.16.10.6:28737',\n" + + " 'jdbc-url' = 'jdbc:mysql://10.16.10.6:29737',\n" + " 'table.identifier' = 'ssb.customer',\n" + " 'lookup.jdbc.async' = 'true',\n" + " 'username' = 'root',\n" @@ -83,7 +83,8 @@ public void cancel() {} Table table = tEnv.sqlQuery( - "select a.id,a.uuid,b.c_name,b.c_nation,b.c_phone from doris_source a " + "select /*+ LOOKUP('table'='b', 'async'='true', 'output-mode' = 'allow_unordered', 'retry-predicate'='lookup_miss','retry-strategy'='fixed_delay', 'fixed-delay'='1s', 'max-attempts'='10') */" + + " a.id,a.uuid,b.c_name,b.c_nation,b.c_phone from doris_source a " + "left join lookup_dim_tbl FOR SYSTEM_TIME AS OF a.process_time b " + "ON a.id = b.c_custkey"); diff --git a/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java new file mode 100644 index 000000000..7cdf2608f --- /dev/null +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.flink.lookup; + +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import org.apache.doris.flink.container.AbstractITCaseService; +import org.apache.doris.flink.container.ContainerUtils; +import org.apache.doris.flink.table.DorisConfigOptions; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DorisLookupTableITCase extends AbstractITCaseService { + + private static final Logger LOG = LoggerFactory.getLogger(DorisLookupTableITCase.class); + private static final String DATABASE = "test_lookup"; + private static final String TABLE_READ_TBL = "tbl_read_tbl"; + private static final String TABLE_DIM_TBL = "tbl_dim_tbl"; + + private StreamExecutionEnvironment env; + private StreamTableEnvironment tEnv; + + @Before + public void before() { + env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(DEFAULT_PARALLELISM); + tEnv = StreamTableEnvironment.create(env); + } + + @Test + public void testLookupTable() throws Exception { + initializeTable(); + DataStreamSource sourceStream = env.fromElements(1, 2, 3, 4); + final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); + Schema schema = + Schema.newBuilder() + .column("f0", DataTypes.INT()) + .columnByExpression("proctime", "PROCTIME()") + .build(); + Table table = tEnv.fromDataStream(sourceStream, schema); + tEnv.createTemporaryView("source", table); + + String lookupDDL = + String.format( + "CREATE TABLE `doris_lookup`(" + + " `id` INTEGER," + + " `tinyintColumn` TINYINT," + + " `smallintColumn` SMALLINT," + + " `bigintColumn` BIGINT," + + " PRIMARY KEY (`id`) NOT ENFORCED" + + ") WITH (" + + "'connector' = '" + + DorisConfigOptions.IDENTIFIER + + "'," + + "'fenodes' = '%s'," + + "'jdbc-url' = '%s'," + + "'table.identifier' = '%s'," + + "'username' = '%s'," + + "'password' = '%s'," + + "'lookup.cache.max-rows' = '100'" + + ")", + getFenodes(), + getDorisQueryUrl(), + DATABASE + "." + TABLE_READ_TBL, + getDorisUsername(), + getDorisPassword()); + tEnv.executeSql(lookupDDL); + TableResult tableResult = + tEnv.executeSql( + "select source.f0," + + "tinyintColumn," + + "smallintColumn," + + "bigintColumn" + + " from `source`" + + " inner join `doris_lookup` FOR SYSTEM_TIME AS OF source.proctime on source.f0 = doris_lookup.id"); + + List actual = new ArrayList<>(); + try (CloseableIterator iterator = tableResult.collect()) { + while (iterator.hasNext()) { + actual.add(iterator.next().toString()); + } + } + + String[] expected = + new String[] { + "+I[1, 97, 27479, 8670353564751764000]", + "+I[2, 79, 17119, -4381380624467725000]", + "+I[3, -106, -14878, 1466614815449373200]" + }; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + } + + private void initializeTable() { + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE), + String.format( + "DROP TABLE IF EXISTS %s.%s", + DATABASE, DorisLookupTableITCase.TABLE_READ_TBL), + String.format( + "CREATE TABLE %s.%s ( \n" + + "`id` int(11),\n" + + "`tinyintColumn` tinyint(4),\n" + + "`smallintColumn` smallint(6),\n" + + "`bigintColumn` bigint(20),\n" + + ") DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ")\n", + DATABASE, DorisLookupTableITCase.TABLE_READ_TBL), + String.format( + "insert into %s.%s values (1,97,27479,8670353564751764000)", + DATABASE, DorisLookupTableITCase.TABLE_READ_TBL), + String.format( + "insert into %s.%s values (2,79,17119,-4381380624467725000)", + DATABASE, DorisLookupTableITCase.TABLE_READ_TBL), + String.format( + "insert into %s.%s values (3,-106,-14878,1466614815449373200)", + DATABASE, DorisLookupTableITCase.TABLE_READ_TBL)); + } + + @Test + public void testLookup() throws Exception { + initFlinkTable(); + Schema schema = + Schema.newBuilder() + .column("f0", DataTypes.INT()) + .columnByExpression("proctime", "PROCTIME()") + .build(); + + Table table = tEnv.fromDataStream(env.fromElements(1, 2, 3), schema); + tEnv.createTemporaryView("fact_table", table); + + String query = + "select fact_table.f0," + + "doris_lookup.score" + + " from fact_table" + + " left join doris_lookup FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id"; + TableResult tableResult = tEnv.executeSql(query); + CloseableIterator collectIter = tableResult.collect(); + List actual = collectSize(collectIter, 3); + String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, null]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + collectIter.close(); + + // mock data + tEnv.dropTemporaryView("fact_table"); + tEnv.createTemporaryView( + "fact_table", tEnv.fromDataStream(env.fromElements(1, 2, 3, 4), schema)); + + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format( + "insert into %s.%s values (3,300)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL)); + + TableResult tableResult2 = tEnv.executeSql(query); + CloseableIterator collectIter2 = tableResult2.collect(); + actual = collectSize(collectIter2, 4); + expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, 300]", "+I[4, null]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + collectIter2.close(); + } + + @Test + public void testRetryLookup() throws Exception { + initFlinkTable(); + Schema schema = + Schema.newBuilder() + .column("f0", DataTypes.INT()) + .columnByExpression("proctime", "PROCTIME()") + .build(); + + Table table = tEnv.fromDataStream(env.fromElements(1, 2, 3), schema); + tEnv.createTemporaryView("fact_table", table); + + String query = + "select /*+ LOOKUP('table'='doris_lookup', 'retry-predicate'='lookup_miss', " + + " 'retry-strategy'='fixed_delay', 'fixed-delay'='1s','max-attempts'='60') */" + + " fact_table.f0," + + "doris_lookup.score" + + " from fact_table" + + " left join doris_lookup FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id"; + TableResult tableResult = tEnv.executeSql(query); + CloseableIterator collectIter = tableResult.collect(); + // wait + Thread.sleep(10000); + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format( + "insert into %s.%s values (3,300)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL)); + + List actual = collectSize(collectIter, 3); + String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]", "+I[3, 300]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + collectIter.close(); + } + + @Test + public void testAsyncRetryLookup() throws Exception { + initFlinkTable(); + Schema schema = + Schema.newBuilder() + .column("f0", DataTypes.INT()) + .columnByExpression("proctime", "PROCTIME()") + .build(); + + Table table = tEnv.fromDataStream(env.fromElements(3, 2, 1), schema); + tEnv.createTemporaryView("fact_table", table); + + String query = + "select /*+ LOOKUP('table'='doris_lookup', 'retry-predicate'='lookup_miss', " + + " 'retry-strategy'='fixed_delay', 'output-mode'='allow_unordered', 'fixed-delay'='1s','max-attempts'='60') */" + + " fact_table.f0," + + "doris_lookup.score" + + " from fact_table" + + " left join doris_lookup /*+ OPTIONS('lookup.jdbc.async'='true') */ FOR SYSTEM_TIME AS OF fact_table.proctime on fact_table.f0 = doris_lookup.id"; + TableResult tableResult = tEnv.executeSql(query); + CloseableIterator collectIter = tableResult.collect(); + List actual = collectSize(collectIter, 2); + String[] expected = new String[] {"+I[1, 100]", "+I[2, 200]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format( + "insert into %s.%s values (3,300)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL)); + actual = collectSize(collectIter, 1); + expected = new String[] {"+I[3, 300]"}; + assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual.toArray())); + collectIter.close(); + } + + private static List collectSize(CloseableIterator iterator, int rows) + throws Exception { + List result = new ArrayList<>(); + while (result.size() < rows && iterator.hasNext()) { + result.add(iterator.next().toString()); + } + return result; + } + + private void initFlinkTable() { + ContainerUtils.executeSQLStatement( + getDorisQueryConnection(), + LOG, + String.format("CREATE DATABASE IF NOT EXISTS %s", DATABASE), + String.format( + "DROP TABLE IF EXISTS %s.%s", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL), + String.format( + "CREATE TABLE %s.%s ( \n" + + "`id` int(11),\n" + + "`score` int(11)\n" + + ") DISTRIBUTED BY HASH(`id`) BUCKETS 4\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ")\n", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL), + String.format( + "insert into %s.%s values (1,100)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL), + String.format( + "insert into %s.%s values (2,200)", + DATABASE, DorisLookupTableITCase.TABLE_DIM_TBL)); + + String lookupDDL = + String.format( + "CREATE TABLE `doris_lookup`(" + + " `id` INTEGER," + + " `score` INTEGER," + + " PRIMARY KEY (`id`) NOT ENFORCED" + + ") WITH (" + + "'connector' = '" + + DorisConfigOptions.IDENTIFIER + + "'," + + "'fenodes' = '%s'," + + "'jdbc-url' = '%s'," + + "'table.identifier' = '%s'," + + "'username' = '%s'," + + "'password' = '%s'" + + ")", + getFenodes(), + getDorisQueryUrl(), + DATABASE + "." + TABLE_DIM_TBL, + getDorisUsername(), + getDorisPassword()); + + tEnv.executeSql(lookupDDL); + } +} diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/RecordTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/lookup/RecordTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestRestService.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/TestRestService.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRouting.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRouting.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/OptionUtils.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/OptionUtils.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestUtil.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/TestUtil.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java index 3eb965978..891643f5b 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java @@ -352,30 +352,8 @@ public void testTableSourceTimestampFilterAndProjectionPushDown() throws Excepti List actualProjectionResult = generateExecuteSQLResult( tEnv, - "SELECT id,birthday,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down order by id"); - - List actualPushDownDatetimeResult = - generateExecuteSQLResult( - tEnv, - "SELECT id,birthday FROM doris_source_datetime_filter_and_projection_push_down where birthday >= '2023-01-01 00:00:00' order by id"); - List actualPushDownMicrosecondResult = - generateExecuteSQLResult( - tEnv, - "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time > '2023-01-01 00:00:00.000001' order by id"); - List actualPushDownNanosecondResult = - generateExecuteSQLResult( - tEnv, - "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time > '2023-01-01 00:00:00.000009001' order by id"); - - List actualPushDownNanosecondRoundDownResult = - generateExecuteSQLResult( - tEnv, - "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time >= '2023-01-01 00:00:00.999999001' order by id"); - List actualPushDownNanosecondRoundUpResult = - generateExecuteSQLResult( - tEnv, - "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time >= '2023-01-01 00:00:00.999999999' order by id"); - + "SELECT id,birthday,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down order by id", + 6); String[] expectedProjectionResult = new String[] { "+I[1, 2023-01-01T00:00, 2023-01-01T00:00:00.000001]", @@ -385,6 +363,16 @@ public void testTableSourceTimestampFilterAndProjectionPushDown() throws Excepti "+I[5, 2023-01-01T00:00:02, 2023-01-01T00:00:00.999999]", "+I[6, 2023-01-01T00:00:02, 2023-01-01T00:00:01]" }; + checkResultInAnyOrder( + "testTableSourceTimestampFilterAndProjectionPushDown", + expectedProjectionResult, + actualProjectionResult.toArray()); + + List actualPushDownDatetimeResult = + generateExecuteSQLResult( + tEnv, + "SELECT id,birthday FROM doris_source_datetime_filter_and_projection_push_down where birthday >= '2023-01-01 00:00:00' order by id", + 6); String[] expectedPushDownDatetimeResult = new String[] { "+I[1, 2023-01-01T00:00]", @@ -394,6 +382,16 @@ public void testTableSourceTimestampFilterAndProjectionPushDown() throws Excepti "+I[5, 2023-01-01T00:00:02]", "+I[6, 2023-01-01T00:00:02]" }; + checkResultInAnyOrder( + "testTableSourceTimestampFilterAndProjectionPushDown", + expectedPushDownDatetimeResult, + actualPushDownDatetimeResult.toArray()); + + List actualPushDownMicrosecondResult = + generateExecuteSQLResult( + tEnv, + "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time > '2023-01-01 00:00:00.000001' order by id", + 5); String[] expectedPushDownWithMicrosecondResult = new String[] { "+I[2, 2023-01-01T00:00:00.005]", @@ -402,7 +400,16 @@ public void testTableSourceTimestampFilterAndProjectionPushDown() throws Excepti "+I[5, 2023-01-01T00:00:00.999999]", "+I[6, 2023-01-01T00:00:01]" }; + checkResultInAnyOrder( + "testTableSourceTimestampFilterAndProjectionPushDown", + expectedPushDownWithMicrosecondResult, + actualPushDownMicrosecondResult.toArray()); + List actualPushDownNanosecondResult = + generateExecuteSQLResult( + tEnv, + "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time > '2023-01-01 00:00:00.000009001' order by id", + 4); String[] expectedPushDownWithNanosecondResult = new String[] { "+I[2, 2023-01-01T00:00:00.005]", @@ -410,40 +417,38 @@ public void testTableSourceTimestampFilterAndProjectionPushDown() throws Excepti "+I[5, 2023-01-01T00:00:00.999999]", "+I[6, 2023-01-01T00:00:01]" }; + checkResultInAnyOrder( + "testTableSourceTimestampFilterAndProjectionPushDown", + expectedPushDownWithNanosecondResult, + actualPushDownNanosecondResult.toArray()); + List actualPushDownNanosecondRoundDownResult = + generateExecuteSQLResult( + tEnv, + "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time >= '2023-01-01 00:00:00.999999001' order by id", + 3); String[] expectedPushDownWithNanosecondRoundDownResult = new String[] { "+I[4, 2023-01-01T00:00:00.999999]", "+I[5, 2023-01-01T00:00:00.999999]", "+I[6, 2023-01-01T00:00:01]" }; + checkResultInAnyOrder( + "testTableSourceTimestampFilterAndProjectionPushDown", + expectedPushDownWithNanosecondRoundDownResult, + actualPushDownNanosecondRoundDownResult.toArray()); + List actualPushDownNanosecondRoundUpResult = + generateExecuteSQLResult( + tEnv, + "SELECT id,brilliant_time FROM doris_source_datetime_filter_and_projection_push_down where brilliant_time >= '2023-01-01 00:00:00.999999999' order by id", + 3); String[] expectedPushDownWithNanosecondRoundUpResult = new String[] { "+I[4, 2023-01-01T00:00:00.999999]", "+I[5, 2023-01-01T00:00:00.999999]", "+I[6, 2023-01-01T00:00:01]" }; - checkResultInAnyOrder( - "testTableSourceTimestampFilterAndProjectionPushDown", - expectedProjectionResult, - actualProjectionResult.toArray()); - checkResultInAnyOrder( - "testTableSourceTimestampFilterAndProjectionPushDown", - expectedPushDownDatetimeResult, - actualPushDownDatetimeResult.toArray()); - checkResultInAnyOrder( - "testTableSourceTimestampFilterAndProjectionPushDown", - expectedPushDownWithMicrosecondResult, - actualPushDownMicrosecondResult.toArray()); - checkResultInAnyOrder( - "testTableSourceTimestampFilterAndProjectionPushDown", - expectedPushDownWithNanosecondResult, - actualPushDownNanosecondResult.toArray()); - checkResultInAnyOrder( - "testTableSourceTimestampFilterAndProjectionPushDown", - expectedPushDownWithNanosecondRoundDownResult, - actualPushDownNanosecondRoundDownResult.toArray()); checkResultInAnyOrder( "testTableSourceTimestampFilterAndProjectionPushDown", expectedPushDownWithNanosecondRoundUpResult, @@ -862,16 +867,16 @@ private static List fetchRows(Iterator iter) { return rows; } - private List generateExecuteSQLResult(StreamTableEnvironment tEnv, String executeSql) - throws Exception { + private List generateExecuteSQLResult( + StreamTableEnvironment tEnv, String executeSql, int rows) throws Exception { List actualResultList = new ArrayList<>(); TableResult tableResult = tEnv.executeSql(executeSql); try (CloseableIterator iterator = tableResult.collect()) { - while (iterator.hasNext()) { - + while (actualResultList.size() < rows && iterator.hasNext()) { actualResultList.add(iterator.next().toString()); } } + return actualResultList; } } diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java similarity index 98% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java index 7f9021f81..5fd313574 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -142,7 +143,7 @@ public void testEvalWithCache() throws Exception { lookupFunction.eval(5, StringData.fromString("5")); RowData keyRow = GenericRowData.of(4, StringData.fromString("D")); RowData keyRowNoExist = GenericRowData.of(5, StringData.fromString("5")); - Cache> cache = lookupFunction.getCache(); + Cache> cache = lookupFunction.getCache(); // empty data should cache assertEquals( cache.getIfPresent(keyRow), diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcDb2SyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMongoSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java similarity index 84% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java index e85e888fc..d2afbb167 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcMysqlSyncDatabaseCase.java @@ -48,32 +48,35 @@ public static void main(String[] args) throws Exception { String tablePrefix = ""; String tableSuffix = ""; Map mysqlConfig = new HashMap<>(); - mysqlConfig.put(MySqlSourceOptions.DATABASE_NAME.key(), "test"); + mysqlConfig.put(MySqlSourceOptions.DATABASE_NAME.key(), "aaaaa"); mysqlConfig.put(MySqlSourceOptions.HOSTNAME.key(), "127.0.0.1"); mysqlConfig.put(MySqlSourceOptions.PORT.key(), "3306"); mysqlConfig.put(MySqlSourceOptions.USERNAME.key(), "root"); - mysqlConfig.put(MySqlSourceOptions.PASSWORD.key(), "12345678"); + mysqlConfig.put(MySqlSourceOptions.PASSWORD.key(), "123456"); // add jdbc properties for MySQL mysqlConfig.put("jdbc.properties.use_ssl", "false"); Configuration config = Configuration.fromMap(mysqlConfig); Map sinkConfig = new HashMap<>(); - sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.20.30.1:8030"); + sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.16.10.6:28737"); sinkConfig.put(DorisConfigOptions.USERNAME.key(), "root"); sinkConfig.put(DorisConfigOptions.PASSWORD.key(), ""); - sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.20.30.1:9030"); + sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.16.10.6:29737"); sinkConfig.put(DorisConfigOptions.SINK_LABEL_PREFIX.key(), UUID.randomUUID().toString()); - sinkConfig.put("sink.enable-delete", "false"); + // sinkConfig.put("sink.enable-delete", "false"); Configuration sinkConf = Configuration.fromMap(sinkConfig); Map tableConfig = new HashMap<>(); tableConfig.put(DorisTableConfig.REPLICATION_NUM, "1"); - tableConfig.put(DorisTableConfig.TABLE_BUCKETS, "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50"); + // tableConfig.put(DorisTableConfig.TABLE_BUCKETS, + // "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50"); // String includingTables = "tbl1|tbl2|tbl3"; - String includingTables = "a_.*|b_.*|c"; + // tableConfig.put(DorisTableConfig.TABLE_PARTITIONS, + // "test_partition:create_dtime:day"); + String includingTables = "test_partition"; String excludingTables = ""; - String multiToOneOrigin = "a_.*|b_.*"; - String multiToOneTarget = "a|b"; + // String multiToOneOrigin = "a_.*|b_.*"; + // String multiToOneTarget = "a|b"; boolean ignoreDefaultValue = false; boolean useNewSchemaChange = true; String schemaChangeMode = SchemaChangeMode.DEBEZIUM_STRUCTURE.getName(); @@ -88,12 +91,12 @@ public static void main(String[] args) throws Exception { .setTableSuffix(tableSuffix) .setIncludingTables(includingTables) .setExcludingTables(excludingTables) - .setMultiToOneOrigin(multiToOneOrigin) - .setMultiToOneTarget(multiToOneTarget) + // .setMultiToOneOrigin(multiToOneOrigin) + // .setMultiToOneTarget(multiToOneTarget) .setIgnoreDefaultValue(ignoreDefaultValue) .setSinkConfig(sinkConf) .setTableConfig(tableConfig) - .setCreateTableOnly(false) + .setCreateTableOnly(true) .setNewSchemaChange(useNewSchemaChange) .setSchemaChangeMode(schemaChangeMode) .setSingleSink(singleSink) diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java similarity index 82% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java index 92600ffd6..d3f637a81 100644 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java +++ b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcOraclelSyncDatabaseCase.java @@ -41,12 +41,12 @@ public static void main(String[] args) throws Exception { String tablePrefix = ""; String tableSuffix = ""; Map sourceConfig = new HashMap<>(); - sourceConfig.put(OracleSourceOptions.DATABASE_NAME.key(), "XE"); + sourceConfig.put(OracleSourceOptions.DATABASE_NAME.key(), "HELOWIN"); sourceConfig.put(OracleSourceOptions.SCHEMA_NAME.key(), "ADMIN"); - sourceConfig.put(OracleSourceOptions.HOSTNAME.key(), "127.0.0.1"); - sourceConfig.put(OracleSourceOptions.PORT.key(), "1521"); + sourceConfig.put(OracleSourceOptions.HOSTNAME.key(), "10.16.10.6"); + sourceConfig.put(OracleSourceOptions.PORT.key(), "1523"); sourceConfig.put(OracleSourceOptions.USERNAME.key(), "admin"); - sourceConfig.put(OracleSourceOptions.PASSWORD.key(), ""); + sourceConfig.put(OracleSourceOptions.PASSWORD.key(), "admin123"); // sourceConfig.put("debezium.database.tablename.case.insensitive","false"); sourceConfig.put("debezium.log.mining.strategy", "online_catalog"); sourceConfig.put("debezium.log.mining.continuous.mine", "true"); @@ -54,18 +54,19 @@ public static void main(String[] args) throws Exception { Configuration config = Configuration.fromMap(sourceConfig); Map sinkConfig = new HashMap<>(); - sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.20.30.1:8030"); + sinkConfig.put(DorisConfigOptions.FENODES.key(), "10.16.10.6:28737"); sinkConfig.put(DorisConfigOptions.USERNAME.key(), "root"); sinkConfig.put(DorisConfigOptions.PASSWORD.key(), ""); - sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.20.30.1:9030"); + sinkConfig.put(DorisConfigOptions.JDBC_URL.key(), "jdbc:mysql://10.16.10.6:29737"); sinkConfig.put(DorisConfigOptions.SINK_LABEL_PREFIX.key(), UUID.randomUUID().toString()); Configuration sinkConf = Configuration.fromMap(sinkConfig); Map tableConfig = new HashMap<>(); tableConfig.put(DorisTableConfig.REPLICATION_NUM, "1"); - tableConfig.put(DorisTableConfig.TABLE_BUCKETS, "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50"); - String includingTables = "a_.*|b_.*|c"; - String excludingTables = ""; + // tableConfig.put(DorisTableConfig.TABLE_BUCKETS, + // "tbl1:10,tbl2:20,a.*:30,b.*:40,.*:50"); + String includingTables = "PER.*|PERSONS"; + String excludingTables = "PER123"; String multiToOneOrigin = "a_.*|b_.*"; String multiToOneTarget = "a|b"; boolean ignoreDefaultValue = false; @@ -79,11 +80,12 @@ public static void main(String[] args) throws Exception { .setTablePrefix(tablePrefix) .setTableSuffix(tableSuffix) .setIncludingTables(includingTables) - .setExcludingTables(excludingTables) - .setMultiToOneOrigin(multiToOneOrigin) - .setMultiToOneTarget(multiToOneTarget) + // .setExcludingTables(excludingTables) + // .setMultiToOneOrigin(multiToOneOrigin) + // .setMultiToOneTarget(multiToOneTarget) .setIgnoreDefaultValue(ignoreDefaultValue) .setSinkConfig(sinkConf) + .setSingleSink(false) .setTableConfig(tableConfig) .setCreateTableOnly(false) .setNewSchemaChange(useNewSchemaChange) diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcPostgresSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcSqlServerSyncDatabaseCase.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/CdcToolsTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DatabaseSyncTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/DorisTableConfigTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/MockSourceSchema.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/db2/Db2TypeTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBDatabaseSyncTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBSchemaTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDBTypeTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoDateConverterTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/tools/cdc/mongodb/MongoParsingProcessFunctionTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/DateToStringConverter.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java b/flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/MockSource.java similarity index 100% rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java rename to flink-doris-connector/flink-doris-connector-common/src/test/java/org/apache/doris/flink/utils/MockSource.java diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql b/flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql similarity index 100% rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql diff --git a/flink-doris-connector/src/test/resources/log4j2-test.properties b/flink-doris-connector/flink-doris-connector-common/src/test/resources/log4j2-test.properties similarity index 100% rename from flink-doris-connector/src/test/resources/log4j2-test.properties rename to flink-doris-connector/flink-doris-connector-common/src/test/resources/log4j2-test.properties diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml index 883964ecb..9d0dce5c0 100644 --- a/flink-doris-connector/pom.xml +++ b/flink-doris-connector/pom.xml @@ -26,8 +26,9 @@ under the License. 23 org.apache.doris - flink-doris-connector-${flink.major.version} - ${revision} + flink-doris-connector + 25.0.0-SNAPSHOT + pom Flink Doris Connector https://doris.apache.org/ @@ -38,6 +39,15 @@ under the License. repo + + flink-doris-connector-1.15 + flink-doris-connector-1.16 + flink-doris-connector-1.17 + flink-doris-connector-1.18 + flink-doris-connector-1.19 + flink-doris-connector-1.20 + flink-doris-connector-common + scm:git:https://git@github.com/apache/doris-flink-connector.git @@ -67,9 +77,8 @@ under the License. - 24.1.0-SNAPSHOT - 1.18.0 - 1.18 + 1.20.0 + 1.20 3.2.1 flink-python 0.16.0 diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java deleted file mode 100644 index f972d6e27..000000000 --- a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.flink.example; - -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -public class LookupJoinCdcExample { - public static void main(String[] args) throws Exception { - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(1); - // env.disableOperatorChaining(); - env.enableCheckpointing(30000); - final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); - - tEnv.executeSql( - "CREATE TABLE mysql_tb (" - + "id INT," - + "name STRING," - + "process_time as proctime()," - + "primary key(id) NOT ENFORCED" - + ") " - + "WITH (\n" - + " 'connector' = 'mysql-cdc',\n" - + " 'hostname' = '127.0.0.1',\n" - + " 'port' = '3306',\n" - + " 'username' = 'root',\n" - + " 'password' = '123456',\n" - + " 'database-name' = 'test',\n" - + " 'scan.startup.mode' = 'latest-offset',\n" - + " 'server-time-zone' = 'Asia/Shanghai',\n" - + " 'table-name' = 'fact_table' " - + ")"); - - tEnv.executeSql( - "CREATE TABLE doris_tb (" - + "id INT," - + "age INT," - + "dt DATE," - + "dtime TIMESTAMP," - + "primary key(id) NOT ENFORCED" - + ") " - + "WITH (\n" - + " 'connector' = 'doris',\n" - + " 'fenodes' = '127.0.0.1:8030',\n" - + " 'jdbc-url' = 'jdbc:mysql://127.0.0.1:9030',\n" - + " 'table.identifier' = 'test.dim_table_dt',\n" - + " 'lookup.cache.max-rows' = '1000'," - + " 'lookup.cache.ttl' = '1 hour'," - + " 'lookup.jdbc.async' = 'true',\n" - + " 'username' = 'root',\n" - + " 'password' = ''\n" - + ")"); - - Table table = - tEnv.sqlQuery( - "SELECT a.id, a.name, b.age, b.dt, b.dtime\n" - + "FROM mysql_tb a\n" - + " left join doris_tb FOR SYSTEM_TIME AS OF a.process_time AS b\n" - + " ON a.id = b.id"); - - tEnv.toRetractStream(table, Row.class).print(); - - env.execute(); - } -}