Merge branch 'master' into develop

Author: nautaa

build.sh

@@ -69,6 +69,10 @@ function parse_args
         then
             CPP_STANDARD_OPTION=20
             CPP_STANDARD_20_OPTION=ON
+        elif [[ "$i" == "-DBUILD_CDC_ONLY=ON" ]]
+        then
+            ENABLE_BOLT_OPTION=OFF
+            BUILD_ARGS+=("$i")
         elif [[ $NEED_MAKE == false ]]
         then
             BUILD_ARGS+=("$i")

cmake/Env.cmake

@@ -7,7 +7,7 @@ ob_define(DEP_3RD_DIR "${CMAKE_SOURCE_DIR}/deps/3rd")
 ob_define(DEVTOOLS_DIR "${CMAKE_SOURCE_DIR}/deps/3rd/usr/local/oceanbase/devtools")
 ob_define(DEP_DIR "${CMAKE_SOURCE_DIR}/deps/3rd/usr/local/oceanbase/deps/devel")
 
-ob_define(OB_BUILD_CDC OFF)
+ob_define(BUILD_CDC_ONLY OFF)
 ob_define(OB_USE_CLANG ON)
 ob_define(OB_ERRSIM OFF)
 ob_define(BUILD_NUMBER 1)
@@ -207,7 +207,7 @@ if(OB_BUILD_CPP_ODPS)
 endif()
 
 # should not use initial-exec for tls-model if building OBCDC.
-if(OB_BUILD_CDC)
+if(BUILD_CDC_ONLY)
   add_definitions(-DOB_BUILD_CDC_DISABLE_VSAG)
 else()
   add_definitions(-DENABLE_INITIAL_EXEC_TLS_MODEL)
@@ -264,11 +264,11 @@ if (OB_USE_CLANG)
   set(_CMAKE_TOOLCHAIN_LOCATION "${DEVTOOLS_DIR}/bin")
 
   if (OB_USE_ASAN)
-    ob_define(CMAKE_ASAN_FLAG "-fstack-protector-strong -fsanitize=address -fno-optimize-sibling-calls -fsanitize-blacklist=${ASAN_IGNORE_LIST}")
+    ob_define(CMAKE_ASAN_FLAG "-mllvm -asan-stack=0 -fsanitize=address -fno-optimize-sibling-calls -fsanitize-blacklist=${ASAN_IGNORE_LIST}")
   endif()
 
   if (OB_USE_LLD)
-    set(LD_OPT "-fuse-ld=${DEVTOOLS_DIR}/bin/ld.lld")
+    set(LD_OPT "-fuse-ld=${DEVTOOLS_DIR}/bin/ld.lld -Wno-unused-command-line-argument")
     set(REORDER_COMP_OPT "-ffunction-sections -fdebug-info-for-profiling")
     set(REORDER_LINK_OPT "-Wl,--no-rosegment,--build-id=sha1 ${HOTFUNC_OPT}")
     set(OB_LD_BIN "${DEVTOOLS_DIR}/bin/ld.lld")

cmake/Pack.cmake

@@ -88,7 +88,7 @@ install(FILES
 endif()
 
 ## oceanbase-cdc
-if (NOT OB_SO_CACHE AND OB_BUILD_CDC)
+if (NOT OB_SO_CACHE AND BUILD_CDC_ONLY)
   list(APPEND CPACK_COMPONENTS_ALL cdc)
   include(GNUInstallDirs)
   install(

cmake/RPM.cmake

@@ -3,9 +3,14 @@ set(CPACK_GENERATOR "RPM")
 set(CPACK_COMPONENTS_IGNORE_GROUPS 1)
 set(CPACK_RPM_COMPONENT_INSTALL ON)
 # use "server" as main component so its RPM filename won't have "server"
-set(CPACK_RPM_MAIN_COMPONENT "server")
+if (BUILD_CDC_ONLY)
+  set(CPACK_RPM_MAIN_COMPONENT "cdc")
+else()
+  set(CPACK_RPM_MAIN_COMPONENT "server")
+endif()
 # let rpmbuild determine rpm filename
 set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
+set(CMAKE_INSTALL_LIBDIR "lib64")
 ## Stardard debuginfo generating instructions in cmake.  However 6u
 ## server with rpm-4.8.0 which doesn't support dwarf4 won't generate
 ## BUILDID for RPM. And Our debuginfo package doesn't contain source
@@ -40,7 +45,7 @@ set(CPACK_RPM_PACKAGE_DESCRIPTION ${CPACK_PACKAGE_DESCRIPTION})
 set(CPACK_RPM_PACKAGE_LICENSE "Mulan PubL v2.")
 set(CPACK_RPM_DEFAULT_USER "admin")
 set(CPACK_RPM_DEFAULT_GROUP "admin")
-if (OB_BUILD_OPENSOURCE)
+if (OB_BUILD_OPENSOURCE AND NOT BUILD_CDC_ONLY)
   set(DEBUG_INSTALL_POST "mv $RPM_BUILD_ROOT/../server/home/admin/oceanbase/bin/obshell %{_builddir}/obshell; %{_rpmconfigdir}/find-debuginfo.sh %{?_find_debuginfo_opts} %{_builddir}/%{?buildsubdir}; mv %{_builddir}/obshell $RPM_BUILD_ROOT/../server/home/admin/oceanbase/bin/obshell; %{nil}")
 else()
   set(DEBUG_INSTALL_POST "%{_rpmconfigdir}/find-debuginfo.sh %{?_find_debuginfo_opts} %{_builddir}/%{?buildsubdir};%{nil}")
@@ -102,14 +107,25 @@ install(FILES
   COMPONENT server)
 endif()
 
+if (BUILD_CDC_ONLY)
+  message(STATUS "oceanbase build cdc only")
+  set(CPACK_COMPONENTS_ALL cdc)
+  set(CPACK_PACKAGE_NAME "oceanbase-cdc")
+  if (OB_BUILD_OPENSOURCE)
+    set(CPACK_PACKAGE_NAME "oceanbase-ce-cdc")
+  endif()
+else()
+  add_custom_target(bitcode_to_elf ALL
+    DEPENDS ${BITCODE_TO_ELF_LIST})
+  add_custom_target(ob_table ALL
+    DEPENDS obtable obtable_static)
+endif()
+message(STATUS "Cpack Components:${CPACK_COMPONENTS_ALL}")
+
 # install cpack to make everything work
 include(CPack)
 
 #add rpm target to create RPMS
 add_custom_target(rpm
   COMMAND +make package
-  DEPENDS
-  observer obcdc_tailf obtable obtable_static
-  ob_admin ob_error ob_sql_proxy_parser_static
-  ${BITCODE_TO_ELF_LIST}
   )

deps/init/oceanbase.al8.aarch64.deps

@@ -27,7 +27,7 @@ devdeps-zlib-static-1.2.13-82024102414.al8.aarch64.rpm
 devdeps-ncurses-static-6.2-112024092810.al8.aarch64.rpm
 devdeps-boost-1.74.0-72024092811.al8.aarch64.rpm
 devdeps-s2geometry-0.9.0-112024100811.al8.aarch64.rpm
-devdeps-icu-69.1-192024092811.al8.aarch64.rpm
+devdeps-icu-69.1-232024120517.al8.aarch64.rpm
 devdeps-cos-c-sdk-5.0.21-282024102810.al8.aarch64.rpm
 devdeps-s3-cpp-sdk-1.11.156-222024100811.al8.aarch64.rpm
 devdeps-protobuf-c-1.4.1-100000132024092815.al8.aarch64.rpm
@@ -51,7 +51,7 @@ obdevtools-llvm-17.0.3-232024111415.al8.aarch64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-122024093010.al8.aarch64.rpm
 obdevtools-obstack-2.0.4-42024101017.al8.aarch64.rpm
 devdeps-rocksdb-6.22.1.1-172024092818.al8.aarch64.rpm
-obshell-4.2.4.2-12024102115.al8.aarch64.rpm target=community
+obshell-4.2.5.0-12024121011.al8.aarch64.rpm target=community
 
 [test-utils]
 ob-deploy-2.10.0-4.al8.aarch64.rpm target=community

deps/init/oceanbase.al8.x86_64.deps

@@ -28,7 +28,7 @@ devdeps-zlib-static-1.2.13-82024102414.al8.x86_64.rpm
 devdeps-ncurses-static-6.2-112024092810.al8.x86_64.rpm
 devdeps-boost-1.74.0-72024092811.al8.x86_64.rpm
 devdeps-s2geometry-0.9.0-112024100811.al8.x86_64.rpm
-devdeps-icu-69.1-192024092811.al8.x86_64.rpm
+devdeps-icu-69.1-232024120517.al8.x86_64.rpm
 devdeps-cloud-qpl-1.1.0-312024092814.al8.x86_64.rpm
 devdeps-cos-c-sdk-5.0.21-282024102810.al8.x86_64.rpm
 devdeps-s3-cpp-sdk-1.11.156-222024100811.al8.x86_64.rpm
@@ -54,7 +54,7 @@ obdevtools-llvm-17.0.3-232024111415.el8.x86_64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-122024093010.al8.x86_64.rpm
 devdeps-rocksdb-6.22.1.1-172024092818.al8.x86_64.rpm
 obdevtools-obstack-2.0.4-42024101017.al8.x86_64.rpm
-obshell-4.2.4.2-12024102115.al8.x86_64.rpm target=community
+obshell-4.2.5.0-12024121011.al8.x86_64.rpm target=community
 
 [test-utils]
 ob-deploy-2.10.0-4.al8.x86_64.rpm target=community

deps/init/oceanbase.el7.aarch64.deps

@@ -51,7 +51,7 @@ obdevtools-llvm-17.0.3-232024111415.el7.aarch64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-52024052811.el7.aarch64.rpm
 devdeps-rocksdb-6.22.1.1-52022100420.el7.aarch64.rpm
 obstack-2.0.4-172024070513.el7.aarch64.rpm
-obshell-4.2.4.3-12024110711.el7.aarch64.rpm target=community
+obshell-4.2.5.0-12024121011.el7.aarch64.rpm target=community
 
 [test-utils]
 ob-deploy-2.10.0-3.el7.aarch64.rpm target=community

deps/init/oceanbase.el7.x86_64.deps

@@ -55,7 +55,7 @@ obdevtools-llvm-17.0.3-232024111415.el7.x86_64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-52024052811.el7.x86_64.rpm
 devdeps-rocksdb-6.22.1.1-52022100420.el7.x86_64.rpm
 obstack-2.0.4-172024070513.el7.x86_64.rpm
-obshell-4.2.4.3-12024110711.el7.x86_64.rpm target=community
+obshell-4.2.5.0-12024121011.el7.x86_64.rpm target=community
 
 [test-utils]
 ob-deploy-2.10.0-3.el7.x86_64.rpm target=community

deps/init/oceanbase.el8.aarch64.deps

@@ -51,7 +51,7 @@ obdevtools-llvm-17.0.3-232024111415.el8.aarch64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-52024052811.el8.aarch64.rpm
 devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm
 obstack-2.0.4-172024070513.el8.aarch64.rpm
-obshell-4.2.4.3-12024110711.el8.aarch64.rpm target=community
+obshell-4.2.5.0-12024121011.el8.aarch64.rpm target=community
 
 [test-utils]
 ob-deploy-2.10.0-3.el8.aarch64.rpm target=community

deps/init/oceanbase.el8.x86_64.deps

@@ -54,7 +54,7 @@ obdevtools-llvm-17.0.3-232024111415.el8.x86_64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-52024052811.el8.x86_64.rpm
 devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm
 obstack-2.0.4-172024070513.el8.x86_64.rpm
-obshell-4.2.4.3-12024110711.el8.x86_64.rpm target=community
+obshell-4.2.5.0-12024121011.el8.x86_64.rpm target=community
 
 [test-utils]
 ob-deploy-2.10.0-3.el8.x86_64.rpm target=community

deps/init/oceanbase.el9.aarch64.deps

@@ -39,6 +39,7 @@ devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
 devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
 devdeps-apache-orc-1.8.3-392024091317.el8.aarch64.rpm
 devdeps-vsag-1.0.0-352024102510.el8.aarch64.rpm
+devdeps-fast-float-6.1.3-42024112122.el8.aarch64.rpm
 
 [deps-el9]
 devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9
@@ -57,7 +58,7 @@ obdevtools-llvm-17.0.3-232024111415.el8.aarch64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-52024052811.el8.aarch64.rpm
 devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm
 obstack-2.0.4-172024070513.el8.aarch64.rpm
-obshell-4.2.4.3-12024110711.el8.aarch64.rpm target=community
+obshell-4.2.5.0-12024121011.el8.aarch64.rpm target=community
 
 [test-utils]
 ob-deploy-1.6.0-41.el8.aarch64.rpm target=community

deps/init/oceanbase.el9.x86_64.deps

@@ -42,6 +42,7 @@ devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
 devdeps-hyperscan-5.4.2-242024090219.el8.x86_64.rpm
 devdeps-apache-orc-1.8.3-392024091317.el8.x86_64.rpm
 devdeps-vsag-1.0.0-352024102510.el8.x86_64.rpm
+devdeps-fast-float-6.1.3-42024112122.el8.x86_64.rpm
 
 [deps-el9]
 devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9
@@ -60,7 +61,7 @@ obdevtools-llvm-17.0.3-232024111415.el8.x86_64.rpm cpp_standard=20
 devdeps-oblogmsg-1.1-52024052811.el8.x86_64.rpm
 devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm
 obstack-2.0.4-172024070513.el8.x86_64.rpm
-obshell-4.2.4.3-12024110711.el8.x86_64.rpm target=community
+obshell-4.2.5.0-12024121011.el8.x86_64.rpm target=community
 
 [test-utils]
 ob-deploy-1.6.0-41.el8.x86_64.rpm target=community

deps/oblib/src/common/ob_target_specific.cpp

@@ -11,11 +11,31 @@
  */
 
 #define USING_LOG_PREFIX COMMON
+#include "common/ob_target_specific.h"
 
 namespace oceanbase
 {
 namespace common
 {
 
+uint32_t arches;
+void init_arches()
+{
+  arches = 0;
+  const CpuFlagSet flags;
+  if (flags.have_flag(CpuFlag::SSE4_2)) {
+    arches |= static_cast<uint32_t>(ObTargetArch::SSE42);
+  }
+  if (flags.have_flag(CpuFlag::AVX)) {
+    arches |= static_cast<uint32_t>(ObTargetArch::AVX);
+  }
+  if (flags.have_flag(CpuFlag::AVX2)) {
+    arches |= static_cast<uint32_t>(ObTargetArch::AVX2);
+  }
+  if (flags.have_flag(CpuFlag::AVX512BW)) {
+    arches |= static_cast<uint32_t>(ObTargetArch::AVX512);
+  }
+}
+
 } // namespace common
 } // namespace oceanbase

deps/oblib/src/common/ob_target_specific.h

@@ -215,27 +215,11 @@ OB_DECLARE_AVX512_SPECIFIC_CODE(
 
 #endif
 
-OB_INLINE uint32_t get_supported_archs()
-{
-  uint32_t result = 0;
-  if (CpuFlagSet::get_instance().have_flag(CpuFlag::SSE4_2)) {
-    result |= static_cast<uint32_t>(ObTargetArch::SSE42);
-  }
-  if (CpuFlagSet::get_instance().have_flag(CpuFlag::AVX)) {
-    result |= static_cast<uint32_t>(ObTargetArch::AVX);
-  }
-  if (CpuFlagSet::get_instance().have_flag(CpuFlag::AVX2)) {
-    result |= static_cast<uint32_t>(ObTargetArch::AVX2);
-  }
-  if (CpuFlagSet::get_instance().have_flag(CpuFlag::AVX512BW)) {
-    result |= static_cast<uint32_t>(ObTargetArch::AVX512);
-  }
-  return result;
-}
+void init_arches();
+extern uint32_t arches;
 
 OB_INLINE bool is_arch_supported(ObTargetArch arch)
 {
-  static uint32_t arches = get_supported_archs();
   return arch == ObTargetArch::Default || (arches & static_cast<uint32_t>(arch));
 }
 

deps/oblib/src/common/storage/ob_io_device.h

@@ -491,7 +491,9 @@ class ObIODevice
   virtual int64_t get_max_block_size(int64_t reserved_size) const = 0;
   virtual int64_t get_max_block_count(int64_t reserved_size) const = 0;
   virtual int64_t get_reserved_block_count() const = 0;
-  virtual int check_space_full(const int64_t required_size) const = 0;
+  virtual int check_space_full(
+    const int64_t required_size,
+    const bool alarm_if_space_full = true) const = 0;
   virtual int check_write_limited() const = 0;
 
   // ref cnt

deps/oblib/src/lib/CMakeLists.txt

@@ -423,9 +423,13 @@ add_library(ob_malloc STATIC)
 target_link_libraries(ob_malloc ob_malloc_object)
 
 add_library(malloc_hook STATIC
-  alloc/malloc_hook.cpp
-  alloc/malloc_hook.h
-  alloc/malloc_hook_extended.cpp)
+    alloc/malloc_hook.cpp
+    alloc/malloc_hook.h
+    alloc/malloc_hook_extended.cpp)
+if (BUILD_CDC_ONLY)
+  set_target_properties(malloc_hook PROPERTIES EXCLUDE_FROM_ALL TRUE)
+endif()
+
 target_link_libraries(malloc_hook oblib_base)
 
 if(OB_BUILD_CLOSE_MODULES)
@@ -445,7 +449,7 @@ target_link_libraries(oblib_lib
   ${DEP_DIR}/lib/liblzma.a
 )
 
-if (NOT OB_BUILD_CDC)
+if (NOT BUILD_CDC_ONLY)
 # vsag vector library
 target_link_directories(
   oblib_lib PUBLIC ${DEP_DIR}/lib ${DEP_DIR}/lib/vsag_lib