Merge branch 'master' into develop

Author: hnwyllmm

.gitignore

@@ -152,11 +152,11 @@ src/sql/parser/sql_parser_oracle_utf8_mode_lex.c
 src/sql/parser/sql_parser_oracle_utf8_mode_lex.h
 src/sql/parser/sql_parser_oracle_utf8_mode_tab.c
 src/sql/parser/sql_parser_oracle_utf8_mode_tab.h
-src/sql/parser/sql_parser_oracle_latin1_mode_lex.c
-src/sql/parser/sql_parser_oracle_latin1_mode_lex.h
-src/sql/parser/sql_parser_oracle_latin1_mode_tab.c
-src/sql/parser/sql_parser_oracle_latin1_mode_tab.h
-src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c
+src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c
+src/sql/parser/sql_parser_oracle_single_byte_mode_lex.h
+src/sql/parser/sql_parser_oracle_single_byte_mode_tab.c
+src/sql/parser/sql_parser_oracle_single_byte_mode_tab.h
+src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c
 src/sql/parser/_gen_parser.output
 src/sql/parser/_gen_parser.error
 src/pl/parser/pl_parser_mysql_mode_lex.c
@@ -171,6 +171,7 @@ src/pl/parser/_gen_parser.error
 src/pl/parser/_gen_pl_parser.output
 src/pl/parser/pl_parser_mysql_mode.output
 src/pl/parser/pl_parser_oracle_mode.output
+src/share/inner_table/sys_package/system_package.cpp
 
 ############# close_modules #############
 close_modules/oracle_pl/pl/parser/*.output

cmake/Env.cmake

@@ -117,6 +117,8 @@ if(OB_BUILD_CLOSE_MODULES)
   ob_define(OB_BUILD_ORACLE_PL ON)
   # dblink
   ob_define(OB_BUILD_DBLINK ON)
+  # odps
+  ob_define(OB_BUILD_CPP_ODPS ON)
   # 仲裁功能
   ob_define(OB_BUILD_ARBITRATION ON)
 
@@ -172,6 +174,10 @@ if(OB_BUILD_DBLINK)
   add_definitions(-DOB_BUILD_DBLINK)
 endif()
 
+if(OB_BUILD_CPP_ODPS)
+  add_definitions(-DOB_BUILD_CPP_ODPS)
+endif()
+
 # should not use initial-exec for tls-model if building OBCDC.
 if(NOT OB_BUILD_CDC)
   add_definitions(-DENABLE_INITIAL_EXEC_TLS_MODEL)

deps/init/oceanbase.el7.aarch64.deps

@@ -33,6 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.aarch64.rpm
 devdeps-protobuf-c-1.4.1-100000072023102410.el7.aarch64.rpm
 devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.aarch64.rpm
 devdeps-apache-arrow-9.0.0-302024052920.el7.aarch64.rpm
+# devdeps-apache-orc-1.8.3-202024072510.el7.aarch64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm

deps/init/oceanbase.el7.x86_64.deps

@@ -36,6 +36,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.x86_64.rpm
 devdeps-protobuf-c-1.4.1-100000062023102016.el7.x86_64.rpm
 devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.x86_64.rpm
 devdeps-apache-arrow-9.0.0-222024052223.el7.x86_64.rpm
+# devdeps-apache-orc-1.8.3-202024072510.el7.x86_64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm

deps/init/oceanbase.el8.aarch64.deps

@@ -33,6 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm
 devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm
 devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
 devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
+# devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm

deps/init/oceanbase.el8.x86_64.deps

@@ -35,6 +35,8 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm
 devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm
 devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
 devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm
+devdeps-odps-cpp-sdk-1.0.0-482024080517.el8.x86_64.rpm
+# devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm
 
 [tools]
 obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm

deps/init/oceanbase.el9.aarch64.deps

@@ -37,6 +37,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm
 devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm
 devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
 devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
+# devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm
 
 [deps-el9]
 devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9

deps/init/oceanbase.el9.x86_64.deps

@@ -39,6 +39,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm
 devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm
 devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm
 devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
+# devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm
 
 [deps-el9]
 devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9

deps/oblib/src/CMakeLists.txt

@@ -21,6 +21,7 @@ target_include_directories(
   ${DEP_DIR}/include/apr-1/
   ${DEP_DIR}/include/icu/common
   ${DEP_DIR}/include/apache-arrow
+  # ${DEP_DIR}/include/apache-orc
   ${USSL_INCLUDE_DIRS}
 )
 
@@ -208,6 +209,12 @@ target_link_libraries(oblib_base_base_base
   ${DEP_DIR}/lib64/libarrow.a
   ${DEP_DIR}/lib64/libparquet.a
   ${DEP_DIR}/lib64/libarrow_bundled_dependencies.a
+  # ${DEP_DIR}/lib64/liborc.a
+  # ${DEP_DIR}/lib64/libsnappy.a
+  # ${DEP_DIR}/lib64/libprotoc.a
+  # ${DEP_DIR}/lib64/libprotobuf.a
+  # ${DEP_DIR}/lib64/liblz4.a
+  # ${DEP_DIR}/lib64/libzstd.a
   -L${DEP_DIR}/var/usr/lib64
   -L${DEP_DIR}/var/usr/lib
   -L${DEP_3RD_DIR}/usr/lib
@@ -237,6 +244,12 @@ target_link_libraries(oblib_base_base_base
   ${DEP_DIR}/lib64/libarrow.a
   ${DEP_DIR}/lib64/libparquet.a
   ${DEP_DIR}/lib64/libarrow_bundled_dependencies.a
+  # ${DEP_DIR}/lib64/liborc.a
+  # ${DEP_DIR}/lib64/libsnappy.a
+  # ${DEP_DIR}/lib64/libprotoc.a
+  # ${DEP_DIR}/lib64/libprotobuf.a
+  # ${DEP_DIR}/lib64/liblz4.a
+  # ${DEP_DIR}/lib64/libzstd.a
   -L${DEP_DIR}/var/usr/lib64
   -L${DEP_DIR}/var/usr/lib
   -L${DEP_3RD_DIR}/usr/lib

deps/oblib/src/common/ob_field.cpp

@@ -31,10 +31,13 @@ int ObParamedSelectItemCtx::deep_copy(const ObParamedSelectItemCtx &other, ObIAl
     LOG_WARN("invalid null allocator", K(ret));
   } else if (OB_FAIL(ob_write_string(*allocator, other.paramed_cname_, paramed_cname_))) {
     LOG_WARN("failed to write stirng", K(ret));
+  } else if (OB_FAIL(param_str_offsets_.assign(other.param_str_offsets_))) {
+    LOG_WARN("failed to deep copy param string offsets", K(ret));
+  } else if (OB_FAIL(param_idxs_.assign(other.param_idxs_))) {
+    LOG_WARN("failed to deep copy param idxs", K(ret));
+  } else if (OB_FAIL(neg_param_idxs_.assign(other.neg_param_idxs_))) {
+    LOG_WARN("failed to deep copy neg param idxs", K(ret));
   } else {
-    param_str_offsets_ = other.param_str_offsets_;
-    param_idxs_ = other.param_idxs_;
-    neg_param_idxs_ = other.neg_param_idxs_;
     esc_str_flag_ = other.esc_str_flag_;
     need_check_dup_name_ = other.need_check_dup_name_;
     is_column_field_ = other.is_column_field_;

deps/oblib/src/common/ob_member.cpp

@@ -86,73 +86,92 @@ void ObMember::reset_migrating()
   flag_ &= ~(1UL << MIGRATING_FLAG_BIT);
 }
 
-OB_SERIALIZE_MEMBER(ObMember, server_, timestamp_, flag_);
-
-bool ObReplicaMember::is_readonly_replica() const
+bool ObMember::is_columnstore() const
 {
-  return REPLICA_TYPE_READONLY == replica_type_;
+  return (flag_ >> COLUMNSTORE_FLAG_BIT) & 1U;
 }
 
-void ObReplicaMember::reset()
+void ObMember::set_columnstore()
 {
-  ObMember::reset();
-  replica_type_ = REPLICA_TYPE_FULL;
-  region_ = DEFAULT_REGION_NAME;
-  memstore_percent_ = 100;
+  flag_ |= (1UL << COLUMNSTORE_FLAG_BIT);
 }
 
-bool ObReplicaMember::is_valid() const
+void ObMember::reset_columnstore()
 {
-  return ObMember::is_valid()
-         && ObReplicaTypeCheck::is_replica_type_valid(replica_type_)
-         && !region_.is_empty()
-         && memstore_percent_ <= 100
-         && memstore_percent_ >= 0;
+  flag_ &= ~(1UL << COLUMNSTORE_FLAG_BIT);
 }
 
-common::ObReplicaType ObReplicaMember::get_replica_type() const
+OB_SERIALIZE_MEMBER(ObMember, server_, timestamp_, flag_);
+
+bool ObReplicaMember::is_readonly_replica() const
 {
-  return replica_type_;
+  return REPLICA_TYPE_READONLY == replica_type_;
 }
 
-int ObReplicaMember::set_replica_type(const common::ObReplicaType replica_type)
+int ObReplicaMember::init(
+    const common::ObAddr &server,
+    const int64_t timestamp,
+    const common::ObReplicaType replica_type)
 {
   int ret = OB_SUCCESS;
-  if (!ObReplicaTypeCheck::is_replica_type_valid(replica_type)) {
+  reset();
+  if (OB_UNLIKELY(!server.is_valid()
+                  || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) {
     ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(WARN, "invalid argument", K(ret), K(server), K(replica_type));
   } else {
+    server_ = server;
+    timestamp_ = timestamp;
     replica_type_ = replica_type;
+    if (REPLICA_TYPE_COLUMNSTORE == replica_type) {
+      ObMember::set_columnstore();
+    }
   }
   return ret;
 }
 
-const common::ObRegion &ObReplicaMember::get_region() const
+int ObReplicaMember::init(
+    const ObMember &member,
+    const common::ObReplicaType replica_type)
 {
-  return region_;
+  int ret = OB_SUCCESS;
+  reset();
+  if (OB_UNLIKELY(!member.is_valid()
+                  || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) {
+    ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(WARN, "invalid argument", K(ret), K(member), K(replica_type));
+  } else if (OB_FAIL(ObMember::assign(member))) {
+    COMMON_LOG(WARN, "failed to assign member", K(ret), K(member));
+  } else if (OB_FALSE_IT(replica_type_ = replica_type)) {
+    // should never be here
+  } else if (OB_UNLIKELY(! is_valid())) { // check flag_ and replica_type_ correct
+    ret = OB_INVALID_ARGUMENT;
+    COMMON_LOG(WARN, "invalid argument", K(ret), K(member), K(replica_type), KPC(this));
+  }
+  return ret;
 }
 
-int ObReplicaMember::set_member(const ObMember &member)
+void ObReplicaMember::reset()
 {
- int ret = OB_SUCCESS;
+  ObMember::reset();
+  replica_type_ = REPLICA_TYPE_FULL;
+  memstore_percent_ = 100;
+}
 
- if (!member.is_valid()) {
-   ret = OB_INVALID_ARGUMENT;
-   COMMON_LOG(WARN, "invalid args", K(ret), K(member));
- } else if (OB_FAIL(ObMember::assign(member))) {
-   COMMON_LOG(WARN, "failed to assign member", K(ret), K(member));
- }
- return ret;
+bool ObReplicaMember::is_valid() const
+{
+  // columnstore bit is 1 if and only if replica_type is C
+  bool is_flag_valid = (is_columnstore() == (REPLICA_TYPE_COLUMNSTORE == replica_type_));
+  return ObMember::is_valid()
+         && ObReplicaTypeCheck::is_replica_type_valid(replica_type_)
+         && is_flag_valid
+         && memstore_percent_ <= 100
+         && memstore_percent_ >= 0;
 }
 
-int ObReplicaMember::set_region(const common::ObRegion &region)
+common::ObReplicaType ObReplicaMember::get_replica_type() const
 {
-  int ret = OB_SUCCESS;
-  if (region.is_empty()) {
-    ret = OB_INVALID_ARGUMENT;
-  } else {
-    region_ = region;
-  }
-  return ret;
+  return replica_type_;
 }
 
 ObReplicaMember &ObReplicaMember::operator=(const ObReplicaMember &rhs)

deps/oblib/src/common/ob_member.h

@@ -49,11 +49,16 @@ class ObMember
   void set_migrating();
   void reset_migrating();
 
+  bool is_columnstore() const;
+  void set_columnstore();
+  void reset_columnstore();
+
   TO_STRING_KV(K_(server), K_(timestamp), K_(flag));
   TO_YSON_KV(OB_Y_(server), OB_ID(t), timestamp_, OB_Y_(flag));
   OB_UNIS_VERSION(1);
 protected:
   static const int64_t MIGRATING_FLAG_BIT = 1;
+  static const int64_t COLUMNSTORE_FLAG_BIT = 0;
   common::ObAddr server_;
   int64_t timestamp_;
   int64_t flag_;
@@ -90,61 +95,46 @@ inline int member_to_string(const common::ObMember &member, ObSqlString &member_
 class ObReplicaMember : public ObMember
 {
 public:
+  // default constructor
   ObReplicaMember()
     : ObMember(),
       replica_type_(REPLICA_TYPE_FULL),
-      region_(DEFAULT_REGION_NAME),
       memstore_percent_(100)
   {}
+  // construct with only server and timestamp, when we don't know or care about replica_type
+  // TODO(cangming.zl): remove this constructor when DRTask do not use it.
   ObReplicaMember(const common::ObAddr &server,
                   const int64_t timestamp)
-    : ObMember(server, timestamp),
+    : ObMember(ObMember(server, timestamp)),
       replica_type_(REPLICA_TYPE_FULL),
-      region_(DEFAULT_REGION_NAME),
-      memstore_percent_(100)
-  {}
-  ObReplicaMember(const ObMember &member)
-    : ObMember(member),
-      replica_type_(REPLICA_TYPE_FULL),
-      region_(DEFAULT_REGION_NAME),
-      memstore_percent_(100)
-  {}
-  /* After the subsequent type conversion code is completed, remove the constructor */
-  ObReplicaMember(const common::ObAddr &server,
-                  const int64_t timestamp,
-                  const common::ObReplicaType replica_type)
-    : ObMember(server, timestamp),
-      replica_type_(replica_type),
-      region_(DEFAULT_REGION_NAME),
       memstore_percent_(100)
   {}
+  // construct with server, timestamp and replica_type,
+  //   this func will set columnstore flag if replica_type is C.
   ObReplicaMember(const common::ObAddr &server,
                   const int64_t timestamp,
                   const common::ObReplicaType replica_type,
-                  const int64_t memstore_percent)
-    : ObMember(server, timestamp),
+                  const int64_t memstore_percent = 100)
+    : ObMember(ObMember(server, timestamp)),
       replica_type_(replica_type),
-      region_(DEFAULT_REGION_NAME),
       memstore_percent_(memstore_percent)
-  {}
-  ObReplicaMember(const common::ObAddr &server,
-                  const int64_t timestamp,
-                  const common::ObReplicaType replica_type,
-                  const common::ObRegion &region,
-                  const int64_t memstore_percent)
-    : ObMember(server, timestamp),
-      replica_type_(replica_type),
-      region_(region),
-      memstore_percent_(memstore_percent)
-  {}
+  {
+    if (REPLICA_TYPE_COLUMNSTORE == replica_type) {
+      ObMember::set_columnstore();
+    }
+  }
 public:
+  // init with server, timestamp, replica_type.
+  //   this func will set columnstore flag if replica_type is C.
+  int init(const common::ObAddr &server,
+           const int64_t timestamp,
+           const common::ObReplicaType replica_type);
+  // init with existing member whose flag_ may have been set.
+  //   this function will check whether flag_ is consistent with replica_type.
+  int init(const ObMember &member,
+           const common::ObReplicaType replica_type);
   common::ObReplicaType get_replica_type() const;
-  int set_replica_type(const common::ObReplicaType replica_type);
-  const common::ObRegion &get_region() const;
-  int set_region(const common::ObRegion &region);
-  int set_member(const ObMember &member);
   int64_t get_memstore_percent() const { return memstore_percent_; }
-  void set_memstore_percent(const int64_t memstore_percent) { memstore_percent_ = memstore_percent; }
   virtual void reset();
   virtual bool is_valid() const;
   virtual bool is_readonly_replica() const;
@@ -154,8 +144,8 @@ class ObReplicaMember : public ObMember
   OB_UNIS_VERSION(1);
 private:
   common::ObReplicaType replica_type_;
-  common::ObRegion region_;
-  int64_t memstore_percent_;
+  int64_t memstore_percent_;                       // obsolate, only as placeholder
+  common::ObRegion region_ = DEFAULT_REGION_NAME;  // obsolate, only as placeholder
 };
 } // namespace common
 } // namespace oceanbase