Merge pull request #11 from xujiahe0529/python_udf_merge

update:support load python code from .py file

Author: watch2bear

src/objit/include/objit/common/ob_item_type.h

@@ -2222,6 +2222,9 @@ typedef enum ObItemType
   T_DROP_PYTHON_UDF,  
   T_FUNCTION_ELEMENT_LIST,
   T_PARAM_DEFINITION,
+  T_PYTHON_CODE_TYPE,
+  T_PYTHON_CODE,
+  T_FILE_PATH,
 } ObItemType;
 
 typedef enum ObCacheType

src/share/schema/ob_python_udf.cpp

@@ -26,14 +26,14 @@ namespace schema
 {
 
 ObPythonUDF::ObPythonUDF(common::ObIAllocator *allocator)
-    : ObSchema(allocator), tenant_id_(common::OB_INVALID_ID), udf_id_(common::OB_INVALID_ID), name_(), arg_num_(0), arg_names_(), arg_types_(),
+    : ObSchema(allocator), tenant_id_(common::OB_INVALID_ID), udf_id_(common::OB_INVALID_ID), name_(), model_name_(), arg_num_(0), arg_names_(), arg_types_(),
       ret_(PyUdfRetType::UDF_UNINITIAL), pycall_(), schema_version_(common::OB_INVALID_VERSION)
 {
   reset();
 }
 
 ObPythonUDF::ObPythonUDF(const ObPythonUDF &src_schema)
-    : ObSchema(), tenant_id_(common::OB_INVALID_ID), udf_id_(common::OB_INVALID_ID), name_(), arg_num_(0), arg_names_(), arg_types_(),
+    : ObSchema(), tenant_id_(common::OB_INVALID_ID), udf_id_(common::OB_INVALID_ID), name_(), model_name_(), arg_num_(0), arg_names_(), arg_types_(),
       ret_(PyUdfRetType::UDF_UNINITIAL), pycall_(), schema_version_(common::OB_INVALID_VERSION)
 {
   reset();

src/share/schema/ob_python_udf.h

@@ -41,7 +41,7 @@ class ObPythonUDF : public ObSchema
     };
 
 public:
-    ObPythonUDF() : ObSchema(), tenant_id_(common::OB_INVALID_ID), udf_id_(common::OB_INVALID_ID), name_(), arg_num_(0), arg_names_(), 
+    ObPythonUDF() : ObSchema(), tenant_id_(common::OB_INVALID_ID), udf_id_(common::OB_INVALID_ID), name_(), model_name_(), arg_num_(0), arg_names_(), 
                     arg_types_(), ret_(PyUdfRetType::UDF_UNINITIAL), pycall_(), schema_version_(common::OB_INVALID_VERSION) 
                     { reset(); };
     explicit ObPythonUDF(common::ObIAllocator *allocator);
@@ -57,6 +57,7 @@ class ObPythonUDF : public ObSchema
     inline void set_tenant_id(const uint64_t id) { tenant_id_ = id; }
     inline void set_udf_id(const uint64_t id) { udf_id_ = id; }
     inline int set_name(const common::ObString &name) { return deep_copy_str(name, name_); }
+    inline int set_model_name(const common::ObString &model_name) { return deep_copy_str(model_name, model_name_); }
     inline void set_ret(const enum PyUdfRetType ret) { ret_ = ret; }
     inline void set_ret(const int ret) { ret_ = PyUdfRetType(ret); }
     inline void set_arg_num(const int arg_num) { arg_num_ = arg_num; }
@@ -70,6 +71,8 @@ class ObPythonUDF : public ObSchema
     inline uint64_t get_udf_id() const { return udf_id_; }
     inline const char *get_name() const { return extract_str(name_); }
     inline const common::ObString &get_name_str() const { return name_; }
+    inline const char *get_model_name() const { return extract_str(model_name_); }
+    inline const common::ObString &get_model_name_str() const { return model_name_; }
     inline int get_arg_num() const { return arg_num_; }
     const char *get_arg_names() const { return extract_str(arg_names_); }
     inline const common::ObString &get_arg_names_str() const { return arg_names_; }
@@ -93,6 +96,7 @@ class ObPythonUDF : public ObSchema
     TO_STRING_KV(K_(tenant_id),
                  K_(udf_id),
                  K_(name),
+                 K_(model_name),
                  K_(arg_num),
                  K_(arg_names),
                  K_(arg_types),
@@ -104,6 +108,7 @@ class ObPythonUDF : public ObSchema
     uint64_t tenant_id_;
     uint64_t udf_id_;
     common::ObString name_;
+    common::ObString model_name_;
     int arg_num_; //参数数量
     common::ObString arg_names_; //参数名称
     common::ObString arg_types_; //参数类型

src/sql/parser/non_reserved_keywords_mysql_mode.c

@@ -275,6 +275,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] =
   {"fields", FIELDS},
   {"file", FILEX},
   {"file_id", FILE_ID},
+  {"file_path", FILE_PATH},
   {"final_count", FINAL_COUNT},
   {"first", FIRST},
   {"first_value", FIRST_VALUE},
@@ -584,6 +585,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] =
   {"partitioning", PARTITIONING},
   {"partitions", PARTITIONS},
   {"password", PASSWORD},
+  {"path", PATH},
   {"pause", PAUSE},
   {"percentage", PERCENTAGE},
   {"percent_rank", PERCENT_RANK},
@@ -625,6 +627,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] =
   {"preceding", PRECEDING},
   {"pctfree", PCTFREE},
   {"ps", PS},
+  {"python_code", PYTHON_CODE},
   {"python_udf", PYTHON_UDF},
   {"quarter", QUARTER},
   {"query", QUERY},
@@ -865,6 +868,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] =
   {"trace", TRACE},
   {"traditional", TRADITIONAL},
   {"trailing", TRAILING},
+  {"train", TRAIN},
   {"transaction", TRANSACTION},
   {"trigger", TRIGGER},
   {"triggers", TRIGGERS},

src/sql/parser/sql_parser_mysql_mode.y

@@ -268,7 +268,7 @@ END_P SET_VAR DELIMITER
         ESCAPE EVENT EVENTS EVERY EXCHANGE EXECUTE EXPANSION EXPIRE EXPIRE_INFO EXPORT OUTLINE EXTENDED
         EXTENDED_NOADDR EXTENT_SIZE EXTRACT EXCEPT EXPIRED
 
-        FAILOVER FAST FAULTS FIELDS FILEX FINAL_COUNT FIRST FIRST_VALUE FIXED FLUSH FOLLOWER FORMAT
+        FAILOVER FAST FAULTS FIELDS FILEX FILE_PATH FINAL_COUNT FIRST FIRST_VALUE FIXED FLUSH FOLLOWER FORMAT
         FOUND FREEZE FREQUENCY FUNCTION FOLLOWING FLASHBACK FULL FRAGMENTATION FROZEN FILE_ID
 
         GENERAL GEOMETRY GEOMCOLLECTION GEOMETRYCOLLECTION GET_FORMAT GLOBAL GRANTS GROUP_CONCAT GROUPING GTS
@@ -304,11 +304,11 @@ END_P SET_VAR DELIMITER
         OBSOLETE OCCUR OF OFF OFFSET OLD OLD_PASSWORD ONE ONE_SHOT ONLY OPEN OPTIONS ORIG_DEFAULT OWNER OLD_KEY OVER
         OBCONFIG_URL OJ
 
-        PACK_KEYS PAGE PARALLEL PARAMETERS PARSER PARTIAL PARTITION_ID PARTITIONING PARTITIONS PASSWORD PAUSE PERCENTAGE
+        PACK_KEYS PAGE PARALLEL PARAMETERS PARSER PARTIAL PARTITION_ID PARTITIONING PARTITIONS PASSWORD PATH PAUSE PERCENTAGE
         PERCENT_RANK PHASE PLAN PHYSICAL PLANREGRESS PLUGIN PLUGIN_DIR PLUGINS POINT POLYGON PERFORMANCE
         PROTECTION PRIORITY PL POLICY POOL PORT POSITION PREPARE PRESERVE PRETTY PRETTY_COLOR PREV PRIMARY_ZONE PRIVILEGES PROCESS
         PROCESSLIST PROFILE PROFILES PROXY PRECEDING PCTFREE P_ENTITY P_CHUNK
-        PUBLIC PROGRESSIVE_MERGE_NUM PREVIEW PS PLUS PREDICT PYTHON_UDF
+        PUBLIC PROGRESSIVE_MERGE_NUM PREVIEW PS PLUS PREDICT PYTHON_CODE PYTHON_UDF 
 
         QUARTER QUERY QUERY_RESPONSE_TIME QUEUE_TIME QUICK
 
@@ -334,7 +334,7 @@ END_P SET_VAR DELIMITER
 
         TABLE_CHECKSUM TABLE_MODE TABLE_ID TABLE_NAME TABLEGROUPS TABLES TABLESPACE TABLET TABLET_ID TABLET_MAX_SIZE
         TEMPLATE TEMPORARY TEMPTABLE TENANT TEXT THAN TIME TIMESTAMP TIMESTAMPADD TIMESTAMPDIFF TP_NO
-        TP_NAME TRACE TRADITIONAL TRANSACTION TRIGGERS TRIM TRUNCATE TYPE TYPES TASK TABLET_SIZE
+        TP_NAME TRACE TRADITIONAL TRAIN TRANSACTION TRIGGERS TRIM TRUNCATE TYPE TYPES TASK TABLET_SIZE
         TABLEGROUP_ID TENANT_ID THROTTLE TIME_ZONE_INFO TOP_K_FRE_HIST TIMES
 
         UNCOMMITTED UNDEFINED UNDO_BUFFER_SIZE UNDOFILE UNICODE UNINSTALL UNIT UNIT_GROUP UNIT_NUM UNLOCKED UNTIL
@@ -498,7 +498,7 @@ END_P SET_VAR DELIMITER
 %type <node> recover_tenant_stmt recover_point_clause
 /*新增*/ 
 %type <node> create_python_udf_stmt drop_python_udf_stmt
-%type <node> function_element_list function_element param_name param_type
+%type <node> function_element_list function_element param_name param_type python_code_type
 %start sql_stmt
 %%
 ////////////////////////////////////////////////////////////////
@@ -2718,6 +2718,21 @@ MOD '(' expr ',' expr ')'
     store_pl_ref_object_symbol($$, result, REF_FUNC);
   }
 }
+| TRAIN function_name '(' opt_expr_as_list ')'
+{
+  if (NULL != $4)
+  {
+    ParseNode *params = NULL;
+    merge_nodes(params, result, T_EXPR_LIST, $4);
+    malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_PYTHON_UDF, 2, $2, params);
+    store_pl_ref_object_symbol($$, result, REF_FUNC);
+  }
+  else
+  {
+    malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_SYS_PYTHON_UDF, 1, $2);
+    store_pl_ref_object_symbol($$, result, REF_FUNC);
+  }
+}
 | sys_interval_func
 {
   $$ = $1;
@@ -4413,15 +4428,27 @@ NUMERIC
 ;
 
 create_python_udf_stmt:
-CREATE PYTHON_UDF NAME_OB '(' function_element_list ')' RETURNS ret_type '{' STRING_VALUE '}'
+CREATE PYTHON_UDF NAME_OB '(' function_element_list ')' RETURNS ret_type python_code_type
 {
   ParseNode *function_elements = NULL;
+  ParseNode *code_type_node = NULL;
   merge_nodes(function_elements, result, T_FUNCTION_ELEMENT_LIST, $5);
+  merge_nodes(code_type_node, result, T_PYTHON_CODE_TYPE, $9);
   malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_PYTHON_UDF, 4, 
                            $3,                             /* udf name */
                            function_elements,              /* function parameter */
                            $8,                             /* return type */
-                           $10);                           /* python code */
+                           code_type_node);                /* python code type */
+}
+|
+CREATE PYTHON_UDF NAME_OB MODEL NAME_OB python_code_type
+{
+  ParseNode *code_type_node = NULL;
+  merge_nodes(code_type_node, result, T_PYTHON_CODE_TYPE, $6);
+  malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_PYTHON_UDF, 3, 
+                           $3,                             /* udf name */
+                           $5,                             /* model name */
+                           code_type_node);               /* python code type */
 }
 ;
 
@@ -4491,6 +4518,18 @@ NUMERIC
 }
 ;
 
+python_code_type:
+PYTHON_CODE '{' STRING_VALUE '}'
+{
+  malloc_non_terminal_node($$, result->malloc_pool_, T_PYTHON_CODE, 1, $3);
+}
+|
+FILE_PATH STRING_VALUE
+{
+  malloc_non_terminal_node($$, result->malloc_pool_, T_FILE_PATH, 1, $2);
+}
+;
+
 create_function_stmt:
 create_with_opt_hint opt_agg FUNCTION NAME_OB RETURNS ret_type SONAME STRING_VALUE
 {