diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java index 4df1d81ded..35ae1f6935 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite; +import java.util.ArrayList; +import java.util.List; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder.AggCall; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -33,6 +35,10 @@ public AggCall visitAlias(Alias node, CalcitePlanContext context) { @Override public AggCall visitAggregateFunction(AggregateFunction node, CalcitePlanContext context) { RexNode field = rexNodeVisitor.analyze(node.getField(), context); - return AggregateUtils.translate(node, field, context); + List argList = new ArrayList<>(); + for (UnresolvedExpression arg : node.getArgList()) { + argList.add(rexNodeVisitor.analyze(arg, context)); + } + return AggregateUtils.translate(node, field, context, argList); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index fd67d08bf1..9d9176a140 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.ast.expression.SpanUnit.NONE; import static org.opensearch.sql.ast.expression.SpanUnit.UNKNOWN; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.translateArgument; import java.math.BigDecimal; import java.util.List; @@ -240,6 +241,7 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) { List arguments = node.getFuncArgs().stream().map(arg -> analyze(arg, context)).collect(Collectors.toList()); return context.rexBuilder.makeCall( - BuiltinFunctionUtils.translate(node.getFuncName()), arguments); + BuiltinFunctionUtils.translate(node.getFuncName()), + translateArgument(node.getFuncName(), arguments, context)); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/Accumulator.java b/core/src/main/java/org/opensearch/sql/calcite/udf/Accumulator.java new file mode 100644 index 0000000000..56080cc52b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/Accumulator.java @@ -0,0 +1,10 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf; + +public interface Accumulator { + Object result(); +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/UserDefinedAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/UserDefinedAggFunction.java new file mode 100644 index 0000000000..2488b3ed70 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/UserDefinedAggFunction.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf; + +public interface UserDefinedAggFunction { + S init(); + + Object result(S accumulator); + + // Add values to the accumulator + S add(S acc, Object... values); +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/UserDefinedFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/UserDefinedFunction.java new file mode 100644 index 0000000000..20908943c4 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/UserDefinedFunction.java @@ -0,0 +1,10 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf; + +public interface UserDefinedFunction { + Object eval(Object... args); +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/IfFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/IfFunction.java new file mode 100644 index 0000000000..f9983d5fd7 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/IfFunction.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.conditionUDF; + +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class IfFunction implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + Object condition = args[0]; + Object trueValue = args[1]; + Object falseValue = args[2]; + if (condition instanceof Boolean) { + return (Boolean) condition ? trueValue : falseValue; + } + return trueValue; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/IfNullFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/IfNullFunction.java new file mode 100644 index 0000000000..0da04699ac --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/IfNullFunction.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.conditionUDF; + +import java.util.Objects; +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class IfNullFunction implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + Object conditionValue = args[0]; + Object defaultValue = args[1]; + if (Objects.isNull(conditionValue)) { + return defaultValue; + } + return conditionValue; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/NullIfFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/NullIfFunction.java new file mode 100644 index 0000000000..38fde7dc43 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/conditionUDF/NullIfFunction.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.conditionUDF; + +import java.util.Objects; +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class NullIfFunction implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + Object firstValue = args[0]; + Object secondValue = args[1]; + if (Objects.equals(firstValue, secondValue)) { + return null; + } + return firstValue; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/CRC32Function.java b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/CRC32Function.java new file mode 100644 index 0000000000..603c2de9b8 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/CRC32Function.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.mathUDF; + +import java.util.zip.CRC32; +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class CRC32Function implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + Object value = args[0]; + CRC32 crc = new CRC32(); + crc.update(value.toString().getBytes()); + return crc.getValue(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/EulerFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/EulerFunction.java new file mode 100644 index 0000000000..a33cc4cc04 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/EulerFunction.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.mathUDF; + +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class EulerFunction implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + return Math.E; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/ModFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/ModFunction.java new file mode 100644 index 0000000000..532a7f9eaa --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/ModFunction.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.mathUDF; + +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class ModFunction implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + if (args.length < 2) { + throw new IllegalArgumentException("At least two arguments are required"); + } + + // Get the two values + Object mod0 = args[0]; + Object mod1 = args[1]; + + // Handle numbers dynamically + if (mod0 instanceof Integer && mod1 instanceof Integer) { + return (Integer) mod0 % (Integer) mod1; + } else if (mod0 instanceof Number && mod1 instanceof Number) { + double num0 = ((Number) mod0).doubleValue(); + double num1 = ((Number) mod1).doubleValue(); + + if (num1 == 0) { + throw new ArithmeticException("Modulo by zero is not allowed"); + } + + return num0 % num1; // Handles both float and double cases + } else { + throw new IllegalArgumentException("Invalid argument types: Expected numeric values"); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/SqrtFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/SqrtFunction.java new file mode 100644 index 0000000000..e67fcb906d --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/mathUDF/SqrtFunction.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.mathUDF; + +import static java.lang.Math.sqrt; + +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class SqrtFunction implements UserDefinedFunction { + @Override + public Object eval(Object... args) { + if (args.length < 1) { + throw new IllegalArgumentException("At least one argument is required"); + } + + // Get the input value + Object input = args[0]; + + // Handle numbers dynamically + if (input instanceof Number) { + double num = ((Number) input).doubleValue(); + + if (num < 0) { + throw new ArithmeticException("Cannot compute square root of a negative number"); + } + + return sqrt(num); // Computes sqrt using Math.sqrt() + } else { + throw new IllegalArgumentException("Invalid argument type: Expected a numeric value"); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java new file mode 100644 index 0000000000..cc3b2a9a95 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java @@ -0,0 +1,57 @@ +package org.opensearch.sql.calcite.udf.udaf; + +import com.tdunning.math.stats.AVLTreeDigest; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import org.opensearch.sql.calcite.udf.Accumulator; +import org.opensearch.sql.calcite.udf.UserDefinedAggFunction; + +public class PercentileApproxFunction + implements UserDefinedAggFunction { + @Override + public PencentileApproAccumulator init() { + return new PencentileApproAccumulator(); + } + + // Add values to the accumulator + @Override + public PencentileApproAccumulator add(PencentileApproAccumulator acc, Object... values) { + List allValues = Arrays.asList(values); + Object targetValue = allValues.get(0); + if (Objects.isNull(targetValue)) { + return acc; + } + Number percentileValue = (Number) allValues.get(1); + acc.evaluate(((Number) targetValue).doubleValue(), percentileValue.intValue()); + return acc; + } + + // Calculate the percentile + @Override + public Object result(PencentileApproAccumulator acc) { + if (acc.size() == 0) { + return null; + } + return acc.result(); + } + + public static class PencentileApproAccumulator extends AVLTreeDigest implements Accumulator { + public static final double DEFAULT_COMPRESSION = 100.0; + private double percent; + + public PencentileApproAccumulator() { + super(DEFAULT_COMPRESSION); + this.percent = 1.0; + } + + public void evaluate(double value, int percent) { + this.percent = percent / 100.0; + this.add(value); + } + + public Object result() { + return this.quantile(this.percent); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/TakeAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/TakeAggFunction.java new file mode 100644 index 0000000000..d44376663a --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/TakeAggFunction.java @@ -0,0 +1,55 @@ +package org.opensearch.sql.calcite.udf.udaf; + +import java.util.ArrayList; +import java.util.List; +import org.opensearch.sql.calcite.udf.Accumulator; +import org.opensearch.sql.calcite.udf.UserDefinedAggFunction; + +public class TakeAggFunction implements UserDefinedAggFunction { + + @Override + public TakeAccumulator init() { + return new TakeAccumulator(); + } + + @Override + public Object result(TakeAccumulator accumulator) { + return accumulator.result(); + } + + @Override + public TakeAccumulator add(TakeAccumulator acc, Object... values) { + Object candidateValue = values[0]; + int size = 0; + if (values.length > 1) { + size = (int) values[1]; + } else { + size = 10; + } + if (size > acc.size()) { + acc.add(candidateValue); + } + return acc; + } + + public static class TakeAccumulator implements Accumulator { + private List hits; + + public TakeAccumulator() { + hits = new ArrayList<>(); + } + + @Override + public Object result() { + return hits; + } + + public void add(Object value) { + hits.add(value); + } + + public int size() { + return hits.size(); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/AggregateUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/AggregateUtils.java index 2bba67efb9..993bcfd25b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/AggregateUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/AggregateUtils.java @@ -5,7 +5,10 @@ package org.opensearch.sql.calcite.utils; +import static org.opensearch.sql.calcite.utils.UserDefineFunctionUtils.TransferUserDefinedAggFunction; + import com.google.common.collect.ImmutableList; +import java.util.List; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rex.RexInputRef; @@ -15,12 +18,14 @@ import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.udf.udaf.PercentileApproxFunction; +import org.opensearch.sql.calcite.udf.udaf.TakeAggFunction; import org.opensearch.sql.expression.function.BuiltinFunctionName; public interface AggregateUtils { static RelBuilder.AggCall translate( - AggregateFunction agg, RexNode field, CalcitePlanContext context) { + AggregateFunction agg, RexNode field, CalcitePlanContext context, List argList) { if (BuiltinFunctionName.ofAggregation(agg.getFuncName()).isEmpty()) throw new IllegalStateException("Unexpected value: " + agg.getFuncName()); @@ -51,10 +56,21 @@ static RelBuilder.AggCall translate( // return // context.relBuilder.aggregateCall(SqlStdOperatorTable.PERCENTILE_CONT, field); case PERCENTILE_APPROX: - throw new UnsupportedOperationException("PERCENTILE_APPROX is not supported in PPL"); - // case APPROX_COUNT_DISTINCT: - // return - // context.relBuilder.aggregateCall(SqlStdOperatorTable.APPROX_COUNT_DISTINCT, field); + return TransferUserDefinedAggFunction( + PercentileApproxFunction.class, + "percentile_approx", + UserDefineFunctionUtils.getReturnTypeInference(0), + List.of(field), + argList, + context.relBuilder); + case TAKE: + return TransferUserDefinedAggFunction( + TakeAggFunction.class, + "take", + UserDefineFunctionUtils.getReturnTypeInferenceForArray(), + List.of(field), + argList, + context.relBuilder); } throw new IllegalStateException("Not Supported value: " + agg.getFuncName()); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java index fba0354bc0..b3aece8603 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java @@ -5,10 +5,29 @@ package org.opensearch.sql.calcite.utils; +import static java.lang.Math.E; +import static org.opensearch.sql.calcite.utils.UserDefineFunctionUtils.TransferUserDefinedFunction; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.fun.SqlTrimFunction; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.udf.conditionUDF.IfFunction; +import org.opensearch.sql.calcite.udf.conditionUDF.IfNullFunction; +import org.opensearch.sql.calcite.udf.conditionUDF.NullIfFunction; +import org.opensearch.sql.calcite.udf.mathUDF.CRC32Function; +import org.opensearch.sql.calcite.udf.mathUDF.EulerFunction; +import org.opensearch.sql.calcite.udf.mathUDF.ModFunction; +import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction; public interface BuiltinFunctionUtils { @@ -44,13 +63,98 @@ static SqlOperator translate(String op) { case "/": return SqlStdOperatorTable.DIVIDE; // Built-in String Functions + case "CONCAT": + return SqlLibraryOperators.CONCAT_FUNCTION; + case "CONCAT_WS": + return SqlLibraryOperators.CONCAT_WS; + case "LIKE": + return SqlLibraryOperators.ILIKE; + case "LTRIM", "RTRIM", "TRIM": + return SqlStdOperatorTable.TRIM; + case "LENGTH": + return SqlStdOperatorTable.CHAR_LENGTH; case "LOWER": return SqlStdOperatorTable.LOWER; - case "LIKE": - return SqlStdOperatorTable.LIKE; - // Built-in Math Functions + case "POSITION": + return SqlStdOperatorTable.POSITION; + case "REVERSE": + return SqlLibraryOperators.REVERSE; + case "RIGHT": + return SqlLibraryOperators.RIGHT; + case "SUBSTRING": + return SqlStdOperatorTable.SUBSTRING; + case "UPPER": + return SqlStdOperatorTable.UPPER; + // Built-in condition Functions + // case "IFNULL": + // return SqlLibraryOperators.IFNULL; + case "IF": + return TransferUserDefinedFunction( + IfFunction.class, "if", UserDefineFunctionUtils.getReturnTypeInference(1)); + case "IFNULL": + return TransferUserDefinedFunction( + IfNullFunction.class, "ifnull", UserDefineFunctionUtils.getReturnTypeInference(1)); + case "NULLIF": + return TransferUserDefinedFunction( + NullIfFunction.class, "ifnull", UserDefineFunctionUtils.getReturnTypeInference(0)); + case "IS NOT NULL": + return SqlStdOperatorTable.IS_NOT_NULL; + case "IS NULL": + return SqlStdOperatorTable.IS_NULL; case "ABS": return SqlStdOperatorTable.ABS; + case "ACOS": + return SqlStdOperatorTable.ACOS; + case "ASIN": + return SqlStdOperatorTable.ASIN; + case "ATAN", "ATAN2": + return SqlStdOperatorTable.ATAN2; + case "CEILING": + return SqlStdOperatorTable.CEIL; + case "CONV": + return SqlStdOperatorTable.CONVERT; + case "COS": + return SqlStdOperatorTable.COS; + case "COT": + return SqlStdOperatorTable.COT; + case "CRC32": + return TransferUserDefinedFunction(CRC32Function.class, "crc32", ReturnTypes.BIGINT); + case "DEGREES": + return SqlStdOperatorTable.DEGREES; + case "E": + return TransferUserDefinedFunction(EulerFunction.class, "e", ReturnTypes.DOUBLE); + case "EXP": + return SqlStdOperatorTable.EXP; + case "FLOOR": + return SqlStdOperatorTable.FLOOR; + case "LN": + return SqlStdOperatorTable.LN; + case "LOG": + return SqlLibraryOperators.LOG; + case "LOG2": + return SqlLibraryOperators.LOG2; + case "LOG10": + return SqlStdOperatorTable.LOG10; + case "MOD": + return TransferUserDefinedFunction(ModFunction.class, "mod", ReturnTypes.DOUBLE); + case "PI": + return SqlStdOperatorTable.PI; + case "POW", "POWER": + return SqlStdOperatorTable.POWER; + case "RADIANS": + return SqlStdOperatorTable.RADIANS; + case "RAND": + return SqlStdOperatorTable.RAND; + case "ROUND": + return SqlStdOperatorTable.ROUND; + case "SIGN": + return SqlStdOperatorTable.SIGN; + case "SIN": + return SqlStdOperatorTable.SIN; + case "SQRT": + return TransferUserDefinedFunction(SqrtFunction.class, "sqrt", ReturnTypes.DOUBLE); + case "CBRT": + return SqlStdOperatorTable.CBRT; // Built-in Date Functions case "CURRENT_TIMESTAMP": return SqlStdOperatorTable.CURRENT_TIMESTAMP; @@ -63,8 +167,68 @@ static SqlOperator translate(String op) { case "DATE_ADD": return SqlLibraryOperators.DATEADD; // TODO Add more, ref RexImpTable + case "DATE_SUB": + return SqlLibraryOperators.DATE_SUB; + case "HOUR": + return SqlStdOperatorTable.HOUR; + case "MINUTE": + return SqlStdOperatorTable.MINUTE; default: throw new IllegalArgumentException("Unsupported operator: " + op); } } + + static List translateArgument( + String op, List argList, CalcitePlanContext context) { + switch (op.toUpperCase(Locale.ROOT)) { + case "TRIM": + List trimArgs = + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), + context.rexBuilder.makeLiteral(" "))); + trimArgs.addAll(argList); + return trimArgs; + case "LTRIM": + List LTrimArgs = + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.LEADING), + context.rexBuilder.makeLiteral(" "))); + LTrimArgs.addAll(argList); + return LTrimArgs; + case "RTRIM": + List RTrimArgs = + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.TRAILING), + context.rexBuilder.makeLiteral(" "))); + RTrimArgs.addAll(argList); + return RTrimArgs; + case "ATAN": + List AtanArgs = new ArrayList<>(argList); + if (AtanArgs.size() == 1) { + BigDecimal divideNumber = BigDecimal.valueOf(1); + AtanArgs.add(context.rexBuilder.makeBigintLiteral(divideNumber)); + } + return AtanArgs; + case "LOG": + List LogArgs = new ArrayList<>(); + RelDataTypeFactory typeFactory = context.rexBuilder.getTypeFactory(); + if (argList.size() == 1) { + LogArgs.add(argList.getFirst()); + LogArgs.add( + context.rexBuilder.makeExactLiteral( + BigDecimal.valueOf(E), typeFactory.createSqlType(SqlTypeName.DOUBLE))); + } else if (argList.size() == 2) { + LogArgs.add(argList.get(1)); + LogArgs.add(argList.get(0)); + } else { + throw new IllegalArgumentException("Log cannot accept argument list: " + argList); + } + return LogArgs; + default: + return argList; + } + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 4a992c885a..372dbc6335 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -140,7 +140,7 @@ public static ExprType convertRelDataTypeToExprType(RelDataType type) { return FLOAT; case DOUBLE: return DOUBLE; - case VARCHAR: + case VARCHAR, CHAR: return STRING; case BOOLEAN: return BOOLEAN; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefineFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefineFunctionUtils.java new file mode 100644 index 0000000000..4c44549221 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefineFunctionUtils.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.ScalarFunction; +import org.apache.calcite.schema.impl.AggregateFunctionImpl; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; +import org.apache.calcite.sql.validate.SqlUserDefinedFunction; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.Optionality; +import org.opensearch.sql.calcite.udf.UserDefinedAggFunction; +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class UserDefineFunctionUtils { + public static RelBuilder.AggCall TransferUserDefinedAggFunction( + Class UDAF, + String functionName, + SqlReturnTypeInference returnType, + List fields, + List argList, + RelBuilder relBuilder) { + SqlUserDefinedAggFunction sqlUDAF = + new SqlUserDefinedAggFunction( + new SqlIdentifier(functionName, SqlParserPos.ZERO), + SqlKind.OTHER_FUNCTION, + returnType, + null, + null, + AggregateFunctionImpl.create(UDAF), + false, + false, + Optionality.FORBIDDEN); + List addArgList = new ArrayList<>(fields); + addArgList.addAll(argList); + return relBuilder.aggregateCall(sqlUDAF, addArgList); + } + + public static SqlOperator TransferUserDefinedFunction( + Class UDF, + String functionName, + SqlReturnTypeInference returnType) { + final ScalarFunction udfFunction = + ScalarFunctionImpl.create(Types.lookupMethod(UDF, "eval", Object[].class)); + SqlIdentifier udfLtrimIdentifier = + new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); + return new SqlUserDefinedFunction( + udfLtrimIdentifier, SqlKind.OTHER_FUNCTION, returnType, null, null, udfFunction); + } + + public static SqlReturnTypeInference getReturnTypeInference(int targetPosition) { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + + // Get argument types + List argTypes = opBinding.collectOperandTypes(); + + if (argTypes.isEmpty()) { + throw new IllegalArgumentException("Function requires at least one argument."); + } + RelDataType firstArgType = argTypes.get(targetPosition); + return typeFactory.createSqlType(firstArgType.getSqlTypeName()); + }; + } + + public static SqlReturnTypeInference getReturnTypeInferenceForArray() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + + // Get argument types + List argTypes = opBinding.collectOperandTypes(); + + if (argTypes.isEmpty()) { + throw new IllegalArgumentException("Function requires at least one argument."); + } + RelDataType firstArgType = argTypes.getFirst(); + return createArrayType(typeFactory, firstArgType, true); + }; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java index 8bcd801c12..182703607f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java @@ -7,9 +7,9 @@ import java.io.IOException; import org.junit.Ignore; +import org.junit.Test; import org.opensearch.sql.ppl.LikeQueryIT; -@Ignore("CalciteLikeQueryIT is not supported in OpenSearch yet") public class CalciteLikeQueryIT extends LikeQueryIT { @Override public void init() throws IOException { @@ -17,4 +17,53 @@ public void init() throws IOException { disallowCalciteFallback(); super.init(); } + + @Override + @Test + @Ignore("* in like is handled wrong") + public void test_like_with_escaped_percent() throws IOException { + super.test_like_with_escaped_percent(); + } + + @Override + @Test + @Ignore("* in like is handled wrong") + public void test_like_in_where_with_escaped_underscore() throws IOException { + super.test_like_in_where_with_escaped_underscore(); + } + + @Override + @Test + @Ignore("* in like is handled wrong") + public void test_like_on_text_field_with_one_word() throws IOException { + super.test_like_on_text_field_with_one_word(); + } + + @Override + @Test + @Ignore("* in like is handled wrong") + public void test_like_on_text_keyword_field_with_one_word() throws IOException { + super.test_like_on_text_keyword_field_with_one_word(); + } + + @Override + @Test + @Ignore("* in like is handled wrong") + public void test_like_on_text_keyword_field_with_greater_than_one_word() throws IOException { + super.test_like_on_text_keyword_field_with_greater_than_one_word(); + } + + @Override + @Test + @Ignore("* in like is handled wrong") + public void test_like_on_text_field_with_greater_than_one_word() throws IOException { + super.test_like_on_text_field_with_greater_than_one_word(); + } + + @Override + @Test + @Ignore("ignore this class since IP type is unsupported in calcite engine") + public void test_convert_field_text_to_keyword() throws IOException { + super.test_convert_field_text_to_keyword(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java index 32723154ae..473f8c8461 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.util.MatcherUtils.*; + import java.io.IOException; import org.junit.Ignore; import org.opensearch.sql.ppl.StatsCommandIT; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java index ca1250b2cf..d1ab0ebf32 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBasicIT.java @@ -24,6 +24,20 @@ public void init() throws IOException { request2.setJsonEntity("{\"name\": \"world\", \"age\": 30}"); client().performRequest(request2); + Request request3 = new Request("PUT", "/test_name_null/_doc/1?refresh=true"); + request3.setJsonEntity("{\"name\": \"hello\", \"age\": 20}"); + client().performRequest(request3); + Request request4 = new Request("PUT", "/test_name_null/_doc/2?refresh=true"); + request4.setJsonEntity("{\"name\": \"world\", \"age\": 30}"); + client().performRequest(request4); + Request request5 = new Request("PUT", "/test_name_null/_doc/3?refresh=true"); + request5.setJsonEntity("{\"name\": null, \"age\": 30}"); + client().performRequest(request5); + + Request request6 = new Request("PUT", "/people/_doc/2?refresh=true"); + request6.setJsonEntity("{\"name\": \"DummyEntityForMathVerification\", \"age\": 24}"); + client().performRequest(request6); + loadIndex(Index.BANK); } @@ -35,6 +49,28 @@ public void testInvalidTable() { () -> execute("source=unknown")); } + @Ignore + public void testTakeAggregation() { + String actual = execute("source=test | stats take(name, 2)"); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"take(name, 2)\",\n" + + " \"type\": \"array\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " [\"hello\", \"world\"]\n" + + " ]\n" + + " ],\n" + + " \"total\": 1,\n" + + " \"size\": 1\n" + + "}", + actual); + } + @Test public void testSourceQuery() { String actual = execute("source=test"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java new file mode 100644 index 0000000000..a772ccd971 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java @@ -0,0 +1,555 @@ +package org.opensearch.sql.calcite.standalone; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.junit.Ignore; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; + +public class CalcitePPLBuiltinFunctionIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + Request request3 = new Request("PUT", "/test_name_null/_doc/1?refresh=true"); + request3.setJsonEntity("{\"name\": \"hello\", \"age\": 20}"); + client().performRequest(request3); + Request request4 = new Request("PUT", "/test_name_null/_doc/2?refresh=true"); + request4.setJsonEntity("{\"name\": \"world\", \"age\": 30}"); + client().performRequest(request4); + Request request5 = new Request("PUT", "/test_name_null/_doc/3?refresh=true"); + request5.setJsonEntity("{\"name\": null, \"age\": 30}"); + client().performRequest(request5); + + Request request6 = new Request("PUT", "/people/_doc/2?refresh=true"); + request6.setJsonEntity("{\"name\": \"DummyEntityForMathVerification\", \"age\": 24}"); + client().performRequest(request6); + loadIndex(Index.BANK); + } + + @Ignore + @Test + public void testDate() { + String query = + "source=test |eval `DATE('2020-08-26')` = DATE('2020-08-26') | fields `DATE('2020-08-26')`"; + testSimplePPL(query, List.of("2020-08-26 01:00:00", "2020-08-27 01:01:01")); + } + + @Ignore + @Test + public void testDateAdd() { + String query = + "source=test | eval `'2020-08-26' + 1h` = DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR)," + + " `ts '2020-08-26 01:01:01' + 1d` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01')," + + " INTERVAL 1 DAY) | fields `'2020-08-26' + 1h`, `ts '2020-08-26 01:01:01' + 1d`"; + testSimplePPL(query, List.of("2020-08-26 01:00:00", "2020-08-27 01:01:01")); + } + + @Test + public void testConcat() { + String query = + "source=test | eval `CONCAT('hello', 'world')` = CONCAT('hello', 'world')," + + " `CONCAT('hello ', 'whole ', 'world', '!')` = CONCAT('a', 'b ', 'c', 'd', 'e'," + + " 'f', 'g', '1', '2') | fields `CONCAT('hello', 'world')`, `CONCAT('hello '," + + " 'whole ', 'world', '!')`"; + + testSimplePPL(query, List.of("helloworld", "ab cdefg12")); + } + + @Test + public void testConcatWs() { + String query = + "source=test | eval `CONCAT_WS(',', 'hello', 'world')` = CONCAT_WS(',', 'hello'," + + " 'world') | fields `CONCAT_WS(',', 'hello', 'world')`"; + testSimplePPL(query, List.of("hello,world")); + } + + @Test + public void testLength() { + String query = + "source=test | eval `LENGTH('helloworld')` = LENGTH('helloworld') | fields" + + " `LENGTH('helloworld')`"; + testSimplePPL(query, List.of(10)); + } + + @Test + public void testLower() { + String query = + "source=test | eval `LOWER('helloworld')` = LOWER('helloworld'), `LOWER('HELLOWORLD')`" + + " = LOWER('HELLOWORLD') | fields `LOWER('helloworld')`, `LOWER('HELLOWORLD')`"; + testSimplePPL(query, List.of("helloworld", "helloworld")); + } + + @Test + public void testLtrim() { + String query = + "source=test | eval `LTRIM(' hello')` = LTRIM(' hello'), `LTRIM('hello ')` =" + + " LTRIM('hello ') | fields `LTRIM(' hello')`, `LTRIM('hello ')`"; + testSimplePPL(query, List.of("hello", "hello ")); + } + + @Test + public void testPosition() { + String query = + "source=test | eval `POSITION('world' IN 'helloworld')` = POSITION('world' IN" + + " 'helloworld'), `POSITION('invalid' IN 'helloworld')`= POSITION('invalid' IN" + + " 'helloworld') | fields `POSITION('world' IN 'helloworld')`, `POSITION('invalid' IN" + + " 'helloworld')`"; + testSimplePPL(query, List.of(6, 0)); + } + + @Test + public void testReverse() { + String query = + "source=test | eval `REVERSE('abcde')` = REVERSE('abcde') | fields `REVERSE('abcde')`"; + testSimplePPL(query, List.of("edcba")); + } + + // @Ignore + @Test + public void testRight() { + List expected = new ArrayList<>(); + expected.add("world"); + expected.add(""); + String query = + "source=test | eval `RIGHT('helloworld', 5)` = RIGHT('helloworld', 5), `RIGHT('HELLOWORLD'," + + " 0)` = RIGHT('HELLOWORLD', 0) | fields `RIGHT('helloworld', 5)`," + + " `RIGHT('HELLOWORLD', 0)`"; + testSimplePPL(query, expected); + } + + @Test + public void testLike() { + String query = + "source=" + + TEST_INDEX_WILDCARD + + " | WHERE Like(KeywordBody, '\\\\_test wildcard%') | fields KeywordBody"; + } + + @Test + public void testRtrim() { + String query = + "source=test | eval `RTRIM(' hello')` = RTRIM(' hello'), `RTRIM('hello ')` =" + + " RTRIM('hello ') | fields `RTRIM(' hello')`, `RTRIM('hello ')`"; + testSimplePPL(query, List.of(" hello", "hello")); + } + + @Test + public void testSubstring() { + String query = + "source=test | eval `SUBSTRING('helloworld', 5)` = SUBSTRING('helloworld', 5)," + + " `SUBSTRING('helloworld', 5, 3)` = SUBSTRING('helloworld', 5, 3) | fields" + + " `SUBSTRING('helloworld', 5)`, `SUBSTRING('helloworld', 5, 3)`"; + testSimplePPL(query, List.of("oworld", "owo")); + } + + @Test + public void testTrim() { + String query = + "source=test | eval `TRIM(' hello')` = TRIM(' hello'), `TRIM('hello ')` = TRIM('hello" + + " ') | fields `TRIM(' hello')`, `TRIM('hello ')`"; + testSimplePPL(query, List.of("hello", "hello")); + } + + @Test + public void testUpper() { + String query = + "source=test | eval `UPPER('helloworld')` = UPPER('helloworld'), `UPPER('HELLOWORLD')` =" + + " UPPER('HELLOWORLD') | fields `UPPER('helloworld')`, `UPPER('HELLOWORLD')`"; + testSimplePPL(query, List.of("HELLOWORLD", "HELLOWORLD")); + } + + @Test + public void testIf() { + String actual = + execute( + "source=test_name_null | eval result = if(like(name, '%he%'), 'default', name) | fields" + + " result"); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"result\",\n" + + " \"type\": \"string\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"default\"\n" + + " ],\n" + + " [\n" + + " \"default\"\n" + + " ],\n" + + " [\n" + + " \"default\"\n" + + " ]\n" + + " ],\n" + + " \"total\": 3,\n" + + " \"size\": 3\n" + + "}", + actual); + } + + @Test + public void testIfNull() { + String actual = + execute( + "source=test_name_null | eval defaultName=ifnull(name, 'default') | fields" + + " defaultName"); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"defaultName\",\n" + + " \"type\": \"string\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"hello\"\n" + + " ],\n" + + " [\n" + + " \"world\"\n" + + " ],\n" + + " [\n" + + " \"default\"\n" + + " ]\n" + + " ],\n" + + " \"total\": 3,\n" + + " \"size\": 3\n" + + "}", + actual); + } + + @Test + public void testNullIf() { + String actual = + execute( + "source=test_name_null | eval defaultName=nullif(name, 'world') | fields defaultName"); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"defaultName\",\n" + + " \"type\": \"string\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"hello\"\n" + + " ],\n" + + " [\n" + + " null\n" + + " ],\n" + + " [\n" + + " null\n" + + " ]\n" + + " ],\n" + + " \"total\": 3,\n" + + " \"size\": 3\n" + + "}", + actual); + } + + private static JsonArray parseAndGetFirstDataRow(String executionResult) { + JsonObject sqrtResJson = JsonParser.parseString(executionResult).getAsJsonObject(); + JsonArray dataRows = sqrtResJson.getAsJsonArray("datarows"); + return dataRows.get(0).getAsJsonArray(); + } + + private void testSimplePPL(String query, List expectedValues) { + String execResult = execute(query); + JsonArray dataRow = parseAndGetFirstDataRow(execResult); + assertEquals(expectedValues.size(), dataRow.size()); + for (int i = 0; i < expectedValues.size(); i++) { + Object expected = expectedValues.get(i); + if (Objects.isNull(expected)) { + Object actual = dataRow.get(i); + assertNull(actual); + } else if (expected instanceof BigDecimal) { + Number actual = dataRow.get(i).getAsNumber(); + assertEquals(expected, actual); + } else if (expected instanceof Double || expected instanceof Float) { + Number actual = dataRow.get(i).getAsNumber(); + assertDoubleUlpEquals(((Number) expected).doubleValue(), actual.doubleValue(), 8); + } else if (expected instanceof Long || expected instanceof Integer) { + Number actual = dataRow.get(i).getAsNumber(); + assertEquals(((Number) expected).longValue(), actual.longValue()); + } else if (expected instanceof String) { + String actual = dataRow.get(i).getAsString(); + assertEquals(expected, actual); + } else if (expected instanceof Boolean) { + Boolean actual = dataRow.get(i).getAsBoolean(); + assertEquals(expected, actual); + } else { + fail("Unsupported number type: " + expected.getClass().getName()); + } + } + } + + @Test + public void testAbs() { + String absPpl = "source=people | eval `ABS(-1)` = ABS(-1) | fields `ABS(-1)`"; + List expected = List.of(1); + testSimplePPL(absPpl, expected); + } + + @Test + public void testAcos() { + String acosPpl = "source=people | eval `ACOS(0)` = ACOS(0) | fields `ACOS(0)`"; + List expected = List.of(Math.PI / 2); + testSimplePPL(acosPpl, expected); + } + + @Test + public void testAsin() { + String asinPpl = "source=people | eval `ASIN(0)` = ASIN(0) | fields `ASIN(0)`"; + List expected = List.of(0.0); + testSimplePPL(asinPpl, expected); + } + + @Test + public void testAtan() { + // TODO: Error while preparing plan [LogicalProject(ATAN(2)=[ATAN(2)], ATAN(2, 3)=[ATAN(2, 3)]) + // ATAN defined in OpenSearch accepts single and double arguments, while that defined in SQL + // standard library accepts only single argument. + testSimplePPL( + "source=people | eval `ATAN(2)` = ATAN(2), `ATAN(2, 3)` = ATAN(2, 3) | fields `ATAN(2)`," + + " `ATAN(2, 3)`", + List.of(Math.atan(2), Math.atan2(2, 3))); + } + + @Test + public void testAtan2() { + testSimplePPL( + "source=people | eval `ATAN2(2, 3)` = ATAN2(2, 3) | fields `ATAN2(2, 3)`", + List.of(Math.atan2(2, 3))); + } + + @Test + public void testCeiling() { + testSimplePPL( + "source=people | eval `CEILING(0)` = CEILING(0), `CEILING(50.00005)` = CEILING(50.00005)," + + " `CEILING(-50.00005)` = CEILING(-50.00005) | fields `CEILING(0)`," + + " `CEILING(50.00005)`, `CEILING(-50.00005)`", + List.of(Math.ceil(0.0), Math.ceil(50.00005), Math.ceil(-50.00005))); + testSimplePPL( + "source=people | eval `CEILING(3147483647.12345)` = CEILING(3147483647.12345)," + + " `CEILING(113147483647.12345)` = CEILING(113147483647.12345)," + + " `CEILING(3147483647.00001)` = CEILING(3147483647.00001) | fields" + + " `CEILING(3147483647.12345)`, `CEILING(113147483647.12345)`," + + " `CEILING(3147483647.00001)`", + List.of( + Math.ceil(3147483647.12345), + Math.ceil(113147483647.12345), + Math.ceil(3147483647.00001))); + } + + @Ignore + @Test + public void testConv() { + // TODO: Error while preparing plan [LogicalProject(CONV('12', 10, 16)=[CONVERT('12', 10, 16)], + // CONV('2C', 16, 10)=[CONVERT('2C', 16, 10)], CONV(12, 10, 2)=[CONVERT(12, 10, 2)], CONV(1111, + // 2, 10)=[CONVERT(1111, 2, 10)]) + // OpenSearchTableScan(table=[[OpenSearch, people]]) + String convPpl = + "source=people | eval `CONV('12', 10, 16)` = CONV('12', 10, 16), `CONV('2C', 16, 10)` =" + + " CONV('2C', 16, 10), `CONV(12, 10, 2)` = CONV(12, 10, 2), `CONV(1111, 2, 10)` =" + + " CONV(1111, 2, 10) | fields `CONV('12', 10, 16)`, `CONV('2C', 16, 10)`, `CONV(12," + + " 10, 2)`, `CONV(1111, 2, 10)`"; + String execResult = execute(convPpl); + JsonArray dataRow = parseAndGetFirstDataRow(execResult); + assertEquals(4, dataRow.size()); + assertEquals("c", dataRow.get(0).getAsString()); + assertEquals("44", dataRow.get(1).getAsString()); + assertEquals("1100", dataRow.get(2).getAsString()); + assertEquals("15", dataRow.get(3).getAsString()); + } + + @Test + public void testCos() { + testSimplePPL("source=people | eval `COS(0)` = COS(0) | fields `COS(0)`", List.of(1.0)); + } + + @Test + public void testCot() { + testSimplePPL( + "source=people | eval `COT(1)` = COT(1) | fields `COT(1)`", List.of(1.0 / Math.tan(1))); + } + + @Test + public void testCrc32() { + // TODO: No corresponding built-in implementation + testSimplePPL( + "source=people | eval `CRC32('MySQL')` = CRC32('MySQL') | fields `CRC32('MySQL')`", + List.of(3259397556L)); + } + + @Test + public void testDegrees() { + testSimplePPL( + "source=people | eval `DEGREES(1.57)` = DEGREES(1.57) | fields `DEGREES(1.57)`", + List.of(Math.toDegrees(1.57))); + } + + @Test + public void testEuler() { + // TODO: No corresponding built-in implementation + testSimplePPL("source=people | eval `E()` = E() | fields `E()`", List.of(Math.E)); + } + + @Test + public void testExp() { + testSimplePPL("source=people | eval `EXP(2)` = EXP(2) | fields `EXP(2)`", List.of(Math.exp(2))); + } + + @Test + public void testFloor() { + testSimplePPL( + "source=people | eval `FLOOR(0)` = FLOOR(0), `FLOOR(50.00005)` = FLOOR(50.00005)," + + " `FLOOR(-50.00005)` = FLOOR(-50.00005) | fields `FLOOR(0)`, `FLOOR(50.00005)`," + + " `FLOOR(-50.00005)`", + List.of(Math.floor(0.0), Math.floor(50.00005), Math.floor(-50.00005))); + testSimplePPL( + "source=people | eval `FLOOR(3147483647.12345)` = FLOOR(3147483647.12345)," + + " `FLOOR(113147483647.12345)` = FLOOR(113147483647.12345), `FLOOR(3147483647.00001)`" + + " = FLOOR(3147483647.00001) | fields `FLOOR(3147483647.12345)`," + + " `FLOOR(113147483647.12345)`, `FLOOR(3147483647.00001)`", + List.of( + Math.floor(3147483647.12345), + Math.floor(113147483647.12345), + Math.floor(3147483647.00001))); + testSimplePPL( + "source=people | eval `FLOOR(282474973688888.022)` = FLOOR(282474973688888.022)," + + " `FLOOR(9223372036854775807.022)` = FLOOR(9223372036854775807.022)," + + " `FLOOR(9223372036854775807.0000001)` = FLOOR(9223372036854775807.0000001) | fields" + + " `FLOOR(282474973688888.022)`, `FLOOR(9223372036854775807.022)`," + + " `FLOOR(9223372036854775807.0000001)`", + List.of( + Math.floor(282474973688888.022), + Math.floor(9223372036854775807.022), + Math.floor(9223372036854775807.0000001))); + } + + @Test + public void testLn() { + testSimplePPL("source=people | eval `LN(2)` = LN(2) | fields `LN(2)`", List.of(Math.log(2))); + } + + @Test + public void testLog() { + // TODO: No built-in function for 2-operand log + testSimplePPL( + "source=people | eval `LOG(2)` = LOG(2), `LOG(2, 8)` = LOG(2, 8) | fields `LOG(2)`, `LOG(2," + + " 8)`", + List.of(Math.log(2), Math.log(8) / Math.log(2))); + } + + @Test + public void testLog2() { + testSimplePPL( + "source=people | eval `LOG2(8)` = LOG2(8) | fields `LOG2(8)`", + List.of(Math.log(8) / Math.log(2))); + } + + @Test + public void testLog10() { + testSimplePPL( + "source=people | eval `LOG10(100)` = LOG10(100) | fields `LOG10(100)`", + List.of(Math.log10(100))); + } + + @Test + public void testMod() { + // TODO: There is a difference between MOD in OpenSearch and SQL standard library + // For MOD in Calcite, MOD(3.1, 2) = 1 + testSimplePPL( + "source=people | eval `MOD(3, 2)` = MOD(3, 2), `MOD(3.1, 2)` = MOD(3.1, 2) | fields `MOD(3," + + " 2)`, `MOD(3.1, 2)`", + List.of(1, 1.1)); + } + + @Test + public void testPi() { + testSimplePPL("source=people | eval `PI()` = PI() | fields `PI()`", List.of(Math.PI)); + } + + @Test + public void testPowAndPower() { + testSimplePPL( + "source=people | eval `POW(3, 2)` = POW(3, 2), `POW(-3, 2)` = POW(-3, 2), `POW(3, -2)` =" + + " POW(3, -2) | fields `POW(3, 2)`, `POW(-3, 2)`, `POW(3, -2)`", + List.of(Math.pow(3, 2), Math.pow(-3, 2), Math.pow(3, -2))); + testSimplePPL( + "source=people | eval `POWER(3, 2)` = POWER(3, 2), `POWER(-3, 2)` = POWER(-3, 2), `POWER(3," + + " -2)` = POWER(3, -2) | fields `POWER(3, 2)`, `POWER(-3, 2)`, `POWER(3, -2)`", + List.of(Math.pow(3, 2), Math.pow(-3, 2), Math.pow(3, -2))); + } + + @Test + public void testRadians() { + testSimplePPL( + "source=people | eval `RADIANS(90)` = RADIANS(90) | fields `RADIANS(90)`", + List.of(Math.toRadians(90))); + } + + @Test + public void testRand() { + String randPpl = "source=people | eval `RAND(3)` = RAND(3) | fields `RAND(3)`"; + String execResult1 = execute(randPpl); + String execResult2 = execute(randPpl); + assertEquals(execResult1, execResult2); + double val = parseAndGetFirstDataRow(execResult1).get(0).getAsDouble(); + assertTrue(val >= 0 && val <= 1); + } + + @Test + public void testRound() { + testSimplePPL( + "source=people | eval `ROUND(12.34)` = ROUND(12.34), `ROUND(12.34, 1)` = ROUND(12.34, 1)," + + " `ROUND(12.34, -1)` = ROUND(12.34, -1), `ROUND(12, 1)` = ROUND(12, 1) | fields" + + " `ROUND(12.34)`, `ROUND(12.34, 1)`, `ROUND(12.34, -1)`, `ROUND(12, 1)`", + List.of( + Math.round(12.34), + Math.round(12.34 * 10) / 10.0, + Math.round(12.34 / 10) * 10.0, + Math.round(12.0 * 10) / 10.0)); + } + + @Test + public void testSign() { + testSimplePPL( + "source=people | eval `SIGN(1)` = SIGN(1), `SIGN(0)` = SIGN(0), `SIGN(-1.1)` = SIGN(-1.1) |" + + " fields `SIGN(1)`, `SIGN(0)`, `SIGN(-1.1)`", + List.of(1, 0, -1)); + } + + @Test + public void testSin() { + testSimplePPL( + "source=people | eval `SIN(0)` = SIN(0) | fields `SIN(0)`", List.of(Math.sin(0.0))); + } + + @Test + public void testSqrt() { + testSimplePPL( + "source=people | eval `SQRT(4)` = SQRT(4), `SQRT(4.41)` = SQRT(4.41) | fields `SQRT(4)`," + + " `SQRT(4.41)`", + List.of(Math.sqrt(4), Math.sqrt(4.41))); + } + + @Test + public void testCbrt() { + testSimplePPL( + "source=people | eval `CBRT(8)` = CBRT(8), `CBRT(9.261)` = CBRT(9.261), `CBRT(-27)` =" + + " CBRT(-27) | fields `CBRT(8)`, `CBRT(9.261)`, `CBRT(-27)`", + List.of(Math.cbrt(8), Math.cbrt(9.261), Math.cbrt(-27))); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java index 20fb52c112..6b4e26a7e7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteOpenSearchIndexScan.java @@ -125,7 +125,9 @@ public boolean pushDownFilter(Filter filter) { requestBuilder.pushDownFilter(filterBuilder); // TODO: handle the case where condition contains a score function return true; - } catch (ExpressionNotAnalyzableException | PredicateAnalyzerException e) { + } catch (ExpressionNotAnalyzableException + | PredicateAnalyzerException + | UnsupportedOperationException e) { LOG.warn("Cannot analyze the filter condition {}", filter.getCondition(), e); } return false; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java index a0431535ea..f2de4d2dbd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java @@ -8,7 +8,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; +import java.util.Arrays; import lombok.experimental.UtilityClass; +import org.apache.calcite.avatica.util.ArrayImpl; import org.apache.calcite.rel.type.RelDataType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -87,6 +89,12 @@ public static ExprValue getExprValueFromSqlType( case Types.BOOLEAN: value = rs.getBoolean(i); break; + case Types.ARRAY: + value = rs.getArray(i); + if (value instanceof ArrayImpl) { + value = Arrays.asList((Object[]) ((ArrayImpl) value).getArray()); + } + break; default: value = rs.getObject(i); LOG.warn(