Always evaluate built-in functions using the presto.default namespace

presto-main-base/src/main/java/com/facebook/presto/cost/StatsCalculatorModule.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.cost;
 
 import com.facebook.presto.metadata.Metadata;
+import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Binder;
 import com.google.inject.Module;
@@ -46,9 +47,10 @@ public static StatsCalculator createNewStatsCalculator(
             StatsNormalizer normalizer,
             FilterStatsCalculator filterStatsCalculator,
             HistoryBasedPlanStatisticsManager historyBasedPlanStatisticsManager,
-            FragmentStatsProvider fragmentStatsProvider)
+            FragmentStatsProvider fragmentStatsProvider,
+            ExpressionOptimizerManager expressionOptimizerManager)
     {
-        StatsCalculator delegate = createComposableStatsCalculator(metadata, scalarStatsCalculator, normalizer, filterStatsCalculator, fragmentStatsProvider);
+        StatsCalculator delegate = createComposableStatsCalculator(metadata, scalarStatsCalculator, normalizer, filterStatsCalculator, fragmentStatsProvider, expressionOptimizerManager);
         return historyBasedPlanStatisticsManager.getHistoryBasedPlanStatisticsCalculator(delegate);
     }
 
@@ -57,14 +59,15 @@ public static ComposableStatsCalculator createComposableStatsCalculator(
             ScalarStatsCalculator scalarStatsCalculator,
             StatsNormalizer normalizer,
             FilterStatsCalculator filterStatsCalculator,
-            FragmentStatsProvider fragmentStatsProvider)
+            FragmentStatsProvider fragmentStatsProvider,
+            ExpressionOptimizerManager expressionOptimizerManager)
     {
         ImmutableList.Builder<ComposableStatsCalculator.Rule<?>> rules = ImmutableList.builder();
         rules.add(new OutputStatsRule());
         rules.add(new TableScanStatsRule(metadata, normalizer));
         rules.add(new SimpleFilterProjectSemiJoinStatsRule(normalizer, filterStatsCalculator, metadata.getFunctionAndTypeManager())); // this must be before FilterStatsRule
         rules.add(new FilterStatsRule(normalizer, filterStatsCalculator));
-        rules.add(new ValuesStatsRule(metadata));
+        rules.add(new ValuesStatsRule(metadata, expressionOptimizerManager));
         rules.add(new LimitStatsRule(normalizer));
         rules.add(new EnforceSingleRowStatsRule(normalizer));
         rules.add(new ProjectStatsRule(scalarStatsCalculator, normalizer));

presto-main-base/src/main/java/com/facebook/presto/cost/ValuesStatsRule.java

@@ -19,7 +19,9 @@
 import com.facebook.presto.matching.Pattern;
 import com.facebook.presto.metadata.Metadata;
 import com.facebook.presto.spi.plan.ValuesNode;
+import com.facebook.presto.spi.relation.ConstantExpression;
 import com.facebook.presto.spi.relation.VariableReferenceExpression;
+import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
 import com.facebook.presto.sql.planner.TypeProvider;
 import com.facebook.presto.sql.planner.iterative.Lookup;
 
@@ -32,10 +34,12 @@
 
 import static com.facebook.presto.common.type.UnknownType.UNKNOWN;
 import static com.facebook.presto.cost.StatsUtil.toStatsRepresentation;
+import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED;
 import static com.facebook.presto.spi.statistics.SourceInfo.ConfidenceLevel.FACT;
-import static com.facebook.presto.sql.planner.RowExpressionInterpreter.evaluateConstantRowExpression;
 import static com.facebook.presto.sql.planner.plan.Patterns.values;
+import static com.google.common.base.Verify.verify;
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static java.util.Objects.requireNonNull;
 import static java.util.stream.Collectors.toList;
 
 public class ValuesStatsRule
@@ -44,10 +48,12 @@ public class ValuesStatsRule
     private static final Pattern<ValuesNode> PATTERN = values();
 
     private final Metadata metadata;
+    private final ExpressionOptimizerManager expressionOptimizerManager;
 
-    public ValuesStatsRule(Metadata metadata)
+    public ValuesStatsRule(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager)
     {
-        this.metadata = metadata;
+        this.metadata = requireNonNull(metadata, "metadata is null");
+        this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null");
     }
 
     @Override
@@ -82,7 +88,11 @@ private List<Object> getVariableValues(ValuesNode valuesNode, int symbolId, Sess
         }
         return valuesNode.getRows().stream()
                 .map(row -> row.get(symbolId))
-                .map(rowExpression -> evaluateConstantRowExpression(rowExpression, metadata.getFunctionAndTypeManager(), session.toConnectorSession()))
+                .map(rowExpression -> expressionOptimizerManager.getExpressionOptimizer(session.toConnectorSession())
+                        .optimize(rowExpression, EVALUATED, session.toConnectorSession(), i -> i))
+                .peek(rowExpression -> verify(rowExpression instanceof ConstantExpression, "Expected constant expression, but got: %s", rowExpression))
+                .map(rowExpression -> (ConstantExpression) rowExpression)
+                .map(ConstantExpression::getValue)
                 .collect(toList());
     }
 

presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java

@@ -744,7 +744,7 @@ public PlanOptimizers(
         builder.add(predicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate
         builder.add(simplifyRowExpressionOptimizer); // Should be always run after PredicatePushDown
 
-        builder.add(new MetadataQueryOptimizer(metadata));
+        builder.add(new MetadataQueryOptimizer(metadata, expressionOptimizerManager));
 
         // This can pull up Filter and Project nodes from between Joins, so we need to push them down again
         builder.add(

presto-main-base/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java

@@ -109,7 +109,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Predicates.instanceOf;
-import static com.google.common.base.Verify.verify;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.getOnlyElement;
 import static io.airlift.slice.Slices.utf8Slice;
@@ -130,15 +129,6 @@ public class RowExpressionInterpreter
     private final FunctionResolution resolution;
 
     private final Visitor visitor;
-
-    public static Object evaluateConstantRowExpression(RowExpression expression, FunctionAndTypeManager functionAndTypeManager, ConnectorSession session)
-    {
-        // evaluate the expression
-        Object result = new RowExpressionInterpreter(expression, functionAndTypeManager, session, EVALUATED).evaluate();
-        verify(!(result instanceof RowExpression), "RowExpression interpreter returned an unresolved expression");
-        return result;
-    }
-
     public static RowExpressionInterpreter rowExpressionInterpreter(RowExpression expression, FunctionAndTypeManager functionAndTypeManager, ConnectorSession session)
     {
         return new RowExpressionInterpreter(expression, functionAndTypeManager, session, EVALUATED);

presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataQueryOptimizer.java

@@ -46,6 +46,7 @@
 import com.facebook.presto.spi.relation.RowExpression;
 import com.facebook.presto.spi.relation.VariableReferenceExpression;
 import com.facebook.presto.spi.statistics.TableStatistics;
+import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
 import com.facebook.presto.sql.planner.TypeProvider;
 import com.facebook.presto.sql.planner.VariablesExtractor;
 import com.facebook.presto.sql.planner.plan.SimplePlanRewriter;
@@ -64,9 +65,10 @@
 import java.util.Set;
 
 import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL;
-import static com.facebook.presto.sql.planner.RowExpressionInterpreter.evaluateConstantRowExpression;
+import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED;
 import static com.facebook.presto.sql.relational.Expressions.call;
 import static com.facebook.presto.sql.relational.Expressions.constant;
+import static com.google.common.base.Verify.verify;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.getOnlyElement;
 import static java.util.Objects.requireNonNull;
@@ -81,12 +83,13 @@ public class MetadataQueryOptimizer
     private final Set<QualifiedObjectName> allowedFunctions;
     private final Map<QualifiedObjectName, QualifiedObjectName> aggregationScalarMapping;
     private final Metadata metadata;
+    private final ExpressionOptimizerManager expressionOptimizerManager;
 
-    public MetadataQueryOptimizer(Metadata metadata)
+    public MetadataQueryOptimizer(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager)
     {
-        requireNonNull(metadata, "metadata is null");
+        this.metadata = requireNonNull(metadata, "metadata is null");
+        this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null");
 
-        this.metadata = metadata;
         CatalogSchemaName defaultNamespace = metadata.getFunctionAndTypeManager().getDefaultNamespace();
         this.allowedFunctions = ImmutableSet.of(
                 QualifiedObjectName.valueOf(defaultNamespace, "max"),
@@ -104,7 +107,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider
         if (!SystemSessionProperties.isOptimizeMetadataQueries(session) && !SystemSessionProperties.isOptimizeMetadataQueriesIgnoreStats(session)) {
             return PlanOptimizerResult.optimizerResult(plan, false);
         }
-        Optimizer optimizer = new Optimizer(session, metadata, idAllocator);
+        Optimizer optimizer = new Optimizer(session, metadata, idAllocator, expressionOptimizerManager);
         PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(optimizer, plan, null);
         return PlanOptimizerResult.optimizerResult(rewrittenPlan, optimizer.isPlanChanged());
     }
@@ -130,16 +133,18 @@ private static class Optimizer
         private final int metastoreCallNumThreshold;
         private boolean planChanged;
         private final MetadataQueryOptimizer metadataQueryOptimizer;
+        private final ExpressionOptimizerManager expressionOptimizerManager;
 
-        private Optimizer(Session session, Metadata metadata, PlanNodeIdAllocator idAllocator)
+        private Optimizer(Session session, Metadata metadata, PlanNodeIdAllocator idAllocator, ExpressionOptimizerManager expressionOptimizerManager)
         {
             this.session = session;
             this.metadata = metadata;
             this.idAllocator = idAllocator;
             this.determinismEvaluator = new RowExpressionDeterminismEvaluator(metadata);
             this.ignoreMetadataStats = SystemSessionProperties.isOptimizeMetadataQueriesIgnoreStats(session);
             this.metastoreCallNumThreshold = SystemSessionProperties.getOptimizeMetadataQueriesCallThreshold(session);
-            this.metadataQueryOptimizer = new MetadataQueryOptimizer(metadata);
+            this.metadataQueryOptimizer = new MetadataQueryOptimizer(metadata, expressionOptimizerManager);
+            this.expressionOptimizerManager = expressionOptimizerManager;
         }
 
         public boolean isPlanChanged()
@@ -374,15 +379,17 @@ private RowExpression evaluateMinMax(FunctionMetadata aggregationFunctionMetadat
                 List<RowExpression> reducedArguments = new ArrayList<>();
                 // We fold for every 100 values because GREATEST/LEAST has argument count limit
                 for (List<RowExpression> partitionedArguments : Lists.partition(arguments, 100)) {
-                    Object reducedValue = evaluateConstantRowExpression(
+                    RowExpression expression = expressionOptimizerManager.getExpressionOptimizer(connectorSession).optimize(
                             call(
                                     metadata.getFunctionAndTypeManager(),
                                     scalarFunctionName,
                                     returnType,
                                     partitionedArguments),
-                            metadata.getFunctionAndTypeManager(),
-                            connectorSession);
-                    reducedArguments.add(constant(reducedValue, returnType));
+                            EVALUATED,
+                            connectorSession,
+                            i -> i);
+                    verify(expression instanceof ConstantExpression, "Expected constant expression");
+                    reducedArguments.add(expression);
                 }
                 arguments = reducedArguments;
             }

presto-main-base/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java

@@ -13,24 +13,34 @@
  */
 package com.facebook.presto.sql.relational;
 
+import com.facebook.presto.common.CatalogSchemaName;
+import com.facebook.presto.common.QualifiedObjectName;
 import com.facebook.presto.metadata.FunctionAndTypeManager;
 import com.facebook.presto.metadata.Metadata;
 import com.facebook.presto.spi.ConnectorSession;
+import com.facebook.presto.spi.relation.CallExpression;
 import com.facebook.presto.spi.relation.ExpressionOptimizer;
 import com.facebook.presto.spi.relation.RowExpression;
+import com.facebook.presto.spi.relation.RowExpressionVisitor;
 import com.facebook.presto.spi.relation.VariableReferenceExpression;
+import com.facebook.presto.sql.analyzer.TypeSignatureProvider;
 import com.facebook.presto.sql.planner.RowExpressionInterpreter;
 
 import java.util.function.Function;
 
+import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE;
 import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED;
 import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static java.util.Objects.requireNonNull;
+import static java.util.function.UnaryOperator.identity;
 
 public final class RowExpressionOptimizer
         implements ExpressionOptimizer
 {
     private final FunctionAndTypeManager functionAndTypeManager;
+    private final CatalogSchemaName defaultNamespace;
+    private final Function<RowExpression, RowExpression> normalizeRowExpression;
 
     public RowExpressionOptimizer(Metadata metadata)
     {
@@ -39,22 +49,73 @@ public RowExpressionOptimizer(Metadata metadata)
 
     public RowExpressionOptimizer(FunctionAndTypeManager functionAndTypeManager)
     {
+        this.defaultNamespace = requireNonNull(functionAndTypeManager, "functionMetadataManager is null").getDefaultNamespace();
         this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionMetadataManager is null");
+        if (!defaultNamespace.equals(JAVA_BUILTIN_NAMESPACE)) {
+            this.normalizeRowExpression = rowExpression -> rowExpression.accept(new BuiltInFunctionNamespaceOverride(), null);
+        }
+        else {
+            this.normalizeRowExpression = identity();
+        }
     }
 
     @Override
     public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session)
     {
         if (level.ordinal() <= OPTIMIZED.ordinal()) {
-            return toRowExpression(rowExpression.getSourceLocation(), new RowExpressionInterpreter(rowExpression, functionAndTypeManager, session, level).optimize(), rowExpression.getType());
+            RowExpressionInterpreter rowExpressionInterpreter = new RowExpressionInterpreter(
+                    normalizeRowExpression.apply(rowExpression),
+                    functionAndTypeManager,
+                    session,
+                    level);
+            return toRowExpression(rowExpression.getSourceLocation(), rowExpressionInterpreter.optimize(), rowExpression.getType());
         }
         throw new IllegalArgumentException("Not supported optimization level: " + level);
     }
 
     @Override
     public RowExpression optimize(RowExpression expression, Level level, ConnectorSession session, Function<VariableReferenceExpression, Object> variableResolver)
     {
-        RowExpressionInterpreter interpreter = new RowExpressionInterpreter(expression, functionAndTypeManager, session, level);
+        RowExpressionInterpreter interpreter = new RowExpressionInterpreter(
+                normalizeRowExpression.apply(expression),
+                functionAndTypeManager,
+                session,
+                level);
         return toRowExpression(expression.getSourceLocation(), interpreter.optimize(variableResolver::apply), expression.getType());
     }
+
+    /**
+     * TODO: GIANT HACK
+     * This class is a hack and should eventually be removed.  It is used to ensure consistent constant folding behavior when the built-in
+     * function namespace has been switched (for example, to native.default. in the case of native functions).  This will no longer be needed
+     * when the native sidecar is capable of providing its own expression optimizer.
+     */
+    private class BuiltInFunctionNamespaceOverride
+            implements RowExpressionVisitor<RowExpression, Void>
+    {
+        @Override
+        public RowExpression visitExpression(RowExpression expression, Void context)
+        {
+            return expression;
+        }
+
+        @Override
+        public RowExpression visitCall(CallExpression call, Void context)
+        {
+            if (call.getFunctionHandle().getCatalogSchemaName().equals(defaultNamespace)) {
+                call = new CallExpression(
+                        call.getSourceLocation(),
+                        call.getDisplayName(),
+                        functionAndTypeManager.lookupFunction(
+                                QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, call.getDisplayName()),
+                                call.getArguments().stream()
+                                        .map(RowExpression::getType)
+                                        .map(x -> new TypeSignatureProvider(x.getTypeSignature()))
+                                        .collect(toImmutableList())),
+                        call.getType(),
+                        call.getArguments());
+            }
+            return call;
+        }
+    }
 }

presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java

@@ -478,7 +478,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig,
         this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer);
         this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, createTestingSessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig(), featuresConfig, new NodeVersion("1"));
         this.fragmentStatsProvider = new FragmentStatsProvider();
-        this.statsCalculator = createNewStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer, filterStatsCalculator, historyBasedPlanStatisticsManager, fragmentStatsProvider);
+        this.statsCalculator = createNewStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer, filterStatsCalculator, historyBasedPlanStatisticsManager, fragmentStatsProvider, expressionOptimizerManager);
         this.taskCountEstimator = new TaskCountEstimator(() -> nodeCountForStats);
         this.costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator);
         this.estimatedExchangesCostCalculator = new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator);