Skip to content

Commit d2a5953

Browse files
ScrapCodesScrapCodes
committed
Converted the TPCH/TPCDS tests to run per feature.
1 parent b85fc3e commit d2a5953

File tree

3 files changed

+64
-42
lines changed

3 files changed

+64
-42
lines changed

‎presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java

Lines changed: 38 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,15 @@
3939
import java.nio.file.Path;
4040
import java.nio.file.Paths;
4141
import java.util.Map;
42+
import java.util.Map.Entry;
4243
import java.util.stream.Stream;
4344

4445
import static com.facebook.presto.SystemSessionProperties.OPTIMIZER_USE_HISTOGRAMS;
4546
import static com.facebook.presto.SystemSessionProperties.SCALAR_FUNCTION_STATS_PROPAGATION_ENABLED;
4647
import static com.facebook.presto.spi.plan.JoinDistributionType.REPLICATED;
4748
import static com.facebook.presto.spi.plan.JoinType.INNER;
4849
import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED;
49-
import static com.facebook.presto.testing.TestngUtils.toDataProvider;
50+
import static com.facebook.presto.testing.TestngUtils.toDataProviderFromArray;
5051
import static com.google.common.base.Preconditions.checkState;
5152
import static com.google.common.base.Verify.verify;
5253
import static com.google.common.io.Files.createParentDirs;
@@ -62,9 +63,11 @@
6263
public abstract class AbstractCostBasedPlanTest
6364
extends BasePlanTest
6465
{
65-
private final Map<String, String> featureToOutputDir =
66-
ImmutableMap.of(OPTIMIZER_USE_HISTOGRAMS, "histogram",
67-
SCALAR_FUNCTION_STATS_PROPAGATION_ENABLED, "scalar_function_stats_propagation");
66+
private static final String NO_FEATURE_ENABLED = "no_feature_enabled";
67+
68+
private final Map<String, String> featuresMap =
69+
ImmutableMap.of(NO_FEATURE_ENABLED, "", "histogram", OPTIMIZER_USE_HISTOGRAMS,
70+
"scalar_function_stats_propagation", SCALAR_FUNCTION_STATS_PROPAGATION_ENABLED);
6871

6972
public AbstractCostBasedPlanTest(LocalQueryRunnerSupplier supplier)
7073
{
@@ -76,33 +79,29 @@ public AbstractCostBasedPlanTest(LocalQueryRunnerSupplier supplier)
7679
@DataProvider
7780
public Object[][] getQueriesDataProvider()
7881
{
79-
return getQueryResourcePaths()
80-
.collect(toDataProvider());
82+
return featuresMap.keySet().stream().flatMap(feature -> getQueryResourcePaths().map(path -> new String[] {feature, path})).collect(toDataProviderFromArray());
8183
}
8284

8385
@Test(dataProvider = "getQueriesDataProvider")
84-
public void test(String queryResourcePath)
85-
{
86-
assertEquals(generateQueryPlan(read(queryResourcePath)), read(getQueryPlanResourcePath(queryResourcePath)));
87-
}
88-
89-
@Test(dataProvider = "getQueriesDataProvider")
90-
public void featureSpecificPlansMatch(String queryResourcePath)
86+
public void test(String feature, String queryResourcePath)
9187
{
9288
String sql = read(queryResourcePath);
93-
for (Map.Entry<String, String> featureEntry : featureToOutputDir.entrySet()) {
89+
if (!feature.equals(NO_FEATURE_ENABLED)) {
9490
Session featureEnabledSession = Session.builder(getQueryRunner().getDefaultSession())
95-
.setSystemProperty(featureEntry.getKey(), "true")
91+
.setSystemProperty(featuresMap.get(feature), "true")
9692
.build();
9793
Session featureDisabledSession = Session.builder(getQueryRunner().getDefaultSession())
98-
.setSystemProperty(featureEntry.getKey(), "false")
94+
.setSystemProperty(featuresMap.get(feature), "false")
9995
.build();
10096
String regularPlan = generateQueryPlan(sql, featureDisabledSession);
10197
String featureEnabledPlan = generateQueryPlan(sql, featureEnabledSession);
10298
if (!regularPlan.equals(featureEnabledPlan)) {
103-
assertEquals(featureEnabledPlan, read(getSpecificPlanResourcePath(featureEntry.getValue(), getQueryPlanResourcePath(queryResourcePath))));
99+
assertEquals(featureEnabledPlan, read(getSpecificPlanResourcePath(feature, getQueryPlanResourcePath(queryResourcePath))));
104100
}
105101
}
102+
else {
103+
assertEquals(generateQueryPlan(sql), read(getQueryPlanResourcePath(queryResourcePath)));
104+
}
106105
}
107106

108107
private String getQueryPlanResourcePath(String queryResourcePath)
@@ -113,7 +112,7 @@ private String getQueryPlanResourcePath(String queryResourcePath)
113112
private String getSpecificPlanResourcePath(String outDirPath, String regularPlanResourcePath)
114113
{
115114
Path root = Paths.get(regularPlanResourcePath);
116-
return root.getParent().resolve(String.format("%s/%s", outDirPath, root.getFileName())).toString();
115+
return root.getParent().resolve(format("%s/%s", outDirPath, root.getFileName())).toString();
117116
}
118117

119118
private Path getResourceWritePath(String queryResourcePath)
@@ -133,25 +132,28 @@ public void generate()
133132
.parallel()
134133
.forEach(queryResourcePath -> {
135134
try {
136-
for (Map.Entry<String, String> featureEntry : featureToOutputDir.entrySet()) {
137-
Path queryPlanWritePath = getResourceWritePath(queryResourcePath);
138-
createParentDirs(queryPlanWritePath.toFile());
135+
for (Entry<String, String> featureEntry : featuresMap.entrySet()) {
139136
String sql = read(queryResourcePath);
140-
Session featuredisabledSession = Session.builder(getQueryRunner().getDefaultSession())
141-
.setSystemProperty(featureEntry.getKey(), "false")
142-
.build();
143-
String regularPlan = generateQueryPlan(sql, featuredisabledSession);
144-
Session featureEnabledSession = Session.builder(getQueryRunner().getDefaultSession())
145-
.setSystemProperty(featureEntry.getKey(), "true")
146-
.build();
147-
148-
String featureEnabledPlan = generateQueryPlan(sql, featureEnabledSession);
149-
write(regularPlan.getBytes(UTF_8), queryPlanWritePath.toFile());
150-
// write out the feature enabled plan if it differs
151-
if (!regularPlan.equals(featureEnabledPlan)) {
152-
Path featureEnabledPlanWritePath = getResourceWritePath(getSpecificPlanResourcePath(featureEntry.getValue(), queryResourcePath));
153-
createParentDirs(featureEnabledPlanWritePath.toFile());
154-
write(featureEnabledPlan.getBytes(UTF_8), featureEnabledPlanWritePath.toFile());
137+
if (!featureEntry.getKey().equals(NO_FEATURE_ENABLED)) {
138+
Session featureDisabledSession = Session.builder(getQueryRunner().getDefaultSession())
139+
.setSystemProperty(featureEntry.getValue(), "false")
140+
.build();
141+
String regularPlan = generateQueryPlan(sql, featureDisabledSession);
142+
Session featureEnabledSession = Session.builder(getQueryRunner().getDefaultSession())
143+
.setSystemProperty(featureEntry.getValue(), "true")
144+
.build();
145+
String featureEnabledPlan = generateQueryPlan(sql, featureEnabledSession);
146+
// write out the feature enabled plan if it differs
147+
if (!regularPlan.equals(featureEnabledPlan)) {
148+
Path featureEnabledPlanWritePath = getResourceWritePath(getSpecificPlanResourcePath(featureEntry.getKey(), queryResourcePath));
149+
createParentDirs(featureEnabledPlanWritePath.toFile());
150+
write(featureEnabledPlan.getBytes(UTF_8), featureEnabledPlanWritePath.toFile());
151+
}
152+
}
153+
else {
154+
Path queryPlanWritePath = getResourceWritePath(queryResourcePath);
155+
createParentDirs(queryPlanWritePath.toFile());
156+
write(generateQueryPlan(sql).getBytes(UTF_8), queryPlanWritePath.toFile());
155157
}
156158
System.out.println("Generated expected plan for query: " + queryResourcePath);
157159
}

‎presto-main/src/main/java/com/facebook/presto/operator/scalar/MathFunctions.java

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.facebook.presto.spi.function.Description;
2424
import com.facebook.presto.spi.function.LiteralParameters;
2525
import com.facebook.presto.spi.function.ScalarFunction;
26+
import com.facebook.presto.spi.function.ScalarPropagateSourceStats;
2627
import com.facebook.presto.spi.function.Signature;
2728
import com.facebook.presto.spi.function.SqlNullable;
2829
import com.facebook.presto.spi.function.SqlType;
@@ -63,6 +64,7 @@
6364
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
6465
import static com.facebook.presto.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE;
6566
import static com.facebook.presto.spi.function.FunctionKind.SCALAR;
67+
import static com.facebook.presto.spi.function.StatsPropagationBehavior.USE_SOURCE_STATS;
6668
import static com.facebook.presto.type.DecimalOperators.modulusScalarFunction;
6769
import static com.facebook.presto.type.DecimalOperators.modulusSignatureBuilder;
6870
import static com.facebook.presto.util.Failures.checkCondition;
@@ -105,7 +107,8 @@ private MathFunctions() {}
105107
@Description("absolute value")
106108
@ScalarFunction("abs")
107109
@SqlType(StandardTypes.TINYINT)
108-
public static long absTinyint(@SqlType(StandardTypes.TINYINT) long num)
110+
public static long absTinyint(
111+
@ScalarPropagateSourceStats(propagateAllStats = false, nullFraction = USE_SOURCE_STATS) @SqlType(StandardTypes.TINYINT) long num)
109112
{
110113
checkCondition(num != Byte.MIN_VALUE, NUMERIC_VALUE_OUT_OF_RANGE, "Value -128 is out of range for abs(tinyint)");
111114
return Math.abs(num);
@@ -114,7 +117,8 @@ public static long absTinyint(@SqlType(StandardTypes.TINYINT) long num)
114117
@Description("absolute value")
115118
@ScalarFunction("abs")
116119
@SqlType(StandardTypes.SMALLINT)
117-
public static long absSmallint(@SqlType(StandardTypes.SMALLINT) long num)
120+
public static long absSmallint(
121+
@ScalarPropagateSourceStats(propagateAllStats = false, nullFraction = USE_SOURCE_STATS) @SqlType(StandardTypes.SMALLINT) long num)
118122
{
119123
checkCondition(num != Short.MIN_VALUE, NUMERIC_VALUE_OUT_OF_RANGE, "Value -32768 is out of range for abs(smallint)");
120124
return Math.abs(num);
@@ -123,7 +127,8 @@ public static long absSmallint(@SqlType(StandardTypes.SMALLINT) long num)
123127
@Description("absolute value")
124128
@ScalarFunction("abs")
125129
@SqlType(StandardTypes.INTEGER)
126-
public static long absInteger(@SqlType(StandardTypes.INTEGER) long num)
130+
public static long absInteger(
131+
@ScalarPropagateSourceStats(propagateAllStats = false, nullFraction = USE_SOURCE_STATS) @SqlType(StandardTypes.INTEGER) long num)
127132
{
128133
checkCondition(num != Integer.MIN_VALUE, NUMERIC_VALUE_OUT_OF_RANGE, "Value -2147483648 is out of range for abs(integer)");
129134
return Math.abs(num);
@@ -132,7 +137,8 @@ public static long absInteger(@SqlType(StandardTypes.INTEGER) long num)
132137
@Description("absolute value")
133138
@ScalarFunction
134139
@SqlType(StandardTypes.BIGINT)
135-
public static long abs(@SqlType(StandardTypes.BIGINT) long num)
140+
public static long abs(
141+
@ScalarPropagateSourceStats(propagateAllStats = false, nullFraction = USE_SOURCE_STATS) @SqlType(StandardTypes.BIGINT) long num)
136142
{
137143
checkCondition(num != Long.MIN_VALUE, NUMERIC_VALUE_OUT_OF_RANGE, "Value -9223372036854775808 is out of range for abs(bigint)");
138144
return Math.abs(num);
@@ -141,7 +147,8 @@ public static long abs(@SqlType(StandardTypes.BIGINT) long num)
141147
@Description("absolute value")
142148
@ScalarFunction
143149
@SqlType(StandardTypes.DOUBLE)
144-
public static double abs(@SqlType(StandardTypes.DOUBLE) double num)
150+
public static double abs(
151+
@ScalarPropagateSourceStats(propagateAllStats = false, nullFraction = USE_SOURCE_STATS) @SqlType(StandardTypes.DOUBLE) double num)
145152
{
146153
return Math.abs(num);
147154
}
@@ -154,7 +161,8 @@ private Abs() {}
154161

155162
@LiteralParameters({"p", "s"})
156163
@SqlType("decimal(p, s)")
157-
public static long absShort(@SqlType("decimal(p, s)") long arg)
164+
public static long absShort(
165+
@ScalarPropagateSourceStats(propagateAllStats = false, nullFraction = USE_SOURCE_STATS) @SqlType("decimal(p, s)") long arg)
158166
{
159167
return arg > 0 ? arg : -arg;
160168
}

‎presto-main/src/main/java/com/facebook/presto/testing/TestngUtils.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,16 @@ private TestngUtils() {}
3131
},
3232
builder -> builder.toArray(new Object[][] {}));
3333
}
34+
35+
public static <T> Collector<T, ?, Object[][]> toDataProviderFromArray()
36+
{
37+
return Collector.of(
38+
ArrayList::new,
39+
ArrayList::add,
40+
(left, right) -> {
41+
left.addAll(right);
42+
return left;
43+
},
44+
builder -> builder.toArray(new Object[][] {}));
45+
}
3446
}

0 commit comments

Comments
 (0)