From 90458351a3ddbe251c178b964bbb5324dc715a2f Mon Sep 17 00:00:00 2001 From: Nishitha-Bhaskaran Date: Fri, 21 Mar 2025 19:42:16 +0530 Subject: [PATCH 01/10] Upgrade to Hadoop 3.4.1 Co-authored-by: Jalpreet Singh Nanda (:imjalpreet) Co-authored-by: Shijin K --- pom.xml | 32 ++++++++++++++++--- presto-accumulo/pom.xml | 1 + presto-cache/pom.xml | 8 +++-- presto-delta/pom.xml | 8 +++-- presto-druid/pom.xml | 8 +++-- presto-hdfs-core/pom.xml | 8 +++-- presto-hive-common/pom.xml | 8 +++-- presto-hive-function-namespace/pom.xml | 8 +++-- presto-hive-hadoop2/pom.xml | 9 ++++-- presto-hive-metastore/pom.xml | 11 +++++-- presto-hive/pom.xml | 11 +++++-- .../hive/HdfsConfigurationInitializer.java | 10 +++--- .../presto/hive/AbstractTestHiveClient.java | 2 +- presto-hudi/pom.xml | 8 +++-- presto-iceberg/pom.xml | 8 +++-- presto-kudu/pom.xml | 8 +++-- presto-orc/pom.xml | 11 +++++-- presto-parquet/pom.xml | 8 +++-- presto-rcfile/pom.xml | 17 ++++++++-- presto-spark-base/pom.xml | 11 +++++-- presto-spark-testing/pom.xml | 11 +++++-- 21 files changed, 155 insertions(+), 51 deletions(-) diff --git a/pom.xml b/pom.xml index 1bda530fde409..f4a4a7aabc4c9 100644 --- a/pom.xml +++ b/pom.xml @@ -216,6 +216,22 @@ presto-function-server + + + + + + + + + + + + jitpack.io + https://jitpack.io + + + @@ -231,6 +247,12 @@ 3.6.0 + + com.github.imjalpreet + presto-hadoop-apache2 + 3.4.1-1-Java17-2-SNAPSHOT + + com.squareup.okio okio-jvm @@ -994,11 +1016,11 @@ 2.5.2 - - com.facebook.presto.hadoop - hadoop-apache2 - 2.7.4-12 - + + + + + com.facebook.presto.hive diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index b3109275baac8..5717de1c796eb 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -175,6 +175,7 @@ com.facebook.presto.hadoop hadoop-apache2 + 2.7.4-12 diff --git a/presto-cache/pom.xml b/presto-cache/pom.xml index 7b0ce764f6b5a..28b03a8392aa5 100644 --- a/presto-cache/pom.xml +++ b/presto-cache/pom.xml @@ -16,9 +16,13 @@ + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-delta/pom.xml b/presto-delta/pom.xml index 1a32ab32fb5c3..db36ee6874674 100644 --- a/presto-delta/pom.xml +++ b/presto-delta/pom.xml @@ -195,9 +195,13 @@ presto-memory-context + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-druid/pom.xml b/presto-druid/pom.xml index 373e283f06ccf..b0eef82650833 100644 --- a/presto-druid/pom.xml +++ b/presto-druid/pom.xml @@ -221,9 +221,13 @@ joda-time + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-hdfs-core/pom.xml b/presto-hdfs-core/pom.xml index d8f6d934d467d..0fbc044e3757e 100644 --- a/presto-hdfs-core/pom.xml +++ b/presto-hdfs-core/pom.xml @@ -35,9 +35,13 @@ presto-spi + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-hive-common/pom.xml b/presto-hive-common/pom.xml index e72703aff2347..c2261fc709684 100644 --- a/presto-hive-common/pom.xml +++ b/presto-hive-common/pom.xml @@ -60,9 +60,13 @@ presto-plugin-toolkit + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-hive-function-namespace/pom.xml b/presto-hive-function-namespace/pom.xml index 9f39e31e065fc..1c5d8730fc5ff 100644 --- a/presto-hive-function-namespace/pom.xml +++ b/presto-hive-function-namespace/pom.xml @@ -26,9 +26,13 @@ jsr305 + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 0ce4a3eeee1b7..710bd0ee3618f 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -37,9 +37,14 @@ guava + + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 runtime diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml index 35621504fe753..f4b10c1213902 100644 --- a/presto-hive-metastore/pom.xml +++ b/presto-hive-metastore/pom.xml @@ -156,9 +156,14 @@ jmxutils - - com.facebook.presto.hadoop - hadoop-apache2 + + + + + + + com.github.imjalpreet + presto-hadoop-apache2 provided diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 78aaae1c247ae..ad2d3b36b6c42 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -98,9 +98,14 @@ presto-rcfile - - com.facebook.presto.hadoop - hadoop-apache2 + + + + + + + com.github.imjalpreet + presto-hadoop-apache2 provided diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java b/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java index 991b364614cd5..d1502b744db78 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hadoop.SocksSocketFactory; import com.facebook.presto.hive.gcs.GcsConfigurationInitializer; import com.facebook.presto.hive.s3.S3ConfigurationUpdater; import com.google.common.annotations.VisibleForTesting; @@ -21,9 +22,9 @@ import io.airlift.units.Duration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; import org.apache.hadoop.net.DNSToSwitchMapping; -import org.apache.hadoop.net.SocksSocketFactory; import javax.inject.Inject; import javax.net.SocketFactory; @@ -41,9 +42,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY; public class HdfsConfigurationInitializer { @@ -119,7 +119,7 @@ public void updateConfiguration(Configuration config) // only enable short circuit reads if domain socket path is properly configured if (!config.get(DFS_DOMAIN_SOCKET_PATH_KEY, "").trim().isEmpty()) { - config.setBooleanIfUnset(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); + config.setBooleanIfUnset(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true); } config.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, toIntExact(dfsTimeout.toMillis())); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 9035d5b5de72d..df671f177a574 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -2748,7 +2748,7 @@ private void checkSupportedStorageFormat(HiveStorageFormat storageFormat) } } - @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Error opening Hive split .*SequenceFile.*EOFException") + @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Error opening Hive split .*SequenceFile") public void testEmptySequenceFile() throws Exception { diff --git a/presto-hudi/pom.xml b/presto-hudi/pom.xml index 6d8efe8128f0b..6150e447458e9 100644 --- a/presto-hudi/pom.xml +++ b/presto-hudi/pom.xml @@ -117,9 +117,13 @@ presto-hive + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index 076d74de97fbd..a39b5924f532a 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -130,9 +130,13 @@ presto-plugin-toolkit + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-kudu/pom.xml b/presto-kudu/pom.xml index 35a324de4e534..952036da7f7da 100644 --- a/presto-kudu/pom.xml +++ b/presto-kudu/pom.xml @@ -29,9 +29,13 @@ + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 8099f839ddb99..6f61a5577da80 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -150,9 +150,14 @@ test - - com.facebook.presto.hadoop - hadoop-apache2 + + + + + + + com.github.imjalpreet + presto-hadoop-apache2 test diff --git a/presto-parquet/pom.xml b/presto-parquet/pom.xml index be54cee1733fd..0bdd75390e044 100644 --- a/presto-parquet/pom.xml +++ b/presto-parquet/pom.xml @@ -84,9 +84,13 @@ runtime + + + + - com.facebook.presto.hadoop - hadoop-apache2 + com.github.imjalpreet + presto-hadoop-apache2 diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index f145d33453ee4..36efe0777718c 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -65,9 +65,14 @@ - - com.facebook.presto.hadoop - hadoop-apache2 + + + + + + + com.github.imjalpreet + presto-hadoop-apache2 provided @@ -77,6 +82,12 @@ true + + org.xerial.snappy + snappy-java + runtime + + com.facebook.presto diff --git a/presto-spark-base/pom.xml b/presto-spark-base/pom.xml index b5d8d1bad05c1..0948ebeaa4470 100644 --- a/presto-spark-base/pom.xml +++ b/presto-spark-base/pom.xml @@ -206,9 +206,14 @@ test - - com.facebook.presto.hadoop - hadoop-apache2 + + + + + + + com.github.imjalpreet + presto-hadoop-apache2 test diff --git a/presto-spark-testing/pom.xml b/presto-spark-testing/pom.xml index c7ee30ddf707a..618b1b141126b 100644 --- a/presto-spark-testing/pom.xml +++ b/presto-spark-testing/pom.xml @@ -107,9 +107,14 @@ test - - com.facebook.presto.hadoop - hadoop-apache2 + + + + + + + com.github.imjalpreet + presto-hadoop-apache2 test From 192239360d0948ed4bd7b2b4edb3e1f0d35c0ec0 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Thu, 24 Apr 2025 14:30:21 +0530 Subject: [PATCH 02/10] Enable tests on Hadoop 3 (with Hive 3) image Open new ports for Hadoop 3 HDFS NameNode and DataNode --- presto-hive-hadoop2/bin/common.sh | 8 +- presto-hive-hadoop2/conf/docker-compose.yml | 1 + presto-hive-hadoop2/conf/files/tez-site.xml | 100 ++++++++++++++++++ .../hive/containers/HiveHadoopContainer.java | 6 +- .../hive/containers/HiveMinIODataLake.java | 4 +- presto-product-tests/bin/run_on_docker.sh | 2 +- .../conf/docker/common/compose-commons.sh | 4 +- .../conf/docker/common/standard.yml | 8 +- .../main/resources/tempto-configuration.yaml | 2 +- 9 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 presto-hive-hadoop2/conf/files/tez-site.xml diff --git a/presto-hive-hadoop2/bin/common.sh b/presto-hive-hadoop2/bin/common.sh index 1b13e40732a7f..9c3559e4f6c76 100755 --- a/presto-hive-hadoop2/bin/common.sh +++ b/presto-hive-hadoop2/bin/common.sh @@ -33,8 +33,8 @@ function check_hadoop() { HADOOP_MASTER_CONTAINER=$(hadoop_master_container) docker exec ${HADOOP_MASTER_CONTAINER} supervisorctl status hive-server2 | grep -iq running && \ docker exec ${HADOOP_MASTER_CONTAINER} supervisorctl status hive-metastore | grep -iq running && \ - docker exec ${HADOOP_MASTER_CONTAINER} netstat -lpn | grep -iq 0.0.0.0:10000 && - docker exec ${HADOOP_MASTER_CONTAINER} netstat -lpn | grep -iq 0.0.0.0:9083 + docker exec ${HADOOP_MASTER_CONTAINER} netstat -lpn | grep -iq :10000 && + docker exec ${HADOOP_MASTER_CONTAINER} netstat -lpn | grep -iq :9083 } function exec_in_hadoop_master_container() { @@ -67,8 +67,8 @@ function termination_handler(){ exit 130 } -export HADOOP_BASE_IMAGE="${HADOOP_BASE_IMAGE:-prestodb/hdp2.6-hive}" -export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-5} +export HADOOP_BASE_IMAGE="${HADOOP_BASE_IMAGE:-imjalpreet/hdp3.1-hive}" +export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-latest} SCRIPT_DIR="${BASH_SOURCE%/*}" INTEGRATION_TESTS_ROOT="${SCRIPT_DIR}/.." diff --git a/presto-hive-hadoop2/conf/docker-compose.yml b/presto-hive-hadoop2/conf/docker-compose.yml index c5cc888cce663..4984dce4a6c1f 100644 --- a/presto-hive-hadoop2/conf/docker-compose.yml +++ b/presto-hive-hadoop2/conf/docker-compose.yml @@ -20,4 +20,5 @@ services: - ./files/words:/usr/share/dict/words:ro - ./files/core-site.xml.s3-template:/etc/hadoop/conf/core-site.xml.s3-template:ro - ./files/hive-site.xml.s3-template:/etc/hive/conf/hive-site.xml.s3-template:ro + - ./files/tez-site.xml:/etc/tez/conf/tez-site.xml:ro - ./files:/tmp/files:ro \ No newline at end of file diff --git a/presto-hive-hadoop2/conf/files/tez-site.xml b/presto-hive-hadoop2/conf/files/tez-site.xml new file mode 100644 index 0000000000000..69e06e472e5c7 --- /dev/null +++ b/presto-hive-hadoop2/conf/files/tez-site.xml @@ -0,0 +1,100 @@ + + + + + + + tez.lib.uris.ignore + false + + + tez.lib.uris + file:///usr/hdp/current/tez-client/lib/tez.tar.gz + + + tez.am.mode.session + false + + + tez.am.acl.enabled + false + + + tez.am.log.level + WARN + + + tez.task.log.level + WARN + + + tez.runtime.io.sort.mb + 8 + + + tez.am.max.app.attempts + 1 + + + tez.am.task.max.failed.attempts + 1 + + + tez.shuffle-vertex-manager.min-src-fraction + 0.10 + + + tez.shuffle-vertex-manager.max-src-fraction + 1.00 + + + tez.am.launch.cmd-opts + -server -Djava.net.preferIPv4Stack=true -XX:+UseParallelGC -Dhadoop.metrics.log.level=WARN + + + tez.am.resource.memory.mb + 512 + + + tez.task.launch.cmd-opts + -server -Djava.net.preferIPv4Stack=true -XX:+UseParallelGC -Dhadoop.metrics.log.level=WARN + + + tez.task.resource.memory.mb + 512 + + + tez.task.resource.cpu.vcores + 1 + + + tez.runtime.sort.threads + 1 + + + tez.runtime.io.sort.factor + 100 + + + tez.runtime.shuffle.memory-to-memory.enable + false + + + tez.runtime.optimize.local.fetch + true + + + hive.tez.container.size + 2048 + + \ No newline at end of file diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveHadoopContainer.java b/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveHadoopContainer.java index 3cd03d55fe6dc..03c5cd4f51d8d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveHadoopContainer.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveHadoopContainer.java @@ -30,9 +30,9 @@ public class HiveHadoopContainer { private static final Logger log = Logger.get(HiveHadoopContainer.class); - private static final String IMAGE_VERSION = "10"; - public static final String DEFAULT_IMAGE = "prestodb/hdp2.6-hive:" + IMAGE_VERSION; - public static final String HIVE3_IMAGE = "prestodb/hive3.1-hive:" + IMAGE_VERSION; + private static final String IMAGE_VERSION = "latest"; + public static final String DEFAULT_IMAGE = "imjalpreet/hdp3.1-hive:" + IMAGE_VERSION; + public static final String HIVE3_IMAGE = "prestodb/hive3.1-hive:10"; public static final String HOST_NAME = "hadoop-master"; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java b/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java index a3fda22a07c6c..417847f078071 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java @@ -26,6 +26,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE3_IMAGE; @@ -67,7 +68,8 @@ public HiveMinIODataLake(String bucketName, Map hiveHadoopFilesT .putAll(hiveHadoopFilesToMount); String hadoopCoreSitePath = "/etc/hadoop/conf/core-site.xml"; - if (hiveHadoopImage == HIVE3_IMAGE) { + + if (Objects.equals(hiveHadoopImage, HIVE3_IMAGE)) { hadoopCoreSitePath = "/opt/hadoop/etc/hadoop/core-site.xml"; filesToMount.put("hive_s3_insert_overwrite/hive-site.xml", "/opt/hive/conf/hive-site.xml"); } diff --git a/presto-product-tests/bin/run_on_docker.sh b/presto-product-tests/bin/run_on_docker.sh index 80a99f5d72875..4e3588658c6b2 100755 --- a/presto-product-tests/bin/run_on_docker.sh +++ b/presto-product-tests/bin/run_on_docker.sh @@ -29,7 +29,7 @@ function hadoop_master_container(){ function check_hadoop() { HADOOP_MASTER_CONTAINER=$(hadoop_master_container) docker exec ${HADOOP_MASTER_CONTAINER} supervisorctl status hive-server2 | grep -iq running && \ - docker exec ${HADOOP_MASTER_CONTAINER} netstat -lpn | grep -iq 0.0.0.0:10000 + docker exec ${HADOOP_MASTER_CONTAINER} netstat -lpn | grep -iq :10000 } function run_in_application_runner_container() { diff --git a/presto-product-tests/conf/docker/common/compose-commons.sh b/presto-product-tests/conf/docker/common/compose-commons.sh index eae9f18ce9583..cb0520982a674 100644 --- a/presto-product-tests/conf/docker/common/compose-commons.sh +++ b/presto-product-tests/conf/docker/common/compose-commons.sh @@ -26,8 +26,8 @@ function export_canonical_path() { source "${BASH_SOURCE%/*}/../../../bin/locations.sh" -export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-10} -export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"prestodb/hdp2.6-hive"} +export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-latest} +export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"imjalpreet/hdp3.1-hive"} # This is the directory for the overriden JDK to use for starting Presto export OVERRIDE_JDK_DIR=${OVERRIDE_JDK_DIR:-"/dev/null"} diff --git a/presto-product-tests/conf/docker/common/standard.yml b/presto-product-tests/conf/docker/common/standard.yml index c15ad89190fa3..dc05320e76bd9 100644 --- a/presto-product-tests/conf/docker/common/standard.yml +++ b/presto-product-tests/conf/docker/common/standard.yml @@ -1,7 +1,7 @@ services: java-8-base: - image: 'prestodb/centos6-oj8:${DOCKER_IMAGES_VERSION}' + image: 'imjalpreet/centos7-oj8:${DOCKER_IMAGES_VERSION}' volumes: - ../..:/docker/volumes/conf - ../../target:/docker/volumes/logs @@ -19,10 +19,12 @@ services: - '8088:8088' - '9000:9000' - '9083:9083' + - '9864:9864' # DataNode Web UI since Hadoop 3 + - '9870:9870' # NameNode Web UI since Hadoop 3 - '10000:10000' - '19888:19888' - - '50070:50070' - - '50075:50075' + - '50070:50070' # NameNode Web UI prior to Hadoop 3 + - '50075:50075' # DataNode Web UI prior to Hadoop 3 volumes: - ../../../src/main/resources/avro:/docker/volumes/presto-product-tests/avro diff --git a/presto-product-tests/src/main/resources/tempto-configuration.yaml b/presto-product-tests/src/main/resources/tempto-configuration.yaml index 772dd45b6d2da..b0ab759943a32 100644 --- a/presto-product-tests/src/main/resources/tempto-configuration.yaml +++ b/presto-product-tests/src/main/resources/tempto-configuration.yaml @@ -2,7 +2,7 @@ hdfs: username: hdfs webhdfs: host: ${databases.hive.host} - port: 50070 + port: 9870 databases: default: From b02a9927c8f9927942eb5af35641c22f20f47a48 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Fri, 2 May 2025 00:55:20 +0530 Subject: [PATCH 03/10] Fix bucket/split count check in AbstractTestHiveClient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hive 1 and 2 use “bucketing v1”, which relies on a weaker hash function for floating-point values. However, HDP 2.6 images use MapReduce (MR) by default, which unconditionally creates one file per bucket. In contrast, Hive 3 uses an improved hash function under “bucketing v2”, but it defaults to using Tez, which does not generate files for empty buckets. As a result, even with 32 buckets configured, we may occasionally see fewer than 32 output files. --- .../java/com/facebook/presto/hive/AbstractTestHiveClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index df671f177a574..4a0279ace870c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -2423,9 +2423,9 @@ private static void assertBucketTableEvolutionResult(MaterializedResult result, private void assertTableIsBucketed(Transaction transaction, ConnectorTableHandle tableHandle) { - // the bucketed test tables should have exactly 32 splits + // the bucketed test tables should have ~32 splits List splits = getAllSplits(transaction, tableHandle, TupleDomain.all()); - assertEquals(splits.size(), 32); + assertThat(splits.size()).as("splits.size()").isBetween(31, 32); // verify all paths are unique Set paths = new HashSet<>(); From 2e6c365086d7061a8410e3264ea7559f57525493 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Wed, 7 May 2025 13:39:54 +0530 Subject: [PATCH 04/10] Align Tempto with default Hive user configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Presto’s default configuration interacts with Hive/Hadoop using the hive user (-DHADOOP_USER_NAME=hive). To ensure consistency and avoid permission issues, Tempto should also use the same hive user for data setup and as the default user for executing queries. Also, modify temporary storage location in hdfs where user hive has access --- .../main/resources/sql-tests/testcases/csv_table.sql | 4 ++-- .../testcases/csv_table_with_custom_parameters.sql | 4 ++-- .../src/main/resources/tempto-configuration.yaml | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table.sql b/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table.sql index b0b68d88619ae..c6250686c23a2 100644 --- a/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table.sql +++ b/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table.sql @@ -11,10 +11,10 @@ c_varchar|varchar||| --! name: show create csv table SHOW CREATE TABLE csv_table --! -CREATE TABLE hive.default.csv_table (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n external_location = 'hdfs://hadoop-master:9000/product-test/datasets/csv_table',\n format = 'CSV'\n) +CREATE TABLE hive.default.csv_table (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n external_location = 'hdfs://hadoop-master:9000/tmp/product-test/datasets/csv_table',\n format = 'CSV'\n) --! name: create table like DROP TABLE IF EXISTS like_csv_table; CREATE TABLE like_csv_table (LIKE csv_table INCLUDING PROPERTIES); SHOW CREATE TABLE like_csv_table --! -CREATE TABLE hive.default.like_csv_table (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n external_location = 'hdfs://hadoop-master:9000/product-test/datasets/csv_table',\n format = 'CSV'\n) +CREATE TABLE hive.default.like_csv_table (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n external_location = 'hdfs://hadoop-master:9000/tmp/product-test/datasets/csv_table',\n format = 'CSV'\n) diff --git a/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table_with_custom_parameters.sql b/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table_with_custom_parameters.sql index 17d74463672dd..2cb64fbd9500a 100644 --- a/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table_with_custom_parameters.sql +++ b/presto-product-tests/src/main/resources/sql-tests/testcases/csv_table_with_custom_parameters.sql @@ -11,10 +11,10 @@ c_varchar|varchar||| --! name: show create csv table SHOW CREATE TABLE csv_table_with_custom_parameters --! -CREATE TABLE hive.default.csv_table_with_custom_parameters (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n csv_escape = 'E',\n csv_quote = 'Q',\n csv_separator = 'S',\n external_location = 'hdfs://hadoop-master:9000/product-test/datasets/csv_table_with_custom_parameters',\n format = 'CSV'\n) +CREATE TABLE hive.default.csv_table_with_custom_parameters (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n csv_escape = 'E',\n csv_quote = 'Q',\n csv_separator = 'S',\n external_location = 'hdfs://hadoop-master:9000/tmp/product-test/datasets/csv_table_with_custom_parameters',\n format = 'CSV'\n) --! name: create table like DROP TABLE IF EXISTS like_csv_table; CREATE TABLE like_csv_table (LIKE csv_table_with_custom_parameters INCLUDING PROPERTIES); SHOW CREATE TABLE like_csv_table --! -CREATE TABLE hive.default.like_csv_table (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n csv_escape = 'E',\n csv_quote = 'Q',\n csv_separator = 'S',\n external_location = 'hdfs://hadoop-master:9000/product-test/datasets/csv_table_with_custom_parameters',\n format = 'CSV'\n) +CREATE TABLE hive.default.like_csv_table (\n "c_bigint" varchar,\n "c_varchar" varchar\n)\nWITH (\n csv_escape = 'E',\n csv_quote = 'Q',\n csv_separator = 'S',\n external_location = 'hdfs://hadoop-master:9000/tmp/product-test/datasets/csv_table_with_custom_parameters',\n format = 'CSV'\n) diff --git a/presto-product-tests/src/main/resources/tempto-configuration.yaml b/presto-product-tests/src/main/resources/tempto-configuration.yaml index b0ab759943a32..8a89a330dee15 100644 --- a/presto-product-tests/src/main/resources/tempto-configuration.yaml +++ b/presto-product-tests/src/main/resources/tempto-configuration.yaml @@ -1,5 +1,5 @@ hdfs: - username: hdfs + username: hive webhdfs: host: ${databases.hive.host} port: 9870 @@ -12,7 +12,7 @@ databases: host: hadoop-master jdbc_driver_class: org.apache.hive.jdbc.HiveDriver jdbc_url: jdbc:hive2://${databases.hive.host}:10000 - jdbc_user: hdfs + jdbc_user: hive jdbc_password: na jdbc_pooling: false schema: default @@ -32,7 +32,7 @@ databases: server_address: http://${databases.presto.host}:${databases.presto.port} jdbc_driver_class: ${PRESTO_JDBC_DRIVER_CLASS:-com.facebook.presto.jdbc.PrestoDriver} jdbc_url: jdbc:presto://${databases.presto.host}:${databases.presto.port}/hive/${databases.hive.schema} - jdbc_user: hdfs + jdbc_user: hive jdbc_password: "***empty***" jdbc_pooling: false @@ -40,7 +40,7 @@ databases: host: localhost jdbc_driver_class: ${databases.presto.jdbc_driver_class} jdbc_url: jdbc:presto://${databases.presto.host}:${databases.presto.port}/hive/tpcds - jdbc_user: hdfs + jdbc_user: hive jdbc_password: "***empty***" jdbc_pooling: false @@ -124,7 +124,7 @@ databases: tests: hdfs: - path: /product-test + path: /tmp/product-test assert: float_tolerance: 0.000001 From fbb2c8fdb30653d5ce8f0f8fe09b6c9bfed3e2f6 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Wed, 7 May 2025 18:54:42 +0530 Subject: [PATCH 05/10] Fix tests due to implementation changes in Hadoop 3.x and Hive 3.x Statistics are collected by default in Hive 3.x Added a new column list for the drop column test case, as Hive 3.x forbids certain coersions Enable metastore impersonation in singlenode-kerberos-hdfs-impersonation environment Drop table using same user that was used to create table. Otherwise, with Hive 3 and Hadoop 3 the table directory won't be deleted Co-authored-by: Shijin K --- .../presto/hive/AbstractTestHiveClient.java | 14 ++-- .../hive.properties | 1 + .../iceberg.properties | 1 + .../TestAllDatatypesFromHiveConnector.java | 6 +- .../tests/hive/TestExternalHiveTable.java | 9 --- .../hive/TestHiveBasicTableStatistics.java | 12 ---- .../tests/hive/TestHiveTableStatistics.java | 69 ++----------------- .../facebook/presto/tests/hive/TestRoles.java | 4 +- 8 files changed, 22 insertions(+), 94 deletions(-) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 4a0279ace870c..23aba773c0c51 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -385,6 +385,12 @@ public abstract class AbstractTestHiveClient .add(ColumnMetadata.builder().setName("t_row").setType(ROW_TYPE).build()) .build(); + private static final List CREATE_TABLE_COLUMNS_FOR_DROP = ImmutableList.builder() + .add(ColumnMetadata.builder().setName("id").setType(BIGINT).build()) + .add(ColumnMetadata.builder().setName("t_string").setType(createUnboundedVarcharType()).build()) + .add(ColumnMetadata.builder().setName("t_double").setType(DOUBLE).build()) + .build(); + private static final MaterializedResult CREATE_TABLE_DATA = MaterializedResult.resultBuilder(SESSION, BIGINT, createUnboundedVarcharType(), TINYINT, SMALLINT, INTEGER, BIGINT, REAL, DOUBLE, BOOLEAN, ARRAY_TYPE, MAP_TYPE, ROW_TYPE) .row(1L, "hello", (byte) 45, (short) 345, 234, 123L, -754.1985f, 43.5, true, ImmutableList.of("apple", "banana"), ImmutableMap.of("one", 1L, "two", 2L), ImmutableList.of("true", 1L, true)) @@ -3968,14 +3974,14 @@ public void testDropColumn() { SchemaTableName tableName = temporaryTable("test_drop_column"); try { - doCreateEmptyTable(tableName, ORC, CREATE_TABLE_COLUMNS); + doCreateEmptyTable(tableName, ORC, CREATE_TABLE_COLUMNS_FOR_DROP); ExtendedHiveMetastore metastoreClient = getMetastoreClient(); - metastoreClient.dropColumn(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), CREATE_TABLE_COLUMNS.get(0).getName()); + metastoreClient.dropColumn(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), CREATE_TABLE_COLUMNS_FOR_DROP.get(0).getName()); Optional table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName()); assertTrue(table.isPresent()); List columns = table.get().getDataColumns(); - assertEquals(columns.get(0).getName(), CREATE_TABLE_COLUMNS.get(1).getName()); - assertFalse(columns.stream().map(Column::getName).anyMatch(colName -> colName.equals(CREATE_TABLE_COLUMNS.get(0).getName()))); + assertEquals(columns.get(0).getName(), CREATE_TABLE_COLUMNS_FOR_DROP.get(1).getName()); + assertFalse(columns.stream().map(Column::getName).anyMatch(colName -> colName.equals(CREATE_TABLE_COLUMNS_FOR_DROP.get(0).getName()))); } finally { dropTable(tableName); diff --git a/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/hive.properties b/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/hive.properties index d0bffe9d7bc56..fad6876b4fb1c 100644 --- a/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/hive.properties +++ b/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/hive.properties @@ -15,6 +15,7 @@ hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab +hive.metastore-impersonation-enabled=true hive.hdfs.authentication.type=KERBEROS hive.hdfs.impersonation.enabled=true diff --git a/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/iceberg.properties b/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/iceberg.properties index b021d6f54105c..f8148e80acb05 100644 --- a/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/iceberg.properties +++ b/presto-product-tests/conf/presto/etc/environment-specific-catalogs/singlenode-kerberos-hdfs-impersonation/iceberg.properties @@ -6,6 +6,7 @@ hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab +hive.metastore-impersonation-enabled=true hive.hdfs.authentication.type=KERBEROS hive.hdfs.impersonation.enabled=true diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestAllDatatypesFromHiveConnector.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestAllDatatypesFromHiveConnector.java index f3455abadc366..2bcf2128790eb 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestAllDatatypesFromHiveConnector.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestAllDatatypesFromHiveConnector.java @@ -278,7 +278,8 @@ public void testSelectAllDatatypesAvro() 234.567, new BigDecimal("346"), new BigDecimal("345.67800"), - Timestamp.valueOf(LocalDateTime.of(2015, 5, 10, 12, 15, 35, 123_000_000)), + // TODO: requires https://issues.apache.org/jira/browse/HIVE-21002 + Timestamp.valueOf(LocalDateTime.of(2015, 5, 10, 18, 0, 35, 123_000_000)), Date.valueOf("2015-05-10"), "ala ma kota", "ala ma kot", @@ -397,7 +398,8 @@ public void testSelectAllDatatypesParquetFile() 234.567, new BigDecimal("346"), new BigDecimal("345.67800"), - Timestamp.valueOf(LocalDateTime.of(2015, 5, 10, 12, 15, 35, 123_000_000)), + // TODO: requires https://issues.apache.org/jira/browse/HIVE-21002 + Timestamp.valueOf(LocalDateTime.of(2015, 5, 10, 18, 0, 35, 123_000_000)), "ala ma kota", "ala ma kot", "ala ma ", diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestExternalHiveTable.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestExternalHiveTable.java index 5a880eb631522..20cb83fe13e92 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestExternalHiveTable.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestExternalHiveTable.java @@ -53,15 +53,6 @@ public void testShowStatisticsForExternalTable() onHive().executeQuery("CREATE EXTERNAL TABLE " + EXTERNAL_TABLE_NAME + " LIKE " + nation.getNameInDatabase() + " LOCATION '/tmp/" + EXTERNAL_TABLE_NAME + "_" + nation.getNameInDatabase() + "'"); insertNationPartition(nation, 1); - onHive().executeQuery("ANALYZE TABLE " + EXTERNAL_TABLE_NAME + " PARTITION (p_regionkey) COMPUTE STATISTICS"); - assertThat(query("SHOW STATS FOR " + EXTERNAL_TABLE_NAME)).containsOnly( - row("p_nationkey", null, null, null, null, null, null, null), - row("p_name", null, null, null, null, null, null, null), - row("p_comment", null, null, null, null, null, null, null), - row("p_regionkey", null, 1.0, 0.0, null, "1", "1", null), - row(null, null, null, null, 5.0, null, null, null)); - - onHive().executeQuery("ANALYZE TABLE " + EXTERNAL_TABLE_NAME + " PARTITION (p_regionkey) COMPUTE STATISTICS FOR COLUMNS"); assertThat(query("SHOW STATS FOR " + EXTERNAL_TABLE_NAME)).containsOnly( row("p_nationkey", null, 5.0, 0.0, null, "1", "24", null), row("p_name", 38.0, 5.0, 0.0, null, null, null, null), diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveBasicTableStatistics.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveBasicTableStatistics.java index e9f033422e15b..1e595d6b2a637 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveBasicTableStatistics.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveBasicTableStatistics.java @@ -128,9 +128,6 @@ public void testCreatePartitioned() "WHERE n_nationkey <> 23", tableName)); try { - BasicStatistics tableStatistics = getBasicStatisticsForTable(onHive(), tableName); - assertThatStatisticsAreNotPresent(tableStatistics); - BasicStatistics firstPartitionStatistics = getBasicStatisticsForPartition(onHive(), tableName, "n_regionkey=1"); assertThatStatisticsAreNonZero(firstPartitionStatistics); assertThat(firstPartitionStatistics.getNumRows().getAsLong()).isEqualTo(5); @@ -163,9 +160,6 @@ public void testAnalyzePartitioned() "WHERE n_regionkey = 1", tableName)); try { - BasicStatistics tableStatistics = getBasicStatisticsForTable(onHive(), tableName); - assertThatStatisticsAreNotPresent(tableStatistics); - BasicStatistics partitionStatisticsBefore = getBasicStatisticsForPartition(onHive(), tableName, "n_regionkey=1"); assertThatStatisticsArePresent(partitionStatisticsBefore); @@ -236,9 +230,6 @@ public void testInsertPartitioned() ") ", tableName)); try { - BasicStatistics tableStatisticsAfterCreate = getBasicStatisticsForTable(onHive(), tableName); - assertThatStatisticsAreNotPresent(tableStatisticsAfterCreate); - insertNationData(onPresto(), tableName); BasicStatistics partitionStatisticsFirstInsert = getBasicStatisticsForPartition(onHive(), tableName, "n_regionkey=3"); @@ -310,9 +301,6 @@ public void testInsertBucketedPartitioned() "WHERE n_regionkey = 1", tableName)); try { - BasicStatistics tableStatistics = getBasicStatisticsForTable(onHive(), tableName); - assertThatStatisticsAreNotPresent(tableStatistics); - BasicStatistics firstPartitionStatistics = getBasicStatisticsForPartition(onHive(), tableName, "n_regionkey=1"); assertThatStatisticsAreNonZero(firstPartitionStatistics); assertThat(firstPartitionStatistics.getNumRows().getAsLong()).isEqualTo(5); diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveTableStatistics.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveTableStatistics.java index 59071737e25cc..073735f7f826b 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveTableStatistics.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveTableStatistics.java @@ -201,8 +201,8 @@ public void testStatisticsForUnpartitionedTable() onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS FOR COLUMNS"); assertThat(query(showStatsWholeTable)).containsOnly( - row("n_nationkey", null, 19.0, 0.0, null, "0", "24", null), - row("n_name", 177.0, 24.0, 0.0, null, null, null, null), + row("n_nationkey", null, 25.0, 0.0, null, "0", "24", null), + row("n_name", 177.0, 25.0, 0.0, null, null, null, null), row("n_regionkey", null, 5.0, 0.0, null, "0", "4", null), row("n_comment", 1857.0, 25.0, 0.0, null, null, null, null), row(null, null, null, null, 25.0, null, null, null)); @@ -328,7 +328,7 @@ public void testStatisticsForTablePartitionedByBigint() row(null, null, null, null, 5.0, null, null, null)); assertThat(query(showStatsPartitionTwo)).containsOnly( - row("p_nationkey", null, 4.0, 0.0, null, "8", "21", null), + row("p_nationkey", null, 5.0, 0.0, null, "8", "21", null), row("p_name", 31.0, 5.0, 0.0, null, null, null, null), row("p_regionkey", null, 1.0, 0.0, null, "2", "2", null), row("p_comment", 351.0, 5.0, 0.0, null, null, null, null), @@ -455,7 +455,7 @@ public void testStatisticsForTablePartitionedByVarchar() row(null, null, null, null, 5.0, null, null, null)); assertThat(query(showStatsPartitionTwo)).containsOnly( - row("p_nationkey", null, 4.0, 0.0, null, "8", "21", null), + row("p_nationkey", null, 5.0, 0.0, null, "8", "21", null), row("p_name", 31.0, 5.0, 0.0, null, null, null, null), row("p_regionkey", 20.0, 1.0, 0.0, null, null, null, null), row("p_comment", 351.0, 5.0, 0.0, null, null, null, null), @@ -565,28 +565,6 @@ public void testStatisticsForAllDataTypesOnlyNulls() String tableNameInDatabase = mutableTablesState().get(EMPTY_ALL_TYPES_TABLE_NAME).getNameInDatabase(); onHive().executeQuery("INSERT INTO TABLE " + tableNameInDatabase + " VALUES(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null)"); - onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS"); - - assertThat(query("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( - row("c_tinyint", null, null, null, null, null, null, null), - row("c_smallint", null, null, null, null, null, null, null), - row("c_int", null, null, null, null, null, null, null), - row("c_bigint", null, null, null, null, null, null, null), - row("c_float", null, null, null, null, null, null, null), - row("c_double", null, null, null, null, null, null, null), - row("c_decimal", null, null, null, null, null, null, null), - row("c_decimal_w_params", null, null, null, null, null, null, null), - row("c_timestamp", null, null, null, null, null, null, null), - row("c_date", null, null, null, null, null, null, null), - row("c_string", null, null, null, null, null, null, null), - row("c_varchar", null, null, null, null, null, null, null), - row("c_char", null, null, null, null, null, null, null), - row("c_boolean", null, null, null, null, null, null, null), - row("c_binary", null, null, null, null, null, null, null), - row(null, null, null, null, 1.0, null, null, null)); - - onHive().executeQuery("ANALYZE TABLE " + tableNameInDatabase + " COMPUTE STATISTICS FOR COLUMNS"); - assertThat(query("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( row("c_tinyint", null, 0.0, 1.0, null, null, null, null), row("c_smallint", null, 0.0, 1.0, null, null, null, null), @@ -615,19 +593,6 @@ public void testStatisticsForSkewedTable() onHive().executeQuery("CREATE TABLE " + tableName + " (c_string STRING, c_int INT) SKEWED BY (c_string) ON ('c1')"); onHive().executeQuery("INSERT INTO TABLE " + tableName + " VALUES ('c1', 1), ('c1', 2)"); - assertThat(query("SHOW STATS FOR " + tableName)).containsOnly( - row("c_string", null, null, null, null, null, null, null), - row("c_int", null, null, null, null, null, null, null), - row(null, null, null, null, 2.0, null, null, null)); - - onHive().executeQuery("ANALYZE TABLE " + tableName + " COMPUTE STATISTICS"); - - assertThat(query("SHOW STATS FOR " + tableName)).containsOnly( - row("c_string", null, null, null, null, null, null, null), - row("c_int", null, null, null, null, null, null, null), - row(null, null, null, null, 2.0, null, null, null)); - - onHive().executeQuery("ANALYZE TABLE " + tableName + " COMPUTE STATISTICS FOR COLUMNS"); assertThat(query("SHOW STATS FOR " + tableName)).containsOnly( row("c_string", 4.0, 1.0, 0.0, null, null, null, null), row("c_int", null, 2.0, 0.0, null, "1", "2", null), @@ -643,12 +608,6 @@ public void testAnalyzesForSkewedTable() onHive().executeQuery("CREATE TABLE " + tableName + " (c_string STRING, c_int INT) SKEWED BY (c_string) ON ('c1')"); onHive().executeQuery("INSERT INTO TABLE " + tableName + " VALUES ('c1', 1), ('c1', 2)"); - assertThat(query("SHOW STATS FOR " + tableName)).containsOnly( - row("c_string", null, null, null, null, null, null, null), - row("c_int", null, null, null, null, null, null, null), - row(null, null, null, null, 2.0, null, null, null)); - - assertThat(query("ANALYZE " + tableName)).containsExactly(row(2)); assertThat(query("SHOW STATS FOR " + tableName)).containsOnly( row("c_string", 4.0, 1.0, 0.0, null, null, null, null), row("c_int", null, 2.0, 0.0, null, "1", "2", null), @@ -977,26 +936,6 @@ public void testAnalyzeForAllDataTypesOnlyNulls() // insert from hive to prevent Presto collecting statistics on insert onHive().executeQuery("INSERT INTO TABLE " + tableNameInDatabase + " VALUES(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null)"); - assertThat(query("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( - row("c_tinyint", null, null, null, null, null, null, null), - row("c_smallint", null, null, null, null, null, null, null), - row("c_int", null, null, null, null, null, null, null), - row("c_bigint", null, null, null, null, null, null, null), - row("c_float", null, null, null, null, null, null, null), - row("c_double", null, null, null, null, null, null, null), - row("c_decimal", null, null, null, null, null, null, null), - row("c_decimal_w_params", null, null, null, null, null, null, null), - row("c_timestamp", null, null, null, null, null, null, null), - row("c_date", null, null, null, null, null, null, null), - row("c_string", null, null, null, null, null, null, null), - row("c_varchar", null, null, null, null, null, null, null), - row("c_char", null, null, null, null, null, null, null), - row("c_boolean", null, null, null, null, null, null, null), - row("c_binary", null, null, null, null, null, null, null), - row(null, null, null, null, 1.0, null, null, null)); - - assertThat(query("ANALYZE " + tableNameInDatabase)).containsExactly(row(1)); - assertThat(query("SHOW STATS FOR " + tableNameInDatabase)).containsOnly( row("c_tinyint", null, 0.0, 1.0, null, null, null, null), row("c_smallint", null, 0.0, 1.0, null, null, null, null), diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java index ccf25f60f5b6d..27dbdef88a324 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java @@ -663,7 +663,7 @@ public void testAdminCanShowAllGrants() } finally { onPrestoAlice().executeQuery("DROP TABLE hive.default.test_table_alice"); - onPrestoAlice().executeQuery("DROP TABLE hive.default.test_table_bob"); + onPrestoBob().executeQuery("DROP TABLE hive.default.test_table_bob"); onPresto().executeQuery("REVOKE admin FROM alice"); } } @@ -703,7 +703,7 @@ public void testAdminCanShowGrantsOnlyFromCurrentSchema() row("alice", "USER", "alice", "USER", "hive", "test", "test_table_bob", "INSERT", "YES", null))); } finally { - onPresto().executeQuery("DROP TABLE hive.default.test_table_bob"); + onPrestoBob().executeQuery("DROP TABLE hive.default.test_table_bob"); onPrestoAlice().executeQuery("DROP TABLE hive.test.test_table_bob"); onPresto().executeQuery("DROP SCHEMA hive.test"); onPresto().executeQuery("REVOKE admin FROM alice"); From 48f6b5acdd16d5d64057baf71ca555bcb686ec00 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Sun, 11 May 2025 08:17:54 +0530 Subject: [PATCH 06/10] Workaround for HDFS permission issue On Hadoop 3.x, the new schema location on HDFS is owned by the hive user and has rwxr-xr-x permissions. As a result, when HDFS impersonation is enabled, the alice user is unable to create a new table within that schema. --- .../src/main/java/com/facebook/presto/tests/hive/TestRoles.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java index 27dbdef88a324..7e00b84631cc6 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestRoles.java @@ -676,7 +676,7 @@ public void testAdminCanShowGrantsOnlyFromCurrentSchema() onPresto().executeQuery("CREATE SCHEMA hive.test"); onPresto().executeQuery("GRANT admin TO alice"); onPrestoAlice().executeQuery("SET ROLE ADMIN"); - onPrestoAlice().executeQuery("CREATE TABLE hive.test.test_table_bob (foo BIGINT)"); + onPrestoAlice().executeQuery("CREATE TABLE hive.test.test_table_bob (foo BIGINT) with (external_location='/tmp')"); QueryAssert.assertThat(onPrestoAlice().executeQuery("SHOW GRANTS ON hive.default.test_table_bob")) .containsOnly(ImmutableList.of( From ca07c6e012c01cc576a9a70010e7b5c32adaeb71 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Sun, 11 May 2025 10:08:07 +0530 Subject: [PATCH 07/10] Force bucketing v1 in hive-tests Hive 3 uses bucketing v2 by default and Presto does not support it yet --- presto-hive/src/test/sql/create-test.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-hive/src/test/sql/create-test.sql b/presto-hive/src/test/sql/create-test.sql index 96bf9d635d3b6..dc4f04bd45fc1 100644 --- a/presto-hive/src/test/sql/create-test.sql +++ b/presto-hive/src/test/sql/create-test.sql @@ -67,7 +67,7 @@ COMMENT 'Presto test bucketed table' PARTITIONED BY (ds STRING) CLUSTERED BY (t_string, t_int) INTO 32 BUCKETS STORED AS RCFILE -TBLPROPERTIES ('RETENTION'='-1') +TBLPROPERTIES ('bucketing_version'='1') ; CREATE TABLE presto_test_bucketed_by_bigint_boolean ( @@ -84,7 +84,7 @@ COMMENT 'Presto test bucketed table' PARTITIONED BY (ds STRING) CLUSTERED BY (t_bigint, t_boolean) INTO 32 BUCKETS STORED AS RCFILE -TBLPROPERTIES ('RETENTION'='-1') +TBLPROPERTIES ('bucketing_version'='1') ; CREATE TABLE presto_test_bucketed_by_double_float ( From e0009df841e34ce39a52c90d3a8d854cd0bd6df6 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Fri, 9 May 2025 16:09:06 +0530 Subject: [PATCH 08/10] Temporarily Disable Product Tests Specific 2.1 - LDAP tests Update openldap image to imjalpreet/centos7-oj8-openldap:latest --- .../workflows/product-tests-specific-environment.yml | 11 ++++++----- .../conf/docker/singlenode-ldap/docker-compose.yml | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index 27218a7b2c28e..7e814b74e8581 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -131,11 +131,12 @@ jobs: run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - - name: Product Tests Specific 2.1 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc +# Temporarily disable till new docker image "centos7-oj8-openldap" is ready +# - name: Product Tests Specific 2.1 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc - name: Product Tests Specific 2.2 if: needs.changes.outputs.codechange == 'true' env: diff --git a/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml b/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml index b170d7d873dcb..50bfc9bee908d 100644 --- a/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml +++ b/presto-product-tests/conf/docker/singlenode-ldap/docker-compose.yml @@ -1,7 +1,7 @@ services: presto-master: - image: 'prestodb/centos6-oj8-openldap:${DOCKER_IMAGES_VERSION}' + image: 'imjalpreet/centos7-oj8-openldap:latest' command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh singlenode-ldap run extra_hosts: - "${LDAP_SERVER_HOST}:${LDAP_SERVER_IP}" @@ -12,10 +12,10 @@ services: - ${OVERRIDE_JDK_DIR}:/docker/volumes/overridejdk application-runner: - image: 'prestodb/centos6-oj8-openldap:${DOCKER_IMAGES_VERSION}' + image: 'imjalpreet/centos7-oj8-openldap:latest' environment: - TEMPTO_PROFILE_CONFIG_FILE=/docker/volumes/conf/tempto/tempto-configuration-for-docker-ldap.yaml - CLI_ARGUMENTS=--server https://presto-master:8443 --keystore-path /etc/openldap/certs/coordinator.jks --keystore-password testldap ldapserver: - image: 'prestodb/centos6-oj8-openldap:${DOCKER_IMAGES_VERSION}' + image: 'imjalpreet/centos7-oj8-openldap:latest' From c213adc029901473b16378bb93ed5e37fb1585f8 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Mon, 19 May 2025 22:51:35 +0530 Subject: [PATCH 09/10] Test singlenode-ldap tests --- .github/workflows/arrow-flight-tests.yml | 7 +- .github/workflows/hive-tests.yml | 3 +- .github/workflows/kudu.yml | 3 +- .../prestocpp-linux-build-and-unit-test.yml | 20 +- .github/workflows/prestocpp-linux-build.yml | 50 ++--- .github/workflows/prestocpp-macos-build.yml | 8 +- .../product-tests-basic-environment.yml | 3 +- .../product-tests-specific-environment.yml | 200 +++++++++--------- .github/workflows/singlestore-tests.yml | 3 +- .github/workflows/spark-integration.yml | 3 +- .github/workflows/test-other-modules.yml | 3 +- .github/workflows/tests.yml | 3 +- 12 files changed, 157 insertions(+), 149 deletions(-) diff --git a/.github/workflows/arrow-flight-tests.yml b/.github/workflows/arrow-flight-tests.yml index 1d8c1064bb0e9..114e449887f2b 100644 --- a/.github/workflows/arrow-flight-tests.yml +++ b/.github/workflows/arrow-flight-tests.yml @@ -1,9 +1,10 @@ name: arrow flight tests on: - pull_request: - paths-ignore: - - 'presto-docs/**' + workflow_dispatch: +# pull_request: +# paths-ignore: +# - 'presto-docs/**' env: CONTINUOUS_INTEGRATION: true diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index 9762ac21ef8fb..0353f9dd9b56c 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -1,7 +1,8 @@ name: hive tests on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml index 86090a5e5b83c..c8dd60c1c4fd4 100644 --- a/.github/workflows/kudu.yml +++ b/.github/workflows/kudu.yml @@ -1,7 +1,8 @@ name: kudu on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml index a7de5927f5899..771b4c62cdfb5 100644 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -2,16 +2,16 @@ name: prestocpp-linux-build-and-unit-test on: workflow_dispatch: - pull_request: - paths: - - 'presto-native-execution/**' - - 'presto-native-sidecar-plugin/**' - - '.github/workflows/prestocpp-linux-build-and-unit-test.yml' - push: - branches: - - master - paths-ignore: - - 'presto-docs/**' +# pull_request: +# paths: +# - 'presto-native-execution/**' +# - 'presto-native-sidecar-plugin/**' +# - '.github/workflows/prestocpp-linux-build-and-unit-test.yml' +# push: +# branches: +# - master +# paths-ignore: +# - 'presto-docs/**' jobs: prestocpp-linux-build-for-test: diff --git a/.github/workflows/prestocpp-linux-build.yml b/.github/workflows/prestocpp-linux-build.yml index 29b92be455364..13fd819c068fd 100644 --- a/.github/workflows/prestocpp-linux-build.yml +++ b/.github/workflows/prestocpp-linux-build.yml @@ -2,31 +2,31 @@ name: prestocpp-linux-build on: workflow_dispatch: - pull_request: - paths: - - 'presto-native-execution/**' - - '.github/workflows/prestocpp-linux-build.yml' - # Build also changes to files that can change the protocol and are referenced in the protocol yaml: - # protocol_core - - 'presto-spi/src/main/java/com/facebook/presto/spi/**' - - 'presto-common/src/main/java/com/facebook/presto/**' - - 'presto-main/src/main/java/com/facebook/presto/**' - - 'presto-client/src/main/java/com/facebook/presto/client/**' - - 'presto-spark-base/src/main/java/com/facebook/presto/spark/execution/**' - - 'presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/**' - - 'presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/**' - - 'presto-hdfs-core/src/main/java/com/facebook/presto/hive/**' - - 'presto-verifier/src/main/java/com/facebook/presto/verifier/framework/**' - # arrow-flight - - 'presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/**' - # hive - - 'presto-hive-metastore/src/main/java/com/facebook/presto/hive/**' - - 'presto-hive-common/src/main/java/com/facebook/presto/hive/**' - - 'presto-hive/src/main/java/com/facebook/presto/hive/**' - # iceberg - - 'presto-iceberg/src/main/java/com/facebook/presto/iceberg/**' - # tpch - - 'presto-tpch/src/main/java/com/facebook/presto/tpch/**' +# pull_request: +# paths: +# - 'presto-native-execution/**' +# - '.github/workflows/prestocpp-linux-build.yml' +# # Build also changes to files that can change the protocol and are referenced in the protocol yaml: +# # protocol_core +# - 'presto-spi/src/main/java/com/facebook/presto/spi/**' +# - 'presto-common/src/main/java/com/facebook/presto/**' +# - 'presto-main/src/main/java/com/facebook/presto/**' +# - 'presto-client/src/main/java/com/facebook/presto/client/**' +# - 'presto-spark-base/src/main/java/com/facebook/presto/spark/execution/**' +# - 'presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/**' +# - 'presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/**' +# - 'presto-hdfs-core/src/main/java/com/facebook/presto/hive/**' +# - 'presto-verifier/src/main/java/com/facebook/presto/verifier/framework/**' +# # arrow-flight +# - 'presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/**' +# # hive +# - 'presto-hive-metastore/src/main/java/com/facebook/presto/hive/**' +# - 'presto-hive-common/src/main/java/com/facebook/presto/hive/**' +# - 'presto-hive/src/main/java/com/facebook/presto/hive/**' +# # iceberg +# - 'presto-iceberg/src/main/java/com/facebook/presto/iceberg/**' +# # tpch +# - 'presto-tpch/src/main/java/com/facebook/presto/tpch/**' jobs: prestocpp-linux-build-engine: diff --git a/.github/workflows/prestocpp-macos-build.yml b/.github/workflows/prestocpp-macos-build.yml index 85a0b5b0351b9..97ab1d8581145 100644 --- a/.github/workflows/prestocpp-macos-build.yml +++ b/.github/workflows/prestocpp-macos-build.yml @@ -2,10 +2,10 @@ name: prestocpp-macos-build on: workflow_dispatch: - pull_request: - paths: - - 'presto-native-execution/**' - - '.github/workflows/prestocpp-macos-build.yml' +# pull_request: +# paths: +# - 'presto-native-execution/**' +# - '.github/workflows/prestocpp-macos-build.yml' jobs: prestocpp-macos-build-engine: diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml index fbbfa2265ea64..d86e13e45c7c2 100644 --- a/.github/workflows/product-tests-basic-environment.yml +++ b/.github/workflows/product-tests-basic-environment.yml @@ -1,7 +1,8 @@ name: product tests (basic) on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index 7e814b74e8581..9dbf9927650cb 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -28,71 +28,71 @@ jobs: filters: | codechange: - '!presto-docs/**' - product-tests-specific-environment1: - strategy: - fail-fast: false - matrix: - java: [ 8.0.442, 17.0.13 ] - runs-on: ubuntu-latest - needs: changes - timeout-minutes: 60 - concurrency: - group: ${{ github.workflow }}-product-tests-specific-environment1-${{ github.event.pull_request.number }}-${{ matrix.java }} - cancel-in-progress: true - steps: - - name: Free Disk Space - if: needs.changes.outputs.codechange == 'true' - run: | - df -h - sudo apt-get clean - rm -rf /opt/hostedtoolcache - df -h - - uses: actions/checkout@v4 - if: needs.changes.outputs.codechange == 'true' - with: - show-progress: false - - uses: actions/setup-java@v4 - if: needs.changes.outputs.codechange == 'true' - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - cache: 'maven' - - name: Download nodejs to maven cache - if: needs.changes.outputs.codechange == 'true' - run: .github/bin/download_nodejs - - name: Maven install - if: needs.changes.outputs.codechange == 'true' - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' - - name: Product Tests Specific 1.1 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode -g hdfs_no_impersonation,avro - - name: Product Tests Specific 1.2 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-no-impersonation -g hdfs_no_impersonation - # temporarily disable this flaky run. see issue #20388 for details - # - name: Product Tests Specific 1.3 - # run: presto-product-tests/bin/run_on_docker.sh singlenode-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation - - name: Product Tests Specific 1.4 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header - - name: Product Tests Specific 1.5 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-cross-realm -g storage_formats,cli,hdfs_impersonation - - name: Product Tests Specific 1.6 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh multinode-tls-kerberos -g cli,group-by,join,tls +# product-tests-specific-environment1: +# strategy: +# fail-fast: false +# matrix: +# java: [ 8.0.442, 17.0.13 ] +# runs-on: ubuntu-latest +# needs: changes +# timeout-minutes: 60 +# concurrency: +# group: ${{ github.workflow }}-product-tests-specific-environment1-${{ github.event.pull_request.number }}-${{ matrix.java }} +# cancel-in-progress: true +# steps: +# - name: Free Disk Space +# if: needs.changes.outputs.codechange == 'true' +# run: | +# df -h +# sudo apt-get clean +# rm -rf /opt/hostedtoolcache +# df -h +# - uses: actions/checkout@v4 +# if: needs.changes.outputs.codechange == 'true' +# with: +# show-progress: false +# - uses: actions/setup-java@v4 +# if: needs.changes.outputs.codechange == 'true' +# with: +# distribution: 'temurin' +# java-version: ${{ matrix.java }} +# cache: 'maven' +# - name: Download nodejs to maven cache +# if: needs.changes.outputs.codechange == 'true' +# run: .github/bin/download_nodejs +# - name: Maven install +# if: needs.changes.outputs.codechange == 'true' +# run: | +# export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" +# ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' +# - name: Product Tests Specific 1.1 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode -g hdfs_no_impersonation,avro +# - name: Product Tests Specific 1.2 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-no-impersonation -g hdfs_no_impersonation +# # temporarily disable this flaky run. see issue #20388 for details +# # - name: Product Tests Specific 1.3 +# # run: presto-product-tests/bin/run_on_docker.sh singlenode-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation +# - name: Product Tests Specific 1.4 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header +# - name: Product Tests Specific 1.5 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-cross-realm -g storage_formats,cli,hdfs_impersonation +# - name: Product Tests Specific 1.6 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh multinode-tls-kerberos -g cli,group-by,join,tls product-tests-specific-environment2: strategy: @@ -132,41 +132,41 @@ jobs: export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl '!presto-docs,!presto-spark-package,!presto-spark-launcher,!presto-spark-testing,!presto-test-coverage' # Temporarily disable till new docker image "centos7-oj8-openldap" is ready -# - name: Product Tests Specific 2.1 -# if: needs.changes.outputs.codechange == 'true' -# env: -# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} -# run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc - - name: Product Tests Specific 2.2 + - name: Product Tests Specific 2.1 if: needs.changes.outputs.codechange == 'true' env: OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh multinode-tls -g smoke,cli,group-by,join,tls - - name: Product Tests Specific 2.3 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-mysql -g mysql_connector,mysql - - name: Product Tests Specific 2.4 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-postgresql -g postgresql_connector - - name: Product Tests Specific 2.5 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-cassandra -g cassandra - # temporarily disable this flaky run. see issue #20388 for details - # - name: Product Tests Specific 2.6 - # run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-with-wire-encryption -g storage_formats,cli,hdfs_impersonation,authorization - - name: Product Tests Specific 2.7 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-kafka -g kafka - - name: Product Tests Specific 2.8 - if: needs.changes.outputs.codechange == 'true' - env: - OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} - run: presto-product-tests/bin/run_on_docker.sh singlenode-sqlserver -g sqlserver + run: presto-product-tests/bin/run_on_docker.sh singlenode-ldap -g ldap -x simba_jdbc +# - name: Product Tests Specific 2.2 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh multinode-tls -g smoke,cli,group-by,join,tls +# - name: Product Tests Specific 2.3 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-mysql -g mysql_connector,mysql +# - name: Product Tests Specific 2.4 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-postgresql -g postgresql_connector +# - name: Product Tests Specific 2.5 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-cassandra -g cassandra +# # temporarily disable this flaky run. see issue #20388 for details +# # - name: Product Tests Specific 2.6 +# # run: presto-product-tests/bin/run_on_docker.sh singlenode-kerberos-hdfs-impersonation-with-wire-encryption -g storage_formats,cli,hdfs_impersonation,authorization +# - name: Product Tests Specific 2.7 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-kafka -g kafka +# - name: Product Tests Specific 2.8 +# if: needs.changes.outputs.codechange == 'true' +# env: +# OVERRIDE_JDK_DIR: ${{ env.JAVA_HOME }} +# run: presto-product-tests/bin/run_on_docker.sh singlenode-sqlserver -g sqlserver diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml index c3b06fe9afadb..75da123cf88b7 100644 --- a/.github/workflows/singlestore-tests.yml +++ b/.github/workflows/singlestore-tests.yml @@ -1,7 +1,8 @@ name: singlestore tests on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml index 08bcff6521791..55d5db93535e5 100644 --- a/.github/workflows/spark-integration.yml +++ b/.github/workflows/spark-integration.yml @@ -1,7 +1,8 @@ name: spark integration on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index 1b7cac05e4c8a..6f90614755d10 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -1,7 +1,8 @@ name: test other modules on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e4a9fb3f96385..61f73693a6f4a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,8 @@ name: test on: - pull_request: + workflow_dispatch: +# pull_request: env: # An envar that signals to tests we are executing in the CI environment From af0fd4e425ab002d251b4643a0266341df437b34 Mon Sep 17 00:00:00 2001 From: "Jalpreet Singh Nanda (:imjalpreet)" Date: Wed, 21 May 2025 19:13:28 +0530 Subject: [PATCH 10/10] Fix LDAP tests --- .../java/com/facebook/presto/tests/cli/PrestoLdapCliTests.java | 2 +- .../com/facebook/presto/tests/jdbc/LdapPrestoJdbcTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoLdapCliTests.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoLdapCliTests.java index 6edc5202c0cfe..a114f3acca74c 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoLdapCliTests.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoLdapCliTests.java @@ -228,7 +228,7 @@ public void shouldFailForIncorrectTrustStore() ldapTruststorePassword = "wrong_password"; launchPrestoCliWithServerArgument("--execute", "select * from hive.default.nation;"); assertThat(trimLines(presto.readRemainingErrorLines())).anySatisfy(line -> - assertThat(line).contains("Keystore was tampered with, or password was incorrect")); + assertThat(line).contains("Error setting up SSL: keystore password was incorrect")); skipAfterTestWithContext(); } diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/LdapPrestoJdbcTests.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/LdapPrestoJdbcTests.java index b3320a9ec7bc0..e6c86b93b99e6 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/LdapPrestoJdbcTests.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/LdapPrestoJdbcTests.java @@ -119,7 +119,7 @@ public void shouldFailForIncorrectTrustStore() fail(); } catch (SQLException exception) { - assertEquals(exception.getMessage(), "Error setting up SSL: Keystore was tampered with, or password was incorrect"); + assertEquals(exception.getMessage(), "Error setting up SSL: keystore password was incorrect"); } }