Merge branch 'main' of github.com:facebookincubator/velox into gha-add-cuda

Author: luhenry

.github/workflows/scheduled.yml

@@ -102,16 +102,19 @@ jobs:
       presto_error: ${{ steps.sig-check.outputs.presto_error }}
       spark_bias: ${{ steps.sig-check.outputs.spark_functions }}
       spark_error: ${{ steps.sig-check.outputs.spark_error }}
+      presto_aggregate_bias: ${{ steps.sig-check.outputs.presto_aggregate_functions }}
+      presto_aggregate_error: ${{ steps.sig-check.outputs.presto_aggregate_error }}
 
     steps:
 
       - name: Get latest commit from main
         if: ${{ github.event_name != 'schedule' }}
+        working-directory: ${{ github.workspace }}
         env:
           GH_TOKEN: ${{ github.token }}
         id: get-head
         run: |
-          if [ '${{ github.event_name = 'push' }}' == "true" ]; then
+          if [ '${{ github.event_name == 'push' }}' == "true" ]; then
             # get the parent commit of the current one to get the relevant function signatures
             head_main=$(gh api -q '.parents.[0].sha' '/repos/facebookincubator/velox/commits/${{ github.sha }}')
           else
@@ -147,14 +150,14 @@ jobs:
           mkdir -p /tmp/signatures
 
       - name: Checkout Main
-        if: ${{ github.even_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
+        if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
         uses: actions/checkout@v4
         with:
           ref: ${{ steps.get-head.outputs.head_main || 'main' }}
           path: velox_main
 
       - name: Build PyVelox
-        if: ${{ github.even_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
+        if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
         working-directory: velox_main
         run: |
           python3 -m venv .venv
@@ -163,16 +166,17 @@ jobs:
           make python-build
 
       - name: Create Baseline Signatures
-        if: ${{ github.even_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
+        if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
         working-directory: velox_main
         run: |
           source .venv/bin/activate
           python3 -m pip install deepdiff
           python3 scripts/signature.py export --spark /tmp/signatures/spark_signatures_main.json
           python3 scripts/signature.py export --presto /tmp/signatures/presto_signatures_main.json
+          python3 scripts/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_main.json
 
       - name: Save Function Signature Stash
-        if: ${{ github.even_name == 'pull_request' && steps.get-sig.outputs.stash-hit != 'true' }}
+        if: ${{ github.event_name == 'pull_request' && steps.get-sig.outputs.stash-hit != 'true' }}
         uses: assignUser/stash/save@v1
         with:
           path: /tmp/signatures
@@ -202,14 +206,14 @@ jobs:
       - name: Save ccache
         # see https://github.com/actions/upload-artifact/issues/543
         continue-on-error: true
-        if: ${{ github.even_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' }}
         uses: assignUser/stash/save@v1
         with:
           path: "${{ env.CCACHE_DIR }}"
           key: ccache-fuzzer-centos
 
       - name: Build PyVelox
-        if: ${{ github.even_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' }}
         env:
           VELOX_BUILD_DIR: "_build/debug"
         run: |
@@ -218,15 +222,19 @@ jobs:
           python3 -m pip install -e .
 
       - name: Create and test new function signatures
-        if: ${{ github.even_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' }}
         id: sig-check
         run: |
           source .venv/bin/activate
           python3 -m pip install deepdiff
           python3 scripts/signature.py gh_bias_check presto spark
+          python3 scripts/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_contendor.json
+          python3 scripts/signature.py bias_aggregates  /tmp/signatures/presto_aggregate_signatures_main.json \
+           /tmp/signatures/presto_aggregate_signatures_contendor.json /tmp/signatures/presto_aggregate_bias_functions \
+           /tmp/signatures/presto_aggregate_errors
 
       - name: Upload Signature Artifacts
-        if: ${{ github.even_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' }}
         uses: actions/upload-artifact@v4
         with:
           name: signatures
@@ -246,7 +254,7 @@ jobs:
           done
 
       - name: Save Function Signature Stash
-        if: ${{ github.even_name == 'push' }}
+        if: ${{ github.event_name == 'push' }}
         uses: assignUser/stash/save@v1
         with:
           path: /tmp/signatures
@@ -703,6 +711,81 @@ jobs:
           path: |
             /tmp/aggregate_fuzzer_repro
             /tmp/server.log
+            
+
+  presto-bias-java-aggregation-fuzzer-run:
+    name: Biased Aggregation Fuzzer with Presto as source of truth
+    needs: compile
+    runs-on: ubuntu-latest
+    container: ghcr.io/facebookincubator/velox-dev:presto-java
+    timeout-minutes: 120
+    if: ${{ needs.compile.outputs.presto_aggregate_bias  == 'true' }}
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/.ccache/"
+      LINUX_DISTRO: "centos"
+    steps:
+
+      - name: Download aggregation fuzzer
+        uses: actions/download-artifact@v4
+        with:
+          name: aggregation
+
+      - name: "Checkout Repo"
+        uses: actions/checkout@v4
+        with:
+          path: velox
+          submodules: 'recursive'
+          ref: "${{ inputs.ref }}"
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory /__w/velox/velox/velox
+
+      - name: Download Signatures
+        uses: actions/download-artifact@v4
+        with:
+          name: signatures
+          path: /tmp/signatures
+
+      - name: "Run Bias Aggregate Fuzzer"
+        run: |
+          cd  velox
+          cp ./scripts/etc/hive.properties $PRESTO_HOME/etc/catalog
+          ls -lR $PRESTO_HOME/etc
+          $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
+          # Sleep for 60 seconds to allow Presto server to start.
+          sleep 60
+          /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
+          cd -
+          mkdir -p /tmp/aggregate_fuzzer_repro/
+          rm -rfv /tmp/aggregate_fuzzer_repro/*
+          chmod -R 777 /tmp/aggregate_fuzzer_repro
+          chmod +x velox_aggregation_fuzzer_test
+          echo "signatures folder"
+          ls /tmp/signatures/
+          echo "Biased functions:"
+          cat /tmp/signatures/presto_aggregate_bias_functions
+          echo "Running Fuzzer for $DURATION"
+          ./velox_aggregation_fuzzer_test \
+                --seed ${RANDOM} \
+                --duration_sec $DURATION \
+                --logtostderr=1 \
+                --minloglevel=1 \
+                --repro_persist_path=/tmp/aggregate_fuzzer_repro \
+                --enable_sorted_aggregations=true \
+                --only=$(cat /tmp/signatures/presto_aggregate_bias_functions) \
+                --presto_url=http://127.0.0.1:8080 \
+          && echo -e "\n\nAggregation fuzzer run finished successfully."
+
+      - name: Archive bias aggregate production artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: presto-bias-sot-aggregate-fuzzer-failure-artifacts
+          path: |
+            /tmp/aggregate_fuzzer_repro
+            /tmp/server.log
 
   surface-signature-errors:
     name: Signature Changes
@@ -721,3 +804,9 @@ jobs:
         run: |
           cat /tmp/signatures/presto_errors
           exit 1
+
+      - name: Surface Aggregate function signature errors
+        if: ${{ needs.compile.outputs.presto_aggregate_error == 'true' }}
+        run: |
+          cat /tmp/signatures/presto_aggregate_errors
+          exit 1

.gitignore

@@ -279,6 +279,7 @@ settings.json
 
 # User's build configuration
 Makefile.config
+CMakeUserPresets.json
 
 # build, distribute, and bins (+ python proto bindings)
 build

CMake/resolve_dependency_modules/README.md

@@ -23,15 +23,15 @@ by Velox. See details on bundling below.
 | zstd              | default         | No       |
 | openssl           | default         | No       |
 | protobuf          | 21 (exact)      | Yes      |
-| boost             | 1.66.0          | Yes      |
+| boost             | 1.77.0          | Yes      |
 | flex              | 2.5.13          | No       |
 | bison             | 3.0.4           | No       |
 | cmake             | 3.14            | No       |
 | double-conversion | 3.1.5           | No       |
 | xsimd             | 10.0.0          | Yes      |
 | re2               | 2021-04-01      | Yes      |
 | fmt               | 10.1.1          | Yes      |
-| simdjson          | 3.2.0           | Yes      |
+| simdjson          | 3.8.0           | Yes      |
 | folly             | v2024.04.01.00  | Yes      |
 | fizz              | v2024.04.01.00  | No       |
 | wangle            | v2024.04.01.00  | No       |
@@ -75,4 +75,3 @@ Ideally all patches should be upstream when possible and removed once merged.
 ## Specify a custom url/file path for an offline build
 
 Set environment variables `VELOX_<PACKAGE>_URL` to specify a custom dependency url or local tar file path, an optional sha256 checksum can be provided as `VELOX_<PACKAGE>_SHA256`.
-

CMake/resolve_dependency_modules/simdjson.cmake

@@ -13,9 +13,9 @@
 # limitations under the License.
 include_guard(GLOBAL)
 
-set(VELOX_SIMDJSON_VERSION 3.2.0)
+set(VELOX_SIMDJSON_VERSION 3.8.0)
 set(VELOX_SIMDJSON_BUILD_SHA256_CHECKSUM
-    75a684dbbe38cf72b8b3bdbdc430764813f3615899a6029931c26ddd89812da4)
+    e28e3f46f0012d405b67de6c0a75e8d8c9a612b0548cb59687822337d73ca78b)
 set(VELOX_SIMDJSON_SOURCE_URL
     "https://github.com/simdjson/simdjson/archive/refs/tags/v${VELOX_SIMDJSON_VERSION}.tar.gz"
 )

CMakeLists.txt

@@ -81,6 +81,12 @@ set(VELOX_DEPENDENCY_SOURCE
       STRING
       "Default source for all dependencies with source builds enabled: AUTO SYSTEM BUNDLED."
 )
+set(VELOX_GFLAGS_TYPE
+    "shared"
+    CACHE
+      STRING
+      "Specify whether to find the gflags package as a shared or static package"
+)
 option(VELOX_ENABLE_EXEC "Build exec." ON)
 option(VELOX_ENABLE_AGGREGATES "Build aggregates." ON)
 option(VELOX_ENABLE_HIVE_CONNECTOR "Build Hive connector." ON)
@@ -378,13 +384,13 @@ set(BOOST_INCLUDE_LIBRARIES
     thread)
 
 set_source(Boost)
-resolve_dependency(Boost 1.66.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES})
+resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES})
 
 # Range-v3 will be enable when the codegen code actually lands keeping it here
 # for reference. find_package(range-v3)
 
 set_source(gflags)
-resolve_dependency(gflags COMPONENTS shared)
+resolve_dependency(gflags COMPONENTS ${VELOX_GFLAGS_TYPE})
 if(NOT TARGET gflags::gflags)
   # This is a bit convoluted, but we want to be able to use gflags::gflags as a
   # target even when velox is built as a subproject which uses
@@ -441,7 +447,7 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
 endif()
 
 set_source(simdjson)
-resolve_dependency(simdjson 3.2.0)
+resolve_dependency(simdjson 3.8.0)
 
 # Locate or build folly.
 add_compile_definitions(FOLLY_HAVE_INT128_T=1)

scripts/signature.py

@@ -14,6 +14,7 @@
 import argparse
 import json
 import os
+import re
 import sys
 from typing import Any
 
@@ -33,6 +34,9 @@ class bcolors:
     BOLD = "\033[1m"
 
 
+aggregate_pattern = re.compile("(.*)(_merge|_merge_extract|_partial)")
+
+
 def get_error_string(error_message):
     return f"""
 Incompatible changes in function signatures have been detected.
@@ -210,6 +214,48 @@ def bias_signatures(base_signatures, contender_signatures, tickets, error_path):
     return "", status
 
 
+def bias_aggregates(args):
+    """
+    Finds and exports aggregates whose signatures have been modified agasint a baseline.
+    Saves the results to a file and sets a Github Actions Output.
+    Currently this is hardcoded to presto aggregates.
+    """
+    with open(args.base) as f:
+        base_signatures = json.load(f)
+
+    with open(args.contender) as f:
+        contender_signatures = json.load(f)
+
+    delta, status = diff_signatures(
+        base_signatures, contender_signatures, args.error_path
+    )
+
+    set_gh_output("presto_aggregate_error", status == 1)
+
+    if not delta:
+        print(f"{bcolors.BOLD} No changes detected: Nothing to do!")
+        return status
+
+    function_set = set()
+    for items in delta.values():
+        for item in items:
+            fn_name = item.get_root_key()
+            pattern = aggregate_pattern.match(fn_name)
+            if pattern:
+                function_set.add(pattern.group(1))
+            else:
+                function_set.add(fn_name)
+
+    if function_set:
+        biased_functions = ",".join(function_set)
+        with open(args.output_path, "w") as f:
+            print(f"{biased_functions}", file=f, end="")
+
+        set_gh_output("presto_aggregate_functions", True)
+
+    return 0
+
+
 def gh_bias_check(args):
     """
     Exports signatures for the given group(s) and checks them for changes compared to a baseline.
@@ -294,6 +340,7 @@ def parse_args(args):
         "ticket_value", type=get_tickets, default=10, nargs="?"
     )
     bias_command_parser.add_argument("error_path", type=str, default="")
+
     gh_command_parser = command.add_parser("gh_bias_check")
     gh_command_parser.add_argument(
         "group",
@@ -314,6 +361,12 @@ def parse_args(args):
         "--output_postfix", type=str, default="_bias_functions"
     )
 
+    bias_aggregate_command_parser = command.add_parser("bias_aggregates")
+    bias_aggregate_command_parser.add_argument("base", type=str)
+    bias_aggregate_command_parser.add_argument("contender", type=str)
+    bias_aggregate_command_parser.add_argument("output_path", type=str)
+    bias_aggregate_command_parser.add_argument("error_path", type=str, default="")
+
     parser.set_defaults(command="help")
 
     return parser.parse_args(args)