Skip to content

Commit 528081c

Browse files
committed
Merge branch 'circleci-add-cuda' into circleci-add-cuda-1
2 parents 9e4e9f7 + 9bbbfee commit 528081c

File tree

543 files changed

+21517
-25473
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

543 files changed

+21517
-25473
lines changed

.circleci/dist_compile.yml

-53
Original file line numberDiff line numberDiff line change
@@ -299,12 +299,6 @@ jobs:
299299
DuckDB_SOURCE: SYSTEM
300300
steps:
301301
- pre-steps
302-
- run:
303-
name: "Install Adapter Dependencies"
304-
command: |
305-
source /opt/rh/gcc-toolset-9/enable
306-
set -xu
307-
PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh abfs
308302
- run:
309303
name: "Install Java for Hadoop"
310304
command: |
@@ -431,43 +425,6 @@ jobs:
431425
fuzzer_exe: "_build/debug/velox/exec/tests/velox_join_fuzzer_test"
432426
fuzzer_args: " --seed ${RANDOM} --duration_sec 3600 --logtostderr=1 --minloglevel=0"
433427

434-
doc-gen-job:
435-
executor: build
436-
steps:
437-
- checkout
438-
- update-submodules
439-
- add_ssh_keys:
440-
fingerprints:
441-
- "7b:24:f3:1a:b1:15:97:c6:fe:06:46:27:3e:b7:6b:96"
442-
- run:
443-
name: "Build docs and update gh-pages"
444-
command: |
445-
for i in {1..3}; do
446-
make clean
447-
git config --global user.email "velox@users.noreply.github.com"
448-
git config --global user.name "velox"
449-
git checkout main
450-
conda init bash
451-
source ~/.bashrc
452-
conda create -y --name docgenenv python=3.7
453-
conda activate docgenenv
454-
pip install sphinx sphinx-tabs breathe sphinx_rtd_theme chardet
455-
source /opt/rh/gcc-toolset-9/enable
456-
./scripts/gen-docs.sh docgenenv
457-
git checkout gh-pages
458-
cp -R velox/docs/_build/html/* docs
459-
git add docs
460-
if [ -n "$(git status --porcelain --untracked-files=no)" ]
461-
then
462-
git commit -m "Update documentation"
463-
git push
464-
if [ $? -eq 0 ]; then
465-
break;
466-
fi
467-
fi
468-
done
469-
470-
471428
linux-pr-fuzzer-run:
472429
executor: build
473430
steps:
@@ -572,11 +529,6 @@ workflows:
572529
- linux-build-options
573530
- linux-adapters
574531
- linux-presto-fuzzer-run
575-
- doc-gen-job:
576-
filters:
577-
branches:
578-
only:
579-
- main
580532

581533
shorter-fuzzer:
582534
unless: << pipeline.parameters.run-longer-expression-fuzzer >>
@@ -585,8 +537,3 @@ workflows:
585537
- linux-pr-fuzzer-run
586538
- linux-build-options
587539
- linux-adapters
588-
- doc-gen-job:
589-
filters:
590-
branches:
591-
only:
592-
- main

velox/experimental/codegen/proto/protobuf.sh .github/dependabot.yml

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
#!/bin/bash
21
# Copyright (c) Facebook, Inc. and its affiliates.
32
#
43
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,4 +12,10 @@
1312
# See the License for the specific language governing permissions and
1413
# limitations under the License.
1514

16-
"${PROTOC}" "${PROTO_FILE}" "--${LANG}_out=$INSTALL_DIR"
15+
version: 2
16+
updates:
17+
- package-ecosystem: "github-actions"
18+
directory: "/"
19+
schedule:
20+
interval: "weekly"
21+

.github/workflows/benchmark.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ concurrency:
3535
defaults:
3636
run:
3737
shell: bash
38-
#TODO concurrency groups?
38+
3939
jobs:
4040
benchmark:
4141
if: github.repository == 'facebookincubator/velox'
@@ -198,7 +198,7 @@ jobs:
198198
echo "::endgroup::"
199199

200200
- name: "Save PR number"
201-
run: echo "${{ github.event.pull_request.number }}" > pr_number.txt
201+
run: echo "${{ github.event.pull_request.number || 0 }}" > pr_number.txt
202202

203203
- name: "Upload PR number"
204204
uses: actions/upload-artifact@v3

.github/workflows/conbench_upload.yml

+13-2
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,18 @@ jobs:
9494
run: |
9595
unzip benchmark-results.zip -d benchmark-results
9696
unzip pr_number.zip
97-
echo "pr_number=$(cat pr_number.txt)" >> $GITHUB_OUTPUT
97+
98+
pr_number=$(grep -ox '[[:digit:]]*' pr_number.txt | head -1)
99+
100+
if [ "$pr_number" -ge 0 ]; then
101+
echo "Found PR number: $pr_number"
102+
else
103+
echo '::error :: Malformed input, aborting!'
104+
exit 1
105+
fi
106+
107+
echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
108+
98109
- uses: actions/checkout@v3
99110
with:
100111
path: velox
@@ -134,7 +145,7 @@ jobs:
134145

135146
- name: "Create a GitHub Status on the contender commit (whether the upload was successful)"
136147
uses: actions/github-script@v6
137-
if: always()
148+
if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
138149
with:
139150
script: |
140151
let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}'

.github/workflows/docs.yml

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Update Documentation
16+
17+
on:
18+
push:
19+
paths:
20+
- "velox/docs/**"
21+
- ".github/workflows/docs.yml"
22+
23+
pull_request:
24+
paths:
25+
- "velox/docs/**"
26+
- ".github/workflows/docs.yml"
27+
28+
permissions:
29+
contents: write
30+
31+
concurrency:
32+
group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }}
33+
cancel-in-progress: true
34+
35+
jobs:
36+
build_docs:
37+
name: Build and Push
38+
runs-on: ubuntu-latest
39+
steps:
40+
41+
- name: Checkout
42+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
43+
with:
44+
fetch-depth: 0
45+
46+
- name: Setup Git User
47+
run: |
48+
git config --global user.email "velox@users.noreply.github.com"
49+
git config --global user.name "velox"
50+
51+
- name: Install Dependencies
52+
run: |
53+
sudo apt update
54+
sudo apt install -y pandoc
55+
pip install sphinx sphinx-tabs breathe sphinx_rtd_theme chardet
56+
57+
- name: Build Documentation
58+
run: |
59+
cd velox/docs
60+
make clean
61+
# pyvelox
62+
mkdir -p bindings/python
63+
pandoc ../../pyvelox/README.md --from markdown --to rst -s -o bindings/python/README_generated_pyvelox.rst
64+
# velox
65+
make html
66+
67+
- name: Push Documentation
68+
if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}}
69+
run: |
70+
git checkout gh-pages
71+
cp -R velox/docs/_build/html/* docs
72+
git add docs
73+
74+
if [ -n "$(git status --porcelain --untracked-files=no)" ]
75+
then
76+
git commit -m "Update documentation"
77+
git push
78+
fi
79+
80+
- name: Upload Documentation
81+
if: github.event_name == 'pull_request'
82+
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
83+
with:
84+
path: velox/docs/_build/html
85+
retention-days: 3

.github/workflows/macos.yml

+2
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ jobs:
5858
with:
5959
submodules: recursive
6060
- name: Install Dependencies
61+
env:
62+
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE"
6163
run: |
6264
brew install \
6365
bison boost ccache double-conversion flex fmt gflags glog \

.github/workflows/scheduled.yml

+6-5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ on:
1818
pull_request:
1919
paths:
2020
- ".github/workflows/scheduled.yml"
21+
- scripts/*
2122

2223
schedule:
2324
- cron: '0 3 * * *'
@@ -75,7 +76,7 @@ jobs:
7576
ref: "${{ inputs.ref || 'main' }}"
7677

7778
- name: "Install dependencies"
78-
run: cd velox && source ./scripts/setup-ubuntu.sh
79+
run: cd velox && sudo ./scripts/setup-ubuntu.sh
7980

8081
- name: "Build"
8182
run: |
@@ -125,7 +126,7 @@ jobs:
125126
ref: "${{ inputs.ref || 'main' }}"
126127

127128
- name: "Install dependencies"
128-
run: source ./scripts/setup-ubuntu.sh
129+
run: sudo ./scripts/setup-ubuntu.sh
129130

130131
- name: Download presto fuzzer
131132
uses: actions/download-artifact@v3
@@ -173,7 +174,7 @@ jobs:
173174
ref: "${{ inputs.ref || 'main' }}"
174175

175176
- name: "Install dependencies"
176-
run: source ./scripts/setup-ubuntu.sh
177+
run: sudo ./scripts/setup-ubuntu.sh
177178

178179
- name: Download spark expression fuzzer
179180
uses: actions/download-artifact@v3
@@ -247,7 +248,7 @@ jobs:
247248
ref: "${{ inputs.ref || 'main' }}"
248249

249250
- name: "Install dependencies"
250-
run: source ./scripts/setup-ubuntu.sh
251+
run: sudo ./scripts/setup-ubuntu.sh
251252

252253
- name: Download aggregation fuzzer
253254
uses: actions/download-artifact@v3
@@ -288,7 +289,7 @@ jobs:
288289
ref: "${{ inputs.ref || 'main' }}"
289290

290291
- name: "Install dependencies"
291-
run: source ./scripts/setup-ubuntu.sh
292+
run: sudo ./scripts/setup-ubuntu.sh
292293

293294
- name: Download join fuzzer
294295
uses: actions/download-artifact@v3

CMake/resolve_dependency_modules/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ by Velox. See details on bundling below.
3131
| xsimd | 10.0.0 | Yes |
3232
| re2 | 2021-04-01 | Yes |
3333
| fmt | 10.1.1 | Yes |
34+
| simdjson | 3.2.0 | Yes |
3435
| folly | v2023.12.04.00 | Yes |
3536
| fizz | v2023.12.04.00 | No |
3637
| wangle | v2023.12.04.00 | No |

CMake/resolve_dependency_modules/boost/CMakeLists.txt

+2-5
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ add_compile_options(-w)
2121
# We need to use boost > 1.70 to build it with CMake 1.81 was the first to be
2222
# released as a github release INCLUDING the cmake files (which are not in the
2323
# officale releases for some reason)
24-
set(VELOX_BOOST_BUILD_VERSION 1.81.0)
24+
set(VELOX_BOOST_BUILD_VERSION 1.84.0)
2525
string(
2626
CONCAT VELOX_BOOST_SOURCE_URL
2727
"https://github.com/boostorg/boost/releases/download/"
2828
"boost-${VELOX_BOOST_BUILD_VERSION}/"
2929
"boost-${VELOX_BOOST_BUILD_VERSION}.tar.gz")
3030
set(VELOX_BOOST_BUILD_SHA256_CHECKSUM
31-
121da556b718fd7bd700b5f2e734f8004f1cfa78b7d30145471c526ba75a151c)
31+
4d27e9efed0f6f152dc28db6430b9d3dfb40c0345da7342eaa5a987dde57bd95)
3232

3333
resolve_dependency_url(BOOST)
3434
message(STATUS "Building boost from source")
@@ -65,8 +65,5 @@ list(REMOVE_ITEM BOOST_INCLUDE_LIBRARIES headers)
6565
set(BUILD_SHARED_LIBS ON)
6666
FetchContent_MakeAvailable(Boost)
6767

68-
# To aling with Boost system install we create Boost::headers target
69-
add_library(boost_headers INTERFACE)
70-
add_library(Boost::headers ALIAS boost_headers)
7168
list(TRANSFORM BOOST_HEADER_ONLY PREPEND Boost::)
7269
target_link_libraries(boost_headers INTERFACE ${BOOST_HEADER_ONLY})

CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
456456
endif()
457457

458458
set_source(simdjson)
459-
resolve_dependency(simdjson)
459+
resolve_dependency(simdjson 3.2.0)
460460

461461
# Locate or build folly.
462462
add_compile_definitions(FOLLY_HAVE_INT128_T=1)

scripts/check.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -189,9 +189,12 @@ def get_files(commit, path):
189189
)
190190
filelist = stdout.splitlines()
191191
else:
192-
for root, dirs, files in os.walk(path):
193-
for name in files:
194-
filelist.append(os.path.join(root, name))
192+
if os.path.isfile(path):
193+
filelist.append(path)
194+
else:
195+
for root, dirs, files in os.walk(path):
196+
for name in files:
197+
filelist.append(os.path.join(root, name))
195198

196199
return [
197200
file

scripts/setup-centos8.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ function install_lzo {
9090
}
9191

9292
function install_boost {
93-
wget_and_untar https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.gz boost
93+
wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
9494
(
9595
cd boost
9696
./bootstrap.sh --prefix=/usr/local

scripts/setup-helper-functions.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ function cmake_install {
158158
mkdir -p "${BINARY_DIR}"
159159
CPU_TARGET="${CPU_TARGET:-unknown}"
160160
COMPILER_FLAGS=$(get_cxx_flags $CPU_TARGET)
161+
SUDO="${SUDO:-""}"
161162

162163
# CMAKE_POSITION_INDEPENDENT_CODE is required so that Velox can be built into dynamic libraries \
163164
cmake -Wno-dev -B"${BINARY_DIR}" \
@@ -169,6 +170,6 @@ function cmake_install {
169170
-DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" \
170171
-DBUILD_TESTING=OFF \
171172
"$@"
172-
ninja -C "${BINARY_DIR}" install
173+
${SUDO} ninja -C "${BINARY_DIR}" install
173174
}
174175

0 commit comments

Comments
 (0)