switch to ci extension tools, use vcpkg from zlib instead

Author: Maxxen

.github/workflows/MainDistributionPipeline.yml

@@ -24,16 +24,16 @@ concurrency:
 jobs:
   duckdb-latest-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.1.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       duckdb_version: main
       extension_name: spatial
-      ci_tools_version: v1.1.3
+      ci_tools_version: main
 
   duckdb-latest-deploy:
     name: Deploy extension binaries
     needs: duckdb-latest-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.1.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
     secrets: inherit
     with:
       duckdb_version: main

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "duckdb"]
 	path = duckdb
 	url = https://github.com/duckdb/duckdb.git
+[submodule "extension-ci-tools"]
+	path = extension-ci-tools
+	url = https://github.com/duckdb/extension-ci-tools.git

CMakeLists.txt

@@ -55,6 +55,7 @@ if(NOT EXISTS ${CMAKE_BINARY_DIR}/deps)
       -DVCPKG_INSTALLED_DIR='${CMAKE_BINARY_DIR}/vcpkg_installed'
       -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
       -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+      -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
       -S ${CMAKE_CURRENT_SOURCE_DIR}/deps -B ${CMAKE_BINARY_DIR}/deps
     RESULT_VARIABLE DEPENDENCIES_GENERATE_RESULT)
   if(NOT DEPENDENCIES_GENERATE_RESULT EQUAL 0)

Makefile

@@ -1,180 +1,8 @@
-.PHONY: all clean format debug release duckdb_debug duckdb_release pull update
+PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
-all: release
+# Configuration of extension
+EXT_NAME=excel
+EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
-DUCKDB_SRCDIR ?= "./duckdb/"
-
-MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
-PROJ_DIR := $(dir $(MKFILE_PATH))
-
-TEST_PATH="/test/unittest"
-DUCKDB_PATH="/duckdb"
-
-# For non-MinGW windows the path is slightly different
-ifeq ($(OS),Windows_NT)
-ifneq ($(CXX),g++)
-	TEST_PATH="/test/Release/unittest.exe"
-	DUCKDB_PATH="/Release/duckdb.exe"
-endif
-endif
-
-#### OSX config
-OSX_BUILD_FLAG=
-ifneq (${OSX_BUILD_ARCH}, "")
-	OSX_BUILD_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH}
-endif
-
-#### VCPKG config
-VCPKG_TOOLCHAIN_PATH?=
-ifneq ("${VCPKG_TOOLCHAIN_PATH}", "")
-	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}'
-endif
-ifneq ("${VCPKG_TARGET_TRIPLET}", "")
-	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
-endif
-ifeq (${USE_MERGED_VCPKG_MANIFEST}, 1)
-	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}build/extension_configuration'
-else
-	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}'
-endif
-
-#### Enable Ninja as generator
-ifeq ($(GEN),ninja)
-	GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1
-endif
-
-EXT_NAME=spatial
-
-#### Configuration for this extension
-EXTENSION_NAME=SPATIAL
-EXTENSION_FLAGS=\
--DENABLE_SANITIZER=OFF \
--DDUCKDB_EXTENSION_NAMES="spatial" \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)spatial/include" \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test" \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_LINKED_LIBS="../../deps/local/lib/*.a"
-
-#### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed)
-EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="parquet;json"
-
-BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}'
-ifeq (${BUILD_SHELL}, 0)
-	BUILD_FLAGS += -DBUILD_SHELL=0
-else
-	ifeq ($(OS),Windows_NT)
-		BUILD_FLAGS += -DBUILD_SHELL=0
-	endif
-endif
-
-ifeq (${BUILD_BENCHMARK}, 1)
-	BUILD_FLAGS += -DBUILD_BENCHMARKS=1
-endif
-
-CLIENT_FLAGS:=
-
-#### Main build
-# For regular CLI build, we link the spatial extension directly into the DuckDB executable
-CLIENT_FLAGS=-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1
-
-debug:
-	mkdir -p  build/debug && \
-	cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \
-	cmake --build build/debug --config Debug
-
-release:
-	mkdir -p build/release && \
-	cmake $(GENERATOR) $(BUILD_FLAGS)  $(CLIENT_FLAGS)  -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \
-	cmake --build build/release --config Release
-
-reldebug:
-	mkdir -p build/reldebug && \
-	cmake $(GENERATOR) $(BUILD_FLAGS)  $(CLIENT_FLAGS)  -DCMAKE_BUILD_TYPE=RelWithDebInfo -S ./duckdb/ -B build/reldebug && \
-	cmake --build build/reldebug --config RelWithDebInfo
-
-##### Client build
-JS_BUILD_FLAGS=-DBUILD_NODE=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0
-PY_BUILD_FLAGS=-DBUILD_PYTHON=1 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0
-
-debug_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS)
-debug_js: debug
-debug_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS)
-debug_python: debug
-release_js: CLIENT_FLAGS=$(JS_BUILD_FLAGS)
-release_js: release
-release_python: CLIENT_FLAGS=$(PY_BUILD_FLAGS)
-release_python: release
-
-# Main tests
-test: test_release
-
-test_release: release
-	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
-
-test_debug: debug
-	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
-
-#### Client tests
-DEBUG_EXT_PATH='$(PROJ_DIR)build/debug/extension/spatial/spatial.duckdb_extension'
-RELEASE_EXT_PATH='$(PROJ_DIR)build/release/extension/spatial/spatial.duckdb_extension'
-
-test_js: test_debug_js
-
-test_debug_js: debug_js
-	cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js"
-
-test_release_js: release_js
-	cd duckdb/tools/nodejs && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) npm run test-path -- "../../../test/nodejs/**/*.js"
-
-test_python: test_debug_python
-
-test_debug_python: debug_python
-	cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(DEBUG_EXT_PATH) python3 -m pytest
-
-test_release_python: release_python
-	cd test/python && ${EXTENSION_NAME}_EXTENSION_BINARY_PATH=$(RELEASE_EXT_PATH) python3 -m pytest
-
-#### Misc
-format:
-	find spatial/src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i
-	find spatial/include -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i
-	cmake-format -i spatial/src/CMakeLists.txt
-
-update:
-	git submodule update --remote --merge
-
-pull:
-	git submodule init
-	git submodule update --recursive --remote
-
-clean:
-	rm -rf build
-	rm -rf testext
-#cd duckdb && make clean
-#cd duckdb && make clean-python
-
-# WASM config
-VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake
-WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(VCPKG_EMSDK_FLAGS)
-WASM_CXX_MVP_FLAGS=
-WASM_CXX_EH_FLAGS=$(WASM_CXX_MVP_FLAGS) -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1
-WASM_CXX_THREADS_FLAGS=$(WASM_COMPILE_TIME_EH_FLAGS) -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread
-WASM_LINK_TIME_FLAGS=-O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_${EXT_NAME}_version,_${EXT_NAME}_init"
-
-# WASM targets
-wasm_mvp:
-	mkdir -p build/wasm_mvp
-	emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="$(WASM_CXX_MVP_FLAGS)" -S $(DUCKDB_SRCDIR) -DDUCKDB_EXPLICIT_PLATFORM=wasm_mvp -DDUCKDB_CUSTOM_PLATFORM=wasm_mvp
-	emmake make -j8 -Cbuild/wasm_mvp
-
-wasm_eh:
-	mkdir -p build/wasm_eh
-	emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="$(WASM_CXX_EH_FLAGS)" -S $(DUCKDB_SRCDIR) -DDUCKDB_EXPLICIT_PLATFORM=wasm_eh -DDUCKDB_CUSTOM_PLATFORM=wasm_eh
-	emmake make -j8 -Cbuild/wasm_eh
-
-wasm_threads:
-	mkdir -p ./build/wasm_threads
-	emcmake cmake $(GENERATOR) $(EXTENSION_FLAGS) $(WASM_COMPILE_TIME_COMMON_FLAGS) -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="$(WASM_CXX_THREADS_FLAGS)" -S $(DUCKDB_SRCDIR) -DDUCKDB_EXPLICIT_PLATFORM=wasm_threads -DDUCKDB_CUSTOM_PLATFORM=wasm_threads
-	emmake make -j8 -Cbuild/wasm_threads
+# Include the Makefile from extension-ci-tools
+include extension-ci-tools/makefiles/duckdb_extension.Makefile

deps/CMakeLists.txt

@@ -11,6 +11,10 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0")
     cmake_policy(SET CMP0135 NEW)
 endif()
 
+if(${VCPKG_TARGET_TRIPLET} STREQUAL "wasm32-emscripten")
+    set(EMSCRIPTEN ON)
+endif()
+
 if (NOT EMSCRIPTEN)
   if (OSX_BUILD_ARCH)
     message(STATUS "building for OSX architecture: ${OSX_BUILD_ARCH}")
@@ -36,37 +40,15 @@ if (EMSCRIPTEN)
   set(OGR_ENABLE_DRIVER_OPENFILEGDB OFF)
 endif()
 
-# ZLIB
-ExternalProject_Add(
-    ZLIB
-    URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/zlib1213.zip
-    CONFIGURE_HANDLED_BY_BUILD TRUE
-    CMAKE_ARGS
-    -DCMAKE_INSTALL_PREFIX:PATH=${LOCAL_INSTALL_DIR}
-    -DCMAKE_PREFIX_PATH=${LOCAL_INSTALL_DIR}
-    -DCMAKE_MODULE_PATH=${LOCAL_INSTALL_DIR}/lib/cmake
-    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-    -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_PACKED}
-    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-    -DBUILD_SHARED_LIBS=OFF
-    ${CMAKE_FLAGS_POINTER_SIZE}
-    # vcpkg options
-    -DVCPKG_MANIFEST_DIR='${VCPKG_MANIFEST_DIR}'
-    -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
-    -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
-    -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
-)
-
 # Use the system sqlite binary on Emscripten, iOS and Android
 if (EMSCRIPTEN OR IOS OR ANDROID)
   find_program(EXE_SQLITE3 sqlite3)
 endif()
-set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} ZLIB)
+set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES})
 
 # SQLite3
 ExternalProject_Add(
     SQLITE3
-    DEPENDS ZLIB
     URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/sqlite3
     CONFIGURE_HANDLED_BY_BUILD TRUE
     CMAKE_ARGS
@@ -80,14 +62,14 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
 )
 set(GDAL_DEPENDENCIES ${GDAL_DEPENDENCIES} SQLITE3)
 
 if(SPATIAL_USE_NETWORK)
 # CURL
 ExternalProject_Add(
     CURL
-    DEPENDS ZLIB
     URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/curl-7.87.0.zip
     CONFIGURE_HANDLED_BY_BUILD TRUE
     CMAKE_ARGS 
@@ -102,6 +84,7 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
     -DBUILD_SHARED_LIBS=OFF
     -DHTTP_ONLY=ON
     # Disable some defaults for now
@@ -118,7 +101,7 @@ endif()
 # PROJ
 ExternalProject_Add(
     PROJ
-    DEPENDS SQLITE3 ZLIB
+    DEPENDS SQLITE3
     URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/proj-9.1.1.zip
     CONFIGURE_HANDLED_BY_BUILD TRUE
     PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/proj${PATCH_NAME_ENDING}.patch"
@@ -136,6 +119,7 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
     # PROJ options
     -DBUILD_SHARED_LIBS=OFF
     -DBUILD_APPS=OFF
@@ -164,6 +148,7 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
     # EXPAT options
     -DEXPAT_SHARED_LIBS=OFF
     -DEXPAT_BUILD_TESTS=OFF
@@ -192,6 +177,7 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
     # GEOS options
     -DBUILD_SHARED_LIBS=OFF
     -DBUILD_TESTING=OFF
@@ -219,6 +205,7 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
     # GeographicLib options
     -DBUILD_SHARED_LIBS=OFF
     -DGEOGRAPHICLIB_DATA=${LOCAL_INSTALL_DIR}/share/GeographicLib
@@ -247,6 +234,7 @@ ExternalProject_Add(
     -DVCPKG_INSTALLED_DIR='${VCPKG_INSTALLED_DIR}'
     -DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'
     -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
+    -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE='${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}'
     # GDAL Options
     -DGDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE=ON # this is needed for GDAL to build with -fPIC
     -DBUILD_TESTING=OFF

extension-ci-tools

@@ -0,0 +1,10 @@
+# This file is included by DuckDB's build system. It specifies which extension to load
+
+# Extension from this repo
+duckdb_extension_load(spatial
+    SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
+    INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/spatial/include
+    LOAD_TESTS
+    DONT_LINK
+    LINKED_LIBS "../../deps/local/lib/*.a"
+)

extension_config.cmake

@@ -1,5 +1,6 @@
 {
     "dependencies": [
-       "openssl"
+       "openssl",
+      "zlib"
     ]
 }