Add support of out-of-tree testing for GNU toolchain

gnu-test-setup/Makefile.gnu

@@ -0,0 +1,52 @@
+TESTSUITE ?= TESTSUITE_PATH
+RUNTEST := $(shell command -v runtest || echo "")
+
+ifeq ($(RUNTEST),)
+$(error "Error: runtest not found in PATH")
+endif
+
+TEST_TARGET:=gcc g++
+PWD:=$(shell pwd)
+
+.PHONY: all check clean
+
+all: $(addsuffix .sum, $(TEST_TARGET))
+
+check: all
+
+gcc-parallel-%: site.exp
+	mkdir -p gcc/$@
+	cp site.exp gcc/$@
+	sed -i 's/set tmpdir.*/set tmpdir "$(subst /,\/,$(PWD))\/gcc\/$@"/g' $(PWD)/gcc/$@/site.exp
+	-cd gcc/$@ && $(RUNTEST) --tool gcc $(RUNTESTFLAGS)
+	touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished
+
+g++-parallel-%: site.exp
+	mkdir -p g++/$@
+	cp site.exp g++/$@
+	sed -i 's/set tmpdir.*/set tmpdir "$(subst /,\/,$(PWD))\/g++\/$@"/g' $(PWD)/g++/$@/site.exp
+	-cd g++/$@ && $(RUNTEST) --tool g++ $(RUNTESTFLAGS)
+	touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished
+
+gcc.sum: site.exp
+	rm -rf $(PWD)/gcc/gcc-parallel
+	mkdir -p $(PWD)/gcc/gcc-parallel
+	export GCC_RUNTEST_PARALLELIZE_DIR=$(PWD)/gcc/gcc-parallel; \
+	$(MAKE) $(addprefix gcc-parallel-,$(shell seq 40))
+	-cd gcc && python3 $(TESTSUITE)/dg-extract-results.py -L */gcc.log > gcc.log
+	-cd gcc && python3 $(TESTSUITE)/dg-extract-results.py */gcc.sum > gcc.sum
+	cp gcc/gcc.log .
+	cp gcc/gcc.sum .
+
+g++.sum: site.exp
+	rm -rf $(PWD)/g++/g++-parallel
+	mkdir -p $(PWD)/g++/g++-parallel
+	export GCC_RUNTEST_PARALLELIZE_DIR=$(PWD)/g++/g++-parallel; \
+	$(MAKE) $(addprefix g++-parallel-,$(shell seq 40))
+	-cd g++ && python3 $(TESTSUITE)/dg-extract-results.py -L */g++.log > g++.log
+	-cd g++ && python3 $(TESTSUITE)/dg-extract-results.py */g++.sum > g++.sum
+	cp g++/g++.log .
+	cp g++/g++.sum .
+
+clean:
+	rm -rf *.log *.sum *.bc *.s gcc g++

gnu-test-setup/fetch-gcc-testharness.sh

@@ -0,0 +1,12 @@
+# make sure we have one argument which is the path of the GNU testsuite
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <path-to-gnu-testsuite>"
+    exit 1
+fi
+git archive --remote=git://gcc.gnu.org/git/gcc.git HEAD contrib/dg-extract-results.py | tar -x --strip-components=1
+git archive --remote=git://gcc.gnu.org/git/gcc.git HEAD gcc/testsuite/lib | tar -x --strip-components=2
+
+# Move lib to testsuite/lib
+mv lib $1
+# Move dg-extract-results.py to testsuite
+mv dg-extract-results.py $1

g++-auto-generated.exp → gnu-test-setup/g++-auto-generated.exp

@@ -30,22 +30,18 @@ if ![info exists DEFAULT_CFLAGS] then {
     set DEFAULT_CFLAGS " -ansi -pedantic-errors"
 }
 
-set gcc_march "rv64gcv_zvfh"
-set gcc_mabi  "lp64d"
-
 # Initialize `dg'.
 dg-init
 
 # Main loop.
-set CFLAGS "$DEFAULT_CFLAGS -march=$gcc_march -mabi=$gcc_mabi -O3"
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/policy_funcs/gnu-api-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/policy_funcs/gnu-overloaded-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/gnu-api-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/gnu-overloaded-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 
 # All done.
 dg-finish

gcc-auto-generated.exp → gnu-test-setup/gcc-auto-generated.exp

@@ -30,22 +30,18 @@ if ![info exists DEFAULT_CFLAGS] then {
     set DEFAULT_CFLAGS " -ansi -pedantic-errors"
 }
 
-set gcc_march "rv64gcv_zvfh"
-set gcc_mabi  "lp64d"
-
 # Initialize `dg'.
 dg-init
 
 # Main loop.
-set CFLAGS "$DEFAULT_CFLAGS -march=$gcc_march -mabi=$gcc_mabi -O3"
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/policy_funcs/gnu-api-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/policy_funcs/gnu-overloaded-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/gnu-api-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/gnu-overloaded-tests/*.\[cS\]]] \
-	"" $CFLAGS
+	"" ""
 
 # All done.
 dg-finish

gnu-test-setup/site.exp

@@ -0,0 +1,8 @@
+set host_triplet "x86_64-pc-linux-gnu"
+set build_triplet "x86_64-pc-linux-gnu"
+set srcdir "SRCDIR"
+set target_triplet "riscv64-unknown-linux-gnu"
+set target_alias "riscv64-unknown-linux-gnu"
+set GCC_UNDER_TEST "TOOLCHAIN_DIR/bin/riscv64-unknown-linux-gnu-gcc"
+set GXX_UNDER_TEST "TOOLCHAIN_DIR/bin/riscv64-unknown-linux-gnu-g++"
+set TEST_ALWAYS_FLAGS "-march=rv64gcv_zvfh -mabi=lp64d"

rvv-intrinsic-generator/Makefile

@@ -93,12 +93,17 @@ TRIGGER_SKIP_DEFAULT_INST := --skip-default-inst
 else
 TRIGGER_SKIP_DEFAULT_INST :=
 endif
-# Directory that stores the gnu toolchain
-GNU_TOOLCHAIN_DIR :=
-# Directory that stores the gcc cases
-GCC_CASES_DIR := $(GNU_TOOLCHAIN_DIR)/gcc/gcc/testsuite/gcc.target/riscv/rvv/gcc-auto-generated
-# Directory that stores the g++ cases
-G++_CASES_DIR := $(GNU_TOOLCHAIN_DIR)/gcc/gcc/testsuite/g++.target/riscv/rvv/g++-auto-generated
+# Directory for GNU_TEST_SETUP
+GNU_TEST_SETUP := $(DIR)/../gnu-test-setup
+
+# Directory for GNU Toolchain API tests
+GNU_TOOLCHAIN_TEST_DIR := $(CURDIR)/gnu-toolchain-tests/testsuite/
+# # Directory that stores the gnu toolchain
+# GNU_TOOLCHAIN_DIR :=
+# # Directory that stores the gcc cases
+GCC_CASES_DIR := $(GNU_TOOLCHAIN_TEST_DIR)/gcc
+# # Directory that stores the g++ cases
+G++_CASES_DIR := $(GNU_TOOLCHAIN_TEST_DIR)/g++
 
 ###############################################################################
 # Functions
@@ -557,20 +562,18 @@ run-test:
 		CC=${COMPILER} \
 		-f ${API_MAKEFILE} -j${nproc}
 
-# Test gnu auto generaged overloaded-api-testing with testing-report
-report-gnu:
-	cd ${GNU_TOOLCHAIN_DIR} && ./configure --prefix="${GNU_TOOLCHAIN_DIR}/install" \
-						--with-arch=rv64gcv \
-						--with-abi=lp64d \
-						--with-isa-spec=20191213 \
-						--with-sim=qemu
-	cd ${GNU_TOOLCHAIN_DIR} && make distclean
-	cd ${GNU_TOOLCHAIN_DIR} && make -j 8
+# Test gnu auto generated overloaded-api-testing with testing-report
+report-gnu: prepare-gnu
+	make -C $(GNU_TOOLCHAIN_TEST_DIR)../ -j${nproc}
+
+prepare-gnu:
+	$(call check_defined, TOOLCHAIN_DIR, path to GNU toolchain)
+	# setup testsuite
 	mkdir -p $(GCC_CASES_DIR)/gnu-api-tests
 	mkdir -p $(GCC_CASES_DIR)/policy_funcs/gnu-api-tests
 	mkdir -p $(GCC_CASES_DIR)/gnu-overloaded-tests
 	mkdir -p $(GCC_CASES_DIR)/policy_funcs/gnu-overloaded-tests
-	cp $(DIR)/../gcc-auto-generated.exp $(GCC_CASES_DIR)/auto-generated.exp
+	cp $(GNU_TEST_SETUP)/gcc-auto-generated.exp $(GCC_CASES_DIR)/gcc.exp
 	cp -r $(DIR)/gnu-api-tests/* $(GCC_CASES_DIR)/gnu-api-tests/
 	cp -r $(DIR)/policy_funcs/gnu-api-tests/* $(GCC_CASES_DIR)/policy_funcs/gnu-api-tests/
 	cp -r $(DIR)/gnu-overloaded-tests/* $(GCC_CASES_DIR)/gnu-overloaded-tests/
@@ -579,12 +582,20 @@ report-gnu:
 	mkdir -p $(G++_CASES_DIR)/policy_funcs/gnu-api-tests
 	mkdir -p $(G++_CASES_DIR)/gnu-overloaded-tests
 	mkdir -p $(G++_CASES_DIR)/policy_funcs/gnu-overloaded-tests
-	cp $(DIR)/../g++-auto-generated.exp $(G++_CASES_DIR)/auto-generated.exp
+	cp $(GNU_TEST_SETUP)/g++-auto-generated.exp $(G++_CASES_DIR)/g++.exp
 	cp -r $(DIR)/gnu-api-tests/* $(G++_CASES_DIR)/gnu-api-tests/
 	cp -r $(DIR)/policy_funcs/gnu-api-tests/* $(G++_CASES_DIR)/policy_funcs/gnu-api-tests/
 	cp -r $(DIR)/gnu-overloaded-tests/* $(G++_CASES_DIR)/gnu-overloaded-tests/
 	cp -r $(DIR)/policy_funcs/gnu-overloaded-tests/* $(G++_CASES_DIR)/policy_funcs/gnu-overloaded-tests/
-	cd ${GNU_TOOLCHAIN_DIR} && make report -j 8 RUNTESTFLAGS="auto-generated.exp"
+	# setup the site.exp
+	cp $(GNU_TEST_SETUP)/site.exp $(GNU_TOOLCHAIN_TEST_DIR)/../site.exp
+	sed -i 's|TOOLCHAIN_DIR|$(TOOLCHAIN_DIR)|g' $(GNU_TOOLCHAIN_TEST_DIR)/../site.exp
+	sed -i 's|SRCDIR|$(GNU_TOOLCHAIN_TEST_DIR)|g' $(GNU_TOOLCHAIN_TEST_DIR)/../site.exp
+	# setup gcc testharness
+	bash $(GNU_TEST_SETUP)/fetch-gcc-testharness.sh $(GNU_TOOLCHAIN_TEST_DIR)
+	# setup makefile
+	cp $(GNU_TEST_SETUP)/Makefile.gnu $(GNU_TOOLCHAIN_TEST_DIR)/../Makefile
+	sed -i 's|TESTSUITE_PATH|$(GNU_TOOLCHAIN_TEST_DIR)|g' $(GNU_TOOLCHAIN_TEST_DIR)/../Makefile
 
 ###############################################################################
 # Custom Vendor Generator Target