Skip to content

Commit

Permalink
Improve CI (#56)
Browse files Browse the repository at this point in the history
Move CI from Travis to Gitlab
  • Loading branch information
mbrobbel authored Nov 22, 2018
1 parent eb71de5 commit 0c79071
Show file tree
Hide file tree
Showing 11 changed files with 313 additions and 83 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Dockerfile
257 changes: 257 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
image: docker:latest

services:
- docker:dind

stages:
- check
- test
- build
- examples

# before_script:
# - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin

.env: &env
ARROW_VERSION: 0.11.1
AWS_FPGA_VERSION: 1.4.4
CAPI_SNAP_VERSION: 1.5.1
CAPI_PSLSE_VERSION: "4.1"

GHDL_IMAGE: ghdl/ghdl
GHDL_TAG: ubuntu18-llvm-5.0

RUST_VERSION: 1.30.1
RUST_VHDL_PARSER_CRATE_VERSION: 0.3.0

.cmake-env: &cmake-env
<<: *env
APT_PACKAGES: cmake g++
CMAKE_BUILD_TYPE: Debug
CTEST_OUTPUT_ON_FAILURE: 1
FLETCHER_CPP: 0
FLETCHER_ECHO: 0
FLETCHER_AWS: 0
FLETCHER_SNAP: 0
FLETCHER_GEN: 0
FLETCHER_PYTHON: 0
FLETCHER_TESTS: 1

.cmake-build-env: &cmake-build-env
<<: *cmake-env
FLETCHER_TESTS: 0
CMAKE_BUILD_TYPE: Release

variables:
<<: *env

.apt-install: &apt-install |
if [ "$APT_PACKAGES" ]; then
apt-get update && apt-get install -y $APT_PACKAGES
fi

.pip-install: &pip-install |
if [ "$PIP_PACKAGES" ]; then
pip3 || curl "https://bootstrap.pypa.io/get-pip.py" | python3
pip3 install -U $PIP_PACKAGES
fi

check-vhdl-syntax:
stage: check
image: rust:${RUST_VERSION}-slim-stretch
before_script:
- cargo install vhdl_parser --version $RUST_VHDL_PARSER_CRATE_VERSION
script:
- find hardware -name "*.vhd" |
xargs vhdl_parser --show

.ghdl-check-job: &ghdl-check-job
stage: check
image: $GHDL_IMAGE:$GHDL_TAG
script:
- find hardware/vhdl -name "*.vhd" |
xargs ghdl -i -v --std=${STD:-08} |
grep entity |
sed -e 's/entity //' |
sed -e 's/ \*\*//' |
xargs -L 1 ghdl -m --std=${STD:-08} --ieee=synopsys

check-vhdl-93c:
<<: *ghdl-check-job
variables:
<<: *env
STD: 93c

check-vhdl-08:
<<: *ghdl-check-job

test-vhdl-08:
<<: *ghdl-check-job
stage: test
allow_failure: true
script:
- find hardware -name "*.vhd" |
xargs ghdl -i -v --std=${STD:-08} |
grep entity |
grep _tb |
sed -e 's/entity //' |
sed -e 's/ \*\*//' |
xargs -i -t bash -c '
ghdl -m --std=${STD:-08} --ieee=synopsys {};
ghdl -r --std=${STD:-08} --ieee=synopsys {} --stop-time=100ns'

.cmake-job: &cmake-job
image: mbrobbel/libarrow:$ARROW_VERSION
variables:
<<: *cmake-env
before_script:
- |
if [ "$FLETCHER_TESTS" -ne 0 ]; then
export APT_PACKAGES="$APT_PACKAGES libgtest-dev"
fi
if [ "$FLETCHER_AWS" -ne 0 ]; then
export APT_PACKAGES="$APT_PACKAGES git"
fi
if [ "$FLETCHER_SNAP" -ne 0 ]; then
export APT_PACKAGES="$APT_PACKAGES git curl"
fi
if [ "$FLETCHER_PYTHON" -ne 0 ]; then
export APT_PACKAGES="$APT_PACKAGES curl"
export PIP_PACKAGES="$PIP_PACKAGES cython numpy pyarrow==$ARROW_VERSION"
fi
- *apt-install
- *pip-install
- |
if [ "$FLETCHER_PYTHON" -ne 0 ]; then
export PYARROW_DIR=`python3 -c "import pyarrow as pa; print(pa.get_library_dirs()[0])"`
fi
if [ "$FLETCHER_AWS" -ne 0 ]; then
git clone --single-branch --depth 1 --branch v$AWS_FPGA_VERSION https://github.com/aws/aws-fpga
pushd aws-fpga
source sdk_setup.sh
popd
fi
if [ "$FLETCHER_SNAP" -ne 0 ]; then
git clone --single-branch --depth 1 --branch v$CAPI_PSLSE_VERSION https://github.com/ibm-capi/pslse
pushd pslse
export PSLSE_ROOT=`pwd`
popd
git clone --single-branch --depth 1 --branch v$CAPI_SNAP_VERSION https://github.com/open-power/snap
pushd snap
export SNAP_ROOT=`pwd`
PSLVER=8 BUILD_SIMCODE=1 make software
popd
fi
- mkdir -p build
script:
- pushd build
- cmake
-DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
-DFLETCHER_CPP=$FLETCHER_CPP
-DFLETCHER_ECHO=$FLETCHER_ECHO
-DFLETCHER_AWS=$FLETCHER_AWS
-DFLETCHER_SNAP=$FLETCHER_SNAP
-DFLETCHER_GEN=$FLETCHER_GEN
-DFLETCHER_PYTHON=$FLETCHER_PYTHON
-DPYARROW_DIR=$PYARROW_DIR
-DFLETCHER_TESTS=$FLETCHER_TESTS
../$SOURCE_PATH
- make -j
- |
if [ "$FLETCHER_TESTS" -ne 0 ]; then
make test
fi
- popd
- |
if [ "$FLETCHER_PYTHON" -ne 0 ]; then
pushd build
make install
popd
pushd runtime/python
python3 setup.py install
ldconfig
python3 testing/test.py
popd
fi
.cmake-test-job: &cmake-test-job
<<: *cmake-job
stage: test

.cmake-build-job: &cmake-build-job
<<: *cmake-job
stage: build
variables:
<<: *cmake-build-env

test-cpp-runtime:
<<: *cmake-test-job
variables:
<<: *cmake-env
SOURCE_PATH: runtime/cpp

test-cpp-common:
<<: *cmake-test-job
variables:
<<: *cmake-env
SOURCE_PATH: common/cpp

test-cpp-all:
<<: *cmake-test-job
variables:
<<: *cmake-env
FLETCHER_CPP: 1
FLETCHER_GEN: 1
FLETCHER_ECHO: 1

test-python-runtime:
<<: *cmake-test-job
variables:
<<: *cmake-env
FLETCHER_ECHO: 1
FLETCHER_TESTS: 0
FLETCHER_CPP: 1
FLETCHER_PYTHON: 1

build-cpp-echo-platform:
<<: *cmake-build-job
variables:
<<: *cmake-build-env
SOURCE_PATH: platforms/echo/runtime

build-cpp-aws-platform:
<<: *cmake-build-job
variables:
<<: *cmake-build-env
FLETCHER_AWS: 1
SOURCE_PATH: platforms/aws-f1/runtime

build-cpp-snap-platform:
<<: *cmake-build-job
variables:
<<: *cmake-build-env
FLETCHER_SNAP: 1
SOURCE_PATH: platforms/snap/runtime

build-cpp-fletchgen:
<<: *cmake-build-job
variables:
<<: *cmake-build-env
SOURCE_PATH: codegen/fletchgen

build-cpp-all:
<<: *cmake-build-job
variables:
<<: *cmake-build-env
FLETCHER_CPP: 1
FLETCHER_GEN: 1
FLETCHER_ECHO: 1

examples-fletchgen-stringread:
image: docker:latest
stage: examples
script:
- docker build -t fletchgen .
- docker run -v `pwd`/hardware/test/fletchgen/stringread:/src -v `pwd`/hardware:/hardware -e "FLETCHER_HARDWARE_DIR=/hardware" fletchgen -i src/test.fbs -o src/test_wrapper.vhd -n test -w test_wrapper -s src/test.fbs -d src/test.rb --sim src/sim_top.vhd -x src/test.srec
- sed -i -e 's/"src\/test.srec"/"src\/test\/fletchgen\/stringread\/test.srec"/' hardware/test/fletchgen/stringread/sim_top.vhd
- docker run -v `pwd`/hardware:/src $GHDL_IMAGE:$GHDL_TAG bash -c "shopt -s globstar && ghdl -i /src/**/*.vhd && ghdl -m --ieee=synopsys sim_top && ghdl -r -v --ieee=synopsys sim_top --stop-time=1ms"
42 changes: 0 additions & 42 deletions .travis.yml

This file was deleted.

6 changes: 5 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ cmake_minimum_required(VERSION 3.10)
# Tests for any subproject
option(FLETCHER_TESTS "Build tests." OFF)

if (FLETCHER_TESTS)
enable_testing()
endif()

########################################################################################################################
# Runtimes
########################################################################################################################
Expand Down Expand Up @@ -54,4 +58,4 @@ if (FLETCHER_GEN)
else ()
add_subdirectory(codegen/fletchgen)
endif ()
endif ()
endif ()
7 changes: 1 addition & 6 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ FROM mbrobbel/libarrow:$ARROW_VERSION

LABEL fletcher=

ENV BUILD_PACKAGES make cmake g++ libboost-all-dev
ENV RUNTIME_PACKAGES libboost-all-dev
ENV BUILD_PACKAGES cmake g++

WORKDIR fletcher
ADD . .
Expand All @@ -15,14 +14,10 @@ RUN apt-get update && \
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DFLETCHER_GEN=1 \
-DFLETCHER_TESTS=0 \
-DFLETCHER_AWS=0 \
-DFLETCHER_SNAP=0 \
.. && \
make && make install && \
cd ../.. && rm -rf fletcher && \
apt-get remove -y --purge $BUILD_PACKAGES && \
apt-get install -y $RUNTIME_PACKAGES && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*

Expand Down
14 changes: 7 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
# Fletcher: A framework to integrate FPGA accelerators with Apache Arrow

[![Build Status](https://travis-ci.org/johanpel/fletcher.svg?branch=master)](https://travis-ci.org/johanpel/fletcher)
[![pipeline status](https://gitlab.com/mbrobbel/fletcher/badges/master/pipeline.svg)](https://gitlab.com/mbrobbel/fletcher/commits/master)

Fletcher is a framework that helps to integrate FPGA accelerators with tools that use
Apache Arrow in their back-ends.

[Apache Arrow](https://arrow.apache.org/) specifies an in-memory format for data and
[Apache Arrow](https://arrow.apache.org/) specifies an in-memory format for data and
provides libraries to various languages to interface with the data in that format. The
format prevents the need for serialization and through the libraries Arrow, zero-copy
format prevents the need for serialization and through the libraries Arrow, zero-copy
interprocess communication is possible. Languages that have Arrow libraries (under development)
include C, C++, Go, Java, JavaScript, Python, Ruby and Rust.

While many software projects can benefit from these advantages, also hardware accelerated
applications have seen serious serialization bottlenecks. Fletcher focuses on FPGA accelerators only.
Through Fletcher and Arrow, efficient FPGA acceleration is made available to all the supported languages.

<sup><sub>(If you are looking for GPGPU's, some work is being done as part of the Arrow project already, go check out
<sup><sub>(If you are looking for GPGPU's, some work is being done as part of the Arrow project already, go check out
[the Arrow repository](https://github.com/apache/arrow).)</sub></sup>

Given an Arrow Schema (description of a tabular datastructure), Fletcher generates the following:
Expand Down Expand Up @@ -48,18 +48,18 @@ following features:
* [Amazon's EC2 f1](https://github.com/aws/aws-fpga) platform is supported.
* POWER8/9 CAPI 1.0/2.0 FPGA platform is supported through the use of [OpenPOWER's CAPI SNAP framework](https://github.com/open-power/snap).
* Our bus interconnect is similar to AXI, so it should be easy to integrate in many existing platforms.
* Output streams are AXI compatible as well, so it should be easy to integrate e.g. Vivado HLS with Fletcher
* Output streams are AXI compatible as well, so it should be easy to integrate e.g. Vivado HLS with Fletcher
output streams through the use of `hls::stream<type>` interfaces.

## Further reading
* [Fletcher wrapper generator](codegen/fletchgen) - The wrapper generator converts Arrow schema's to wrappers around ColumnReaders/ColumnWriters. It also provides instantiation templates for your hardware accelerator implementation.
* [How to use the host-side run-time libraries](runtime).

## Example projects
* [Simple column sum example](examples/sum).
* [How to generate a Column Reader](hardware) - A more flexible but low-level hardware design approach, using ColumnReader/ColumnWriters directly.
* [Regular Expression matching example](examples/regexp).

## External examples
* [Posit BLAS operations using Fletcher](https://github.com/lvandam/posit_blas_hdl)
* [Posit PairHMM using Fletcher](https://github.com/lvandam/pairhmm_posit_hdl_arrow)
Loading

0 comments on commit 0c79071

Please sign in to comment.