Support vcpkg dependency management (#69)

First step towards transitioning away from cxx-common's `pkgman.py` dependency management system towards compatibility with and use of vcpkg (https://github.com/microsoft/vcpkg) to manage dependencies.

This commit attempts to support both the new and old build systems until we can phase out the old completely.

Please see the updated remill and cxx-common repositories to learn how to fetch the required dependencies.

.github/workflows/vcpkg_ci.yml

@@ -0,0 +1,104 @@
+name: VCPKG Continuous Integration
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  schedule:
+    # run CI every day even if no PRs/merges occur
+    - cron:  '0 6 * * *'
+
+jobs:
+  build_linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - { name: 'ubuntu', tag: '18.04' }
+          - { name: 'ubuntu', tag: '20.04' }
+        llvm: [
+          '9',
+          '10',
+          '11'
+          ]
+
+    runs-on: ubuntu-20.04
+    container:
+      image: docker.pkg.github.com/trailofbits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install utility tools
+        shell: bash
+        run: |
+          # TODO some of these should probably live in the Docker build image
+          apt-get update
+          apt-get install -y pixz xz-utils make
+      - name: Build and install remill
+        shell: bash
+        run: |
+          # Change to master after testing
+          git clone --depth=1 --branch master https://github.com/lifting-bits/remill.git
+          cd remill
+          ./scripts/build.sh --llvm-version ${{ matrix.llvm }}
+          cmake --build remill-build --target install
+      - name: Build anvill
+        shell: bash
+        run: |
+          mkdir build && cd build
+          cmake -DCMAKE_VERBOSE_MAKEFILE=ON ..
+          cmake --build . --target install -- -j "$(nproc)"
+      - name: Smoke Test
+        shell: bash
+        run: |
+          anvill-decompile-json-${{ matrix.llvm }}.0 -spec ./examples/ret0.json -bc_out ./ret0.bc -ir_out ret0.ir
+      - name: Tests
+        shell: bash
+        working-directory: ci
+        run: |
+          ./install_binja.sh
+          ./install_clang.sh
+          ./run_tests.sh
+        env:
+          BINJA_DECODE_KEY: ${{ secrets.BINJA_DECODE_KEY }}
+          LLVM_VERSION: "llvm${{ matrix.llvm }}0"
+
+  build_mac:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [
+          'macos-10.15',
+          'macos-11.0'
+          ]
+        llvm: [
+          '9',
+          '10',
+          '11'
+          ]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build and install remill
+        shell: bash
+        run: |
+          # Change to master after testing
+          git clone --branch master https://github.com/lifting-bits/remill.git
+          cd remill
+          ./scripts/build.sh --llvm-version ${{ matrix.llvm }}
+          cmake --build remill-build --target install
+      - name: Build anvill
+        shell: bash
+        run: |
+          mkdir build && cd build
+          cmake -DCMAKE_VERBOSE_MAKEFILE=ON ..
+          cmake --build . --target install -- -j "$(sysctl -n hw.logicalcpu)"
+      - name: Smoke Test
+        shell: bash
+        run: |
+          anvill-decompile-json-${{ matrix.llvm }}.0 -spec ./examples/ret0.json -bc_out ./ret0.bc -ir_out ret0.ir

CMakeLists.txt

@@ -13,8 +13,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+if (NOT DEFINED ENV{TRAILOFBITS_LIBRARIES})
+  message(STATUS "Using new vcpkg build system")
+  include(CMakeLists_vcpkg.txt)
+  return()
+endif()
+
 project(anvill)
-cmake_minimum_required(VERSION 3.2)
+cmake_minimum_required(VERSION 3.14)
 
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake")
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake")

CMakeLists_vcpkg.txt

@@ -0,0 +1,237 @@
+# Copyright (c) 2020 Trail of Bits, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Find remill first because its config has useful dependency-finding info that
+# needs to be found before the CMake `project` declaration
+find_package(remill COMPONENTS VCPKG_DEPS QUIET)
+
+include(cmake/vcpkg_helper.cmake)
+
+project(anvill)
+cmake_minimum_required(VERSION 3.14)
+include(GNUInstallDirs)
+enable_language(C CXX ASM)
+
+include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/settings.cmake")
+include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/utils.cmake")
+
+FindAndSelectClangCompiler()
+
+macro(target_public_headers TARGET)
+  set_target_properties(${TARGET} PROPERTIES PUBLIC_HEADER "${ARGN}")
+endmacro()
+
+option(ANVILL_INSTALL_PYTHON2_LIBS "Install Python 2 libraries")
+option(ANVILL_INSTALL_PYTHON3_LIBS "Install Python 3 libraries" ON)
+
+#
+# libraries
+#
+find_package(remill CONFIG REQUIRED)
+
+#
+# target settings
+#
+
+set(ANVILL anvill-${REMILL_LLVM_VERSION})
+
+add_subdirectory(lib/Version)
+
+add_library(${ANVILL} STATIC
+  include/anvill/Type.h
+
+  lib/Arch/AArch64_C.cpp
+  lib/Arch/SPARC32_C.cpp
+  lib/Arch/SPARC64_C.cpp
+  lib/Arch/X86_C.cpp
+  lib/Arch/X86_FastCall.cpp
+  lib/Arch/X86_StdCall.cpp
+  lib/Arch/X86_ThisCall.cpp
+  lib/Arch/X86_64_SysV.cpp
+
+  lib/Arch/AllocationState.h
+  lib/Arch/AllocationState.cpp
+
+  lib/Arch/Arch.h
+  lib/Arch/Arch.cpp
+
+  include/anvill/TypeParser.h
+  lib/TypeParser.cpp
+
+  include/anvill/TypePrinter.h
+  lib/TypePrinter.cpp
+
+  include/anvill/Program.h
+  lib/Program.cpp
+
+  include/anvill/Decl.h
+  lib/Decl.cpp
+
+  include/anvill/Lift.h
+  lib/Lift.cpp
+
+  include/anvill/MCToIRLifter.h
+  lib/MCToIRLifter.cpp
+
+  include/anvill/Optimize.h
+  lib/Optimize.cpp
+
+  include/anvill/Analyze.h
+  lib/Analyze.cpp
+
+  include/anvill/Util.h
+  lib/Util.cpp)
+
+target_include_directories(${ANVILL} PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:include>
+  )
+
+target_link_libraries(${ANVILL} PUBLIC remill_settings remill AnvillVersion)
+
+target_public_headers(${ANVILL}
+  include/anvill/Analyze.h
+  include/anvill/Decl.h
+  include/anvill/Lift.h
+  include/anvill/Optimize.h
+  include/anvill/Program.h
+  include/anvill/Type.h
+  include/anvill/TypeParser.h
+  include/anvill/TypePrinter.h
+  include/anvill/Version.h
+  include/anvill/Util.h)
+
+#
+# tool target settings
+#
+
+set(DECOMPILE_JSON anvill-decompile-json-${REMILL_LLVM_VERSION})
+
+add_executable(${DECOMPILE_JSON} bin/JSON.cpp)
+target_link_libraries(${DECOMPILE_JSON} PRIVATE ${ANVILL})
+
+
+set(SPECIFY_BITCODE anvill-specify-bitcode-${REMILL_LLVM_VERSION})
+
+add_executable(${SPECIFY_BITCODE} bin/Bitcode.cpp)
+target_link_libraries(${SPECIFY_BITCODE} PRIVATE ${ANVILL})
+
+set(ANVILL_PYTHON_SOURCES
+  setup.py
+  python/anvill/__init__.py
+  python/anvill/__main__.py
+  python/anvill/arch.py
+  python/anvill/binja.py
+  python/anvill/exc.py
+  python/anvill/function.py
+  python/anvill/ida.py
+  python/anvill/loc.py
+  python/anvill/mem.py
+  python/anvill/os.py
+  python/anvill/program.py
+  python/anvill/type.py
+  python/anvill/var.py)
+
+if(NOT DEFINED ENV{VIRTUAL_ENV})
+  # NOT a venv install, specify --user and --prefix
+  set(extra_install_flags --force --user --prefix=)
+else()
+  #virtual env; install normally
+  set(extra_install_flags "")
+endif()
+
+if(ANVILL_INSTALL_PYTHON2_LIBS)
+  add_custom_target(install_anvill_python2
+    DEPENDS ${ANVILL_PYTHON_SOURCES})
+
+  add_custom_command(
+    TARGET install_anvill_python2 POST_BUILD
+    COMMAND which python2 && python2 setup.py install ${extra_install_flags}
+    COMMENT "Installing Anvill Python 2 API"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+
+  add_dependencies(${ANVILL}
+    install_anvill_python2)
+endif()
+
+if(ANVILL_INSTALL_PYTHON3_LIBS)
+  add_custom_target(install_anvill_python3
+    DEPENDS ${ANVILL_PYTHON_SOURCES})
+
+  add_custom_command(
+    TARGET install_anvill_python3 POST_BUILD
+    COMMAND which python3 && python3 setup.py install ${extra_install_flags}
+    COMMENT "Installing Anvill Python 3 API"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+
+  add_dependencies(${ANVILL}
+    install_anvill_python3)
+endif()
+
+#
+# install settings
+#
+
+install(TARGETS ${ANVILL} EXPORT anvillTargets
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/anvill)
+
+install(TARGETS ${DECOMPILE_JSON})
+
+set(ROUNDTRIP_TEST
+  scripts/roundtrip.py)
+
+install(
+  FILES ${ROUNDTRIP_TEST}
+  DESTINATION "${CMAKE_INSTALL_DATADIR}/anvill")
+
+#NOTE(artem): Uncomment and add tests as we pass them
+set(ROUNDTRIP_TEST_FILES
+  #tests/assert.c
+  #tests/bitops.c
+  tests/cast.c
+  #tests/funcptr.c
+  #tests/nullptr.c
+  tests/ret0.c
+  #tests/zeroinit.c
+)
+
+install(
+  FILES ${ROUNDTRIP_TEST_FILES}
+  DESTINATION "${CMAKE_INSTALL_DATADIR}/anvill/tests")
+
+configure_file(
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/anvillConfig_vcpkg.cmake.in"
+  "${CMAKE_CURRENT_BINARY_DIR}/anvillConfig.cmake"
+  @ONLY
+)
+
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/anvillConfig.cmake"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/anvill"
+)
+
+install(EXPORT anvillTargets
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/anvill")
+
+#
+# tests
+#
+# Note: the Binary Ninja or IDA Python API must exist to run tests.
+
+enable_testing()
+
+add_test(NAME test_roundtrip
+  COMMAND ${PROJECT_SOURCE_DIR}/scripts/roundtrip.py $<TARGET_FILE:${DECOMPILE_JSON}> ${PROJECT_SOURCE_DIR}/tests ${CMAKE_C_COMPILER}
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)

Dockerfile

@@ -1,4 +1,4 @@
-ARG LLVM_VERSION=900
+ARG LLVM_VERSION=1000
 ARG ARCH=amd64
 ARG UBUNTU_VERSION=18.04
 ARG DISTRO_BASE=ubuntu${UBUNTU_VERSION}

README.md

@@ -25,12 +25,12 @@ Anvill is supported on Linux platforms and has been tested on Ubuntu 18.04 and 2
 
 ## Dependencies
 
-Most of Anvill's dependencies can be provided by the [cxx-common](https://github.com/trailofbits/cxx-common) repository. Trail of Bits hosts downloadable, pre-built versions of cxx-common, which makes it substantially easier to get up and running with Anvill. Nonetheless, the following table represents most of Anvill's dependencies.
+Most of Anvill's dependencies can be provided by the [cxx-common](https://github.com/trailofbits/cxx-common) repository. Trail of Bits hosts downloadable, pre-built versions of the libraries for select operating systems, which makes it substantially easier to get up and running with Anvill. Nonetheless, the following table represents most of Anvill's dependencies.
 
 | Name | Version |
 | ---- | ------- |
 | [Git](https://git-scm.com/) | Latest |
-| [CMake](https://cmake.org/) | 3.2+ |
+| [CMake](https://cmake.org/) | 3.14+ |
 | [Clang](http://clang.llvm.org/) | 8.0+|
 | [Remill](https://github.com/lifting-bits/remill) | Latest |
 | [Python](https://www.python.org/) | 3.8 |
@@ -71,13 +71,15 @@ git clone https://github.com/lifting-bits/anvill.git
 # create a build dir
 mkdir anvill-build && cd anvill-build
 # configure
-CC=clang cmake ../anvill 
+CC=clang cmake ../anvill
 # build
 make -j 5
 # install
 sudo make install
 ```
 
+Or you can tell CMake where to find the remill installation prefix by passing `-Dremill_DIR="<remill_prefix>/lib/cmake/remill"` during configuration.
+
 Anvill's python plugins provide functionality needed to generate a JSON specification that contains information about the contents of a binary.
 These depend on tools like [IDA Pro](https://www.hex-rays.com/products/ida) or [Binary Ninja](https://binary.ninja/) for various analysis tasks.