Skip to content

Commit

Permalink
Merge #79: Vendor 0.21.2
Browse files Browse the repository at this point in the history
12d7ec3 Fix Bitcoin Core 0.21.2 upgrade (Tobin C. Harding)
a4f30ef Vendor Bitcoin Core v0.21.2 (Tobin C. Harding)
8b3d23f Update README about vendoring (Tobin C. Harding)
3ca86b9 Remove the depend directory (Tobin C. Harding)
619611e Add Bitcoin Core vendor script (Tobin C. Harding)

Pull request description:

  This is #77 without the version bump patch.

  Upgrade to a new way of vendoring Core and vendor version 0.21.2 - note please this will be directly followed by an upgrade to 0.21-final. Done separately to proof out the workflow of such an upgrade.

ACKs for top commit:
  apoelstra:
    ACK 12d7ec3

Tree-SHA512: f708b4bda1a209afe7552ec0e691ed104847d4a4a5fae6cd13f2da8f3a205c0a652ced8883900b9ce312cb8f4f39f77f0abbdf3372016c5af7aeddc3e33984ba
  • Loading branch information
apoelstra committed Jan 8, 2024
2 parents f6a4db5 + 12d7ec3 commit 875511c
Show file tree
Hide file tree
Showing 1,180 changed files with 91,997 additions and 36,442 deletions.
14 changes: 3 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,10 @@ For example, if we upgrade the Bitcoin Core code by a `Patch` version we also bu
One side effect of this is that `crates.io` shows our release versions in yellow as if they were pre-release versions, this is due to us using a `-` which, in semantic versioning, implies a pre-release version.


## Bitcoin Core subtree
## Vendor Bitcoin Core

We use a git subtree to vendor the Bitcoin Core code. This can be seen from the following commits that were created using `git subtree add --prefix='depend/bitcoin' git@github.com:bitcoin/bitcoin.git v0.19.2 --squash`.
```
f751613e62 Squashed 'depend/bitcoin/' content from commit 204cc0f575
264282188a Merge commit 'f751613e6203770fa94143b9aba1d116512f0ce7' as 'depend/bitcoin'
```

To use a later version of Bitcoin Core, for example, v0.20.2
```
git subtree pull --prefix='depend/bitcoin' git@github.com:bitcoin/bitcoin.git v0.20.2 --squash
```
We use a script to vendor the Bitcoin Core code, the script takes the
Bitcoin Core version number to vendor: `./vendor-bitcoin-core.sh 0.21.1`


## MSRV
Expand Down
38 changes: 25 additions & 13 deletions build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,22 @@ fn main() {
let is_big_endian = env::var("CARGO_CFG_TARGET_ENDIAN").expect("No endian is set") == "big";
let mut base_config = cc::Build::new();
base_config
.cpp(true)
.include("depend/bitcoin/src")
.include("depend/bitcoin/src/secp256k1/include")
.define("__STDC_FORMAT_MACROS", None)
.flag_if_supported("-Wno-implicit-fallthrough");

if target.contains("windows") {
base_config.define("WIN32", "1");
}

let mut secp_config = base_config.clone();
let mut consensus_config = base_config;

// **Secp256k1**
if !cfg!(feature = "external-secp") {
base_config
secp_config
.include("depend/bitcoin/src/secp256k1")
.include("depend/bitcoin/src/secp256k1/src")
.flag_if_supported("-Wno-unused-function") // some ecmult stuff is defined but not used upstream
.define("SECP256K1_BUILD", "1")
// Bitcoin core defines libsecp to *not* use libgmp.
Expand All @@ -39,34 +45,40 @@ fn main() {
.define("USE_SCALAR_INV_BUILTIN", "1")
// Technically libconsensus doesn't require the recovery feautre, but `pubkey.cpp` does.
.define("ENABLE_MODULE_RECOVERY", "1")
.define("ECMULT_WINDOW_SIZE", "15")
.define("ECMULT_GEN_PREC_BITS", "4")
.define("ENABLE_MODULE_SCHNORRSIG", "1")
.define("ENABLE_MODULE_EXTRAKEYS", "1")
// The actual libsecp256k1 C code.
.file("depend/bitcoin/src/secp256k1/src/secp256k1.c");

if is_big_endian {
base_config.define("WORDS_BIGENDIAN", "1");
secp_config.define("WORDS_BIGENDIAN", "1");
}

if use_64bit_compilation {
base_config
secp_config
.define("USE_FIELD_5X52", "1")
.define("USE_SCALAR_4X64", "1")
.define("HAVE___INT128", "1");
} else {
base_config.define("USE_FIELD_10X26", "1").define("USE_SCALAR_8X32", "1");
secp_config.define("USE_FIELD_10X26", "1").define("USE_SCALAR_8X32", "1");
}

secp_config.compile("libsecp256k1.a");
}

let tool = base_config.get_compiler();
let tool = consensus_config.get_compiler();
if tool.is_like_msvc() {
base_config.flag("/std:c++14").flag("/wd4100");
consensus_config.flag("/std:c++17").flag("/wd4100");
} else if tool.is_like_clang() || tool.is_like_gnu() {
base_config.flag("-std=c++11").flag("-Wno-unused-parameter");
consensus_config.flag("-std=c++17").flag("-Wno-unused-parameter");
}

if target.contains("windows") {
base_config.define("WIN32", "1");
}
base_config
consensus_config
.cpp(true)
.include("depend/bitcoin/src")
.include("depend/bitcoin/src/secp256k1/include")
.file("depend/bitcoin/src/util/strencodings.cpp")
.file("depend/bitcoin/src/uint256.cpp")
.file("depend/bitcoin/src/pubkey.cpp")
Expand Down
119 changes: 119 additions & 0 deletions contrib/vendor-bitcoin-core.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -e

function usage() {
echo
echo "Usage: script OPTIONS [bitcoin-core-version]"
echo
echo "OPTIONS:"
echo
echo " -f Vendor even if there are local changes to the rust-bitcoinconsensus git index"
echo " -h Print this help an exit"
echo
echo "Example:"
echo
echo " vendor-bitcoin-core v0.21.2"
echo

exit 0
}

if (($# < 1)) || [ "$1" == '-h' ]; then
usage
fi

# Set default variables

if [ -z "$CORE_VENDOR_GIT_ROOT" ]; then
CORE_VENDOR_GIT_ROOT="$(git rev-parse --show-toplevel)"
else
CORE_VENDOR_GIT_ROOT="$(realpath "$CORE_VENDOR_GIT_ROOT")"
fi

DEFAULT_DEPEND_DIR="depend"
DEFAULT_CORE_REPO=https://github.com/bitcoin/bitcoin.git

: "${CORE_VENDOR_DEPEND_DIR:=$DEFAULT_DEPEND_DIR}"
: "${CORE_VENDOR_REPO:=$DEFAULT_CORE_REPO}"

# CP_NOT_CLONE lets us just copy a directory rather than git cloning.
# This is usually a bad idea, since it will bring in build artifacts or any other
# junk from the source directory, but may be useful during development or CI.
: "${CORE_VENDOR_CP_NOT_CLONE:=no}"

echo "Using depend directory $CORE_VENDOR_DEPEND_DIR. Set CORE_VENDOR_DEPEND_DIR to override."
echo "Using bitcoin repository $CORE_VENDOR_REPO. Set CORE_VENDOR_REPO to override."

# Parse command-line options
CORE_REV=""
FORCE=no
while (( "$#" )); do
case "$1" in
-h)
echo ""
usage
;;
-f)
FORCE=yes
;;
*)
if [ -z "$CORE_REV" ]; then
CORE_REV="$1"
else
echo "WARNING: ignoring unknown command-line argument $1"
fi
;;
esac
shift
done

echo
if [ "$CORE_REV" ]; then
echo "Vendoring Bitcoin Core version: $CORE_REV"
else
echo "WARNING: No Bitcoin Core revision specified. Will use whatever we find at the git repo."
fi
echo

# Check if we will do anything destructive.

if [ "$FORCE" == "no" ]; then
if ! git diff --quiet -- "*.rs"; then
echo "ERROR: There appear to be modified source files. Check these in or pass -f (some source files will be modified to have symbols renamed)."
exit 2
fi
if ! git diff --quiet -- "$CORE_VENDOR_DEPEND_DIR"; then
echo "ERROR: The depend directory appears to be modified. Check it in or pass -f (this directory will be deleted)."
exit 2
fi
fi

DIR=./bitcoin

pushd "$CORE_VENDOR_DEPEND_DIR" > /dev/null
rm -rf "$DIR" || true

# Clone the repo. As a special case, if the repo is a local path and we have
# not specified a revision, just copy the directory rather than using 'git clone'.
# This lets us use non-git repos or dirty source trees as secp sources.
if [ "$CORE_VENDOR_CP_NOT_CLONE" == "yes" ]; then
cp -r "$CORE_VENDOR_REPO" "$DIR"
chmod -R +w "$DIR" # cp preserves write perms, which if missing will cause patch to fail
else
git clone "$CORE_VENDOR_REPO" "$DIR"
fi

# Check out specified revision
pushd "$DIR" > /dev/null
if [ -n "$CORE_REV" ]; then
git checkout "$CORE_REV"
fi
SOURCE_REV=$(git rev-parse HEAD || echo "[unknown revision from $CORE_VENDOR_REPO]")
rm -rf .git/ || true
popd

# Record revision
echo "# This file was automatically created by $(basename "$0")" > ./bitcoin-HEAD-revision.txt
echo "$SOURCE_REV" >> ./bitcoin-HEAD-revision.txt

popd > /dev/null
2 changes: 2 additions & 0 deletions depend/bitcoin-HEAD-revision.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# This file was automatically created by vendor-bitcoin-core.sh
af591f2068d0363c92d9756ca39c43db85e5804c
28 changes: 13 additions & 15 deletions depend/bitcoin/.appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,29 +7,24 @@ clone_depth: 5
environment:
PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%'
PYTHONUTF8: 1
QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt598x64_vs2019_v1681/qt598_x64_vs2019_1681.zip'
QT_DOWNLOAD_HASH: '00cf7327818c07d74e0b1a4464ffe987c2728b00d49d4bf333065892af0515c3'
QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
VCPKG_COMMIT_ID: 'f3f329a048eaff759c1992c458f2e12351486bc7'
VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f'
install:
# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
# - cmd: pip install zmq
# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is:
# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is:
# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary,
# b. Install the missing packages.
# b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins).
# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed
# as part of the msbuild command using vcpkg mainfests.
- ps: |
$env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt
Write-Host "vcpkg installing packages: $env:PACKAGES"
cd c:\tools\vcpkg
$env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead.
git pull origin master > $null
git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID
git -c advice.detachedHead=false checkout $env:VCPKG_TAG
.\bootstrap-vcpkg.bat > $null
Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)"
.\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null
Write-Host "vcpkg packages installed successfully."
.\vcpkg integrate install
cd "$env:APPVEYOR_BUILD_FOLDER"
before_build:
# Powershell block below is to download and extract the Qt static libraries. The pseudo code is:
Expand All @@ -55,11 +50,14 @@ after_build:
#- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe
test_script:
- cmd: src\test_bitcoin.exe -l test_suite
- cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL
- cmd: src\bench_bitcoin.exe > NUL
- ps: python test\util\bitcoin-util-test.py
- cmd: python test\util\rpcauth-test.py
# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted.
- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
# functional tests disabled for now. See
# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202
# https://github.com/bitcoin/bitcoin/issues/18623
# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
artifacts:
#- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip
deploy: off
Loading

0 comments on commit 875511c

Please sign in to comment.