From ca1c8b2be67e62e8f459d5d854bbc255f03ae7bd Mon Sep 17 00:00:00 2001 From: Christian Semmler Date: Wed, 5 Mar 2025 15:25:39 -0700 Subject: [PATCH] Add `build-with-entropy` pipeline (#1384) * Try entropy build * Fix * Updates * Remove file * Fix * Add seed parameter to entropy.sh * echo SEED used * Try build pipeline changes * Add python setup * Fix pipeline * Increase number of samples, add entropy to isle and config * Try 32 samples * Empty commit for another CI run * Try 50 samples * Empty commit for another CI run * Empty commit for another CI run * Empty commit for another CI run * Empty commit for another CI run * Empty commit for another CI run * Empty commit for another CI run * Empty commit for another CI run * Empty commit for another CI run * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Trigger CI with 100 entropy samples * Update * Echo seed * Update text * Fix requirements.txt * Cancel previous CI workflows * Try only build,yml * Empty commit - should stop earlier build * Fix upload --- .github/workflows/build.yml | 144 +++++++++++++++++++++++++++++++++--- CMakeLists.txt | 13 ++++ tools/entropy.py | 45 +++++++++++ 3 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 tools/entropy.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eabeb295ea..7dfca4afc4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,10 @@ name: Build on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: fetch-deps: name: Download original binaries @@ -100,9 +104,104 @@ jobs: build/LEGO1.DLL build/LEGO1.PDB + build-with-entropy: + name: 'MSVC 4.20 with entropy' + needs: [fetch-deps] + runs-on: windows-latest + strategy: + matrix: + instance: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: actions/checkout@v4 + with: + repository: itsmattkc/msvc420 + path: msvc420 + + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v2 + with: + # Use minimum supported version + cmake-version: '3.15.x' + + - name: Patch MSVC 4.2 + run: | + tools/patch_c2.py msvc420/bin/C2.EXE + + - name: Generate Entropy + shell: bash + run: | + # Get the first 8 characters of the SHA (enough for a decent seed) + SHA_PREFIX=$(echo "${{ github.sha }}" | cut -c 1-8) + ENTROPY_SEED=$((16#$SHA_PREFIX + ${{ matrix.instance }})) + + echo "Using seed: $ENTROPY_SEED" + python3 tools/entropy.py $ENTROPY_SEED > entropy.h + + - name: Build + shell: cmd + run: | + call .\msvc420\bin\VCVARS32.BAT x86 + cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DISLE_INCLUDE_ENTROPY=ON -G "NMake Makefiles" + cmake --build build + + - name: Restore cached original binaries + id: cache-original-binaries + uses: actions/cache/restore@v4 + with: + enableCrossOsArchive: true + path: legobin + key: legobin + + - name: Install python packages + shell: bash + run: | + pip install -r tools/requirements.txt + + - name: Detect binaries + run: | + reccmp-project detect --what original --search-path legobin + reccmp-project detect --what recompiled --search-path build + + - name: Summarize Accuracy + shell: bash + run: | + reccmp-reccmp --target CONFIG --json CONFIGPROGRESS.json + reccmp-reccmp --target ISLE --json ISLEPROGRESS.json + reccmp-reccmp --target LEGO1 --json LEGO1PROGRESS.json + + - name: Upload Artifact + uses: actions/upload-artifact@main + with: + name: Win32-Entropy-${{ matrix.instance }} + path: | + CONFIGPROGRESS.json + ISLEPROGRESS.json + LEGO1PROGRESS.json + + merge-entropy-artifacts: + name: 'Merge entropy artifacts' + runs-on: ubuntu-latest + needs: build-with-entropy + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: Win32-Entropy + pattern: Win32-Entropy-* + separate-directories: true + compare: name: Compare with master - needs: [build, fetch-deps] + needs: [build, merge-entropy-artifacts, fetch-deps] runs-on: windows-latest steps: - uses: actions/checkout@main @@ -113,8 +212,13 @@ jobs: - uses: actions/download-artifact@main with: - name: Win32 - path: build + name: Win32 + path: build + + - uses: actions/download-artifact@main + with: + name: Win32-Entropy + path: build-entropy - name: Restore cached original binaries id: cache-original-binaries @@ -156,6 +260,28 @@ jobs: reccmp-reccmp --target ISLE --diff ISLEPROGRESS-old.json || echo "Current master not found" reccmp-reccmp --target LEGO1 --diff LEGO1PROGRESS-old.json || echo "Current master not found" + - name: Aggregate Accuracy + shell: bash + run: | + reccmp-aggregate --samples $(find build-entropy -type f -name "CONFIGPROGRESS.json") --output CONFIGPROGRESS-agg.json + reccmp-aggregate --samples $(find build-entropy -type f -name "ISLEPROGRESS.json") --output ISLEPROGRESS-agg.json + reccmp-aggregate --samples $(find build-entropy -type f -name "LEGO1PROGRESS.json") --output LEGO1PROGRESS-agg.json + + - name: Compare Aggregate Accuracy With Current Master + shell: bash + env: + RELEASE_URL: https://github.com/isledecomp/isle/releases/download/continuous + run: | + # Download the current master state + curl -fLSs -o CONFIGPROGRESS-agg-old.json $RELEASE_URL/CONFIGPROGRESS-agg.json || echo "" >CONFIGPROGRESS-agg-old.json + curl -fLSs -o ISLEPROGRESS-agg-old.json $RELEASE_URL/ISLEPROGRESS-agg.json || echo "" >ISLEPROGRESS-agg-old.json + curl -fLSs -o LEGO1PROGRESS-agg-old.json $RELEASE_URL/LEGO1PROGRESS-agg.json || echo "" >LEGO1PROGRESS-agg-old.json + + # Compare with current master + reccmp-aggregate --diff CONFIGPROGRESS-agg-old.json CONFIGPROGRESS-agg.json || echo "Current master not found" + reccmp-aggregate --diff ISLEPROGRESS-agg-old.json ISLEPROGRESS-agg.json || echo "Current master not found" + reccmp-aggregate --diff LEGO1PROGRESS-agg-old.json LEGO1PROGRESS-agg.json || echo "Current master not found" + - name: Test Exports shell: bash run: | @@ -180,9 +306,9 @@ jobs: with: name: Accuracy Report path: | - CONFIGPROGRESS.* - ISLEPROGRESS.* - LEGO1PROGRESS.* + CONFIGPROGRESS* + ISLEPROGRESS* + LEGO1PROGRESS* upload: name: Upload artifacts @@ -212,9 +338,9 @@ jobs: build/CONFIG.EXE \ build/ISLE.EXE \ build/LEGO1.DLL \ - CONFIGPROGRESS.* \ - ISLEPROGRESS.* \ - LEGO1PROGRESS.* + CONFIGPROGRESS* \ + ISLEPROGRESS* \ + LEGO1PROGRESS* curl -X POST -F key=$UPLOAD_KEY -F 'file=@CONFIGPROGRESS.SVG' https://legoisland.org/progress/ curl -X POST -F key=$UPLOAD_KEY -F 'file=@ISLEPROGRESS.SVG' https://legoisland.org/progress/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b337e0a4c..d66a170653 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,7 @@ option(ISLE_DECOMP_ASSERT "Assert struct size" ${MSVC_FOR_DECOMP}) cmake_dependent_option(ISLE_USE_DX5_LIBS "Build with internal DirectX 5 SDK Libraries" ON ISLE_USE_DX5 OFF) option(ISLE_BUILD_LEGO1 "Build LEGO1.DLL library" ON) option(ISLE_BUILD_BETA10 "Build BETA10.DLL library" OFF) +option(ISLE_INCLUDE_ENTROPY "Build with entropy.h" OFF) if(NOT (ISLE_BUILD_LEGO1 OR ISLE_BUILD_BETA10)) message(FATAL_ERROR "ISLE_BUILD_LEGO1 AND ISLE_BUILD_BETA10 cannot be both disabled") @@ -584,6 +585,18 @@ if (MSVC_FOR_DECOMP) set_property(TARGET isle ${lego1_targets} PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") endif() + if (ISLE_INCLUDE_ENTROPY) + foreach(tgt IN LISTS lego1_targets beta10_targets) + target_compile_options(${tgt} PRIVATE /FI${PROJECT_SOURCE_DIR}/entropy.h) + endforeach() + if (TARGET isle) + target_compile_options(isle PRIVATE /FI${PROJECT_SOURCE_DIR}/entropy.h) + endif() + if (TARGET config) + target_compile_options(config PRIVATE /FI${PROJECT_SOURCE_DIR}/entropy.h) + endif() + endif() + if(TARGET lego1) target_link_options(lego1 PRIVATE "/OPT:REF") # Equivalent to target_compile_options(... PRIVATE "/MT$<$:d>") diff --git a/tools/entropy.py b/tools/entropy.py new file mode 100644 index 0000000000..a98a98b894 --- /dev/null +++ b/tools/entropy.py @@ -0,0 +1,45 @@ +import random +import string +import sys + +# Parameters for tweaking: +MAX_CLASSES = 10 +MAX_FUNC_PER_CLASS = 10 + +# Only the unique suffix, not counting "Class" or "Function" +CLASS_NAME_LEN = 6 +FUNC_NAME_LEN = 8 + + +def random_camel_case(length: int) -> str: + """Return a random string with first letter capitalized.""" + return "".join( + [ + random.choice(string.ascii_uppercase), + *random.choices(string.ascii_lowercase, k=length - 1), + ] + ) + + +# If the first parameter is an integer, use it as the seed. +try: + seed = int(sys.argv[1]) +except (IndexError, ValueError): + seed = random.randint(0, 10000) + +random.seed(seed) + +print(f"// Seed: {seed}\n") + +num_classes = random.randint(1, MAX_CLASSES) +for i in range(num_classes): + class_name = "Class" + random_camel_case(CLASS_NAME_LEN) + print(f"class {class_name} {{") + num_functions = random.randint(1, MAX_FUNC_PER_CLASS) + for j in range(num_functions): + function_name = "Function" + random_camel_case(FUNC_NAME_LEN) + print(f"\tinline void {function_name}() {{}}") + + print(f"}};\n") + +print()