From 5220789d8016cf4783c48bac32f632b816ada146 Mon Sep 17 00:00:00 2001 From: disinvite Date: Sun, 23 Feb 2025 15:46:44 -0500 Subject: [PATCH 1/2] Download and use 16-bit skifree for NE tests --- .github/workflows/legobin.yml | 28 ++++++++++++++++++--------- .github/workflows/unittest.yml | 14 +++++++++++--- reccmp/isledecomp/formats/__init__.py | 1 + reccmp/isledecomp/formats/ne.py | 2 +- tests/conftest.py | 28 +++++++++++++++++++++++++-- tests/test_image_ne.py | 14 ++++++++++++++ 6 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 tests/test_image_ne.py diff --git a/.github/workflows/legobin.yml b/.github/workflows/legobin.yml index d9b42bc2..8c92cee3 100644 --- a/.github/workflows/legobin.yml +++ b/.github/workflows/legobin.yml @@ -4,29 +4,39 @@ on: workflow_call: jobs: - fetch: + lego: runs-on: ubuntu-latest steps: - - name: Restore cached original binaries - id: cache-original-binaries - uses: actions/cache/restore@v3 + - name: Cache original binaries + id: cache-lego + uses: actions/cache@v3 with: enableCrossOsArchive: true path: legobin key: legobin - name: Download original island binaries - if: ${{ !steps.cache-original-binaries.outputs.cache-hit }} + if: ${{ !steps.cache-lego.outputs.cache-hit }} run: | wget https://legoisland.org/download/CONFIG.EXE --directory-prefix=legobin wget https://legoisland.org/download/ISLE.EXE --directory-prefix=legobin wget https://legoisland.org/download/LEGO1.DLL --directory-prefix=legobin + skifree: + runs-on: ubuntu-latest + steps: + - name: Cache original binaries - if: ${{ !steps.cache-original-binaries.outputs.cache-hit }} - uses: actions/cache/save@v3 + id: cache-skifree + uses: actions/cache@v3 with: enableCrossOsArchive: true - path: legobin - key: legobin + path: skifree + key: skifree + + - name: Download original ski binaries + if: ${{ !steps.cache-skifree.outputs.cache-hit }} + run: | + wget https://archive.org/download/win3_SKIFREE/SKIFREE.ZIP + unzip -d skifree SKIFREE.ZIP diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 949e7648..a5614283 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -28,14 +28,22 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Restore cached binary files - id: cache-original-binaries + - name: Restore lego files + id: cache-lego uses: actions/cache/restore@v3 with: enableCrossOsArchive: true path: legobin key: legobin + - name: Restore skifree files + id: cache-skifree + uses: actions/cache/restore@v3 + with: + enableCrossOsArchive: true + path: skifree + key: skifree + - name: Install python libraries shell: bash run: | @@ -44,4 +52,4 @@ jobs: - name: Run unit tests shell: bash run: | - pytest . --lego1=legobin/LEGO1.DLL \ No newline at end of file + pytest . --lego1=legobin/LEGO1.DLL --ski=skifree/SKI.EXE \ No newline at end of file diff --git a/reccmp/isledecomp/formats/__init__.py b/reccmp/isledecomp/formats/__init__.py index 4dbd3192..21a85aa7 100644 --- a/reccmp/isledecomp/formats/__init__.py +++ b/reccmp/isledecomp/formats/__init__.py @@ -1,4 +1,5 @@ from .detect import detect_image from .image import Image from .mz import MZImage +from .ne import NEImage from .pe import PEImage diff --git a/reccmp/isledecomp/formats/ne.py b/reccmp/isledecomp/formats/ne.py index 0284effe..c73c5d88 100644 --- a/reccmp/isledecomp/formats/ne.py +++ b/reccmp/isledecomp/formats/ne.py @@ -100,7 +100,7 @@ class NewExeHeader: def from_memory(cls, data: bytes, offset: int) -> tuple["NewExeHeader", int]: if not cls.taste(data, offset): raise ValueError - struct_fmt = "<2s2BHI17HI3H2B4H" + struct_fmt = "<2s2B2HI16HI3H2B4H" struct_size = struct.calcsize(struct_fmt) # fmt: off items: tuple[bytes, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int] = ( diff --git a/tests/conftest.py b/tests/conftest.py index e5ce7c2c..ca6affd0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,17 +3,22 @@ from typing import Iterator import pytest -from reccmp.isledecomp import PEImage, detect_image +from reccmp.isledecomp import NEImage, PEImage, detect_image def pytest_addoption(parser): - """Allow the option to run tests against the original LEGO1.DLL.""" + """Allow the option to run tests against sample binaries.""" parser.addoption("--lego1", action="store", help="Path to LEGO1.DLL") + parser.addoption("--ski", action="store", help="Path to SKI.EXE") # LEGO1.DLL: v1.1 English, September LEGO1_SHA256 = "14645225bbe81212e9bc1919cd8a692b81b8622abb6561280d99b0fc4151ce17" +# SkiFree 1.0 +# https://ski.ihoc.net/ +SKI_SHA256 = "0b97b99fcf34af5f5d624080417c79c7d36ae11351a7870ce6e0a476f03515c2" + @pytest.fixture(name="binfile", scope="session") def fixture_binfile(pytestconfig) -> Iterator[PEImage]: @@ -32,3 +37,22 @@ def fixture_binfile(pytestconfig) -> Iterator[PEImage]: image = detect_image(filename) assert isinstance(image, PEImage) yield image + + +@pytest.fixture(name="skifree", scope="session") +def fixture_skifree(pytestconfig) -> Iterator[NEImage]: + filename = pytestconfig.getoption("--ski") + + # Skip this if we have not provided the path to SKI.EXE. + if filename is None: + pytest.skip(allow_module_level=True, reason="No path to SKI") + + filename = Path(filename) + with filename.open("rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + if digest != SKI_SHA256: + pytest.fail(reason="Did not match expected SKI.EXE") + + image = detect_image(filename) + assert isinstance(image, NEImage) + yield image diff --git a/tests/test_image_ne.py b/tests/test_image_ne.py new file mode 100644 index 00000000..6bd251c4 --- /dev/null +++ b/tests/test_image_ne.py @@ -0,0 +1,14 @@ +from reccmp.isledecomp.formats import NEImage +from reccmp.isledecomp.formats.ne import NESegmentFlags, NETargetOSFlags + + +def test_vitals(skifree: NEImage): + # Linker version 5.5 + assert (skifree.header.ne_ver, skifree.header.ne_rev) == (5, 5) + assert skifree.header.ne_enttab == 0x526 + assert skifree.header.ne_cbenttab == 0x88 + assert skifree.header.ne_heap == 0x4000 + assert skifree.header.ne_stack == 0x4000 + assert skifree.header.ne_flags == NESegmentFlags.NEINST | NESegmentFlags.NEWINAPI + assert skifree.header.ne_exetyp == NETargetOSFlags.NE_WINDOWS + assert skifree.header.ne_flagsothers == 8 # according to ghidra From c73659f5cdb9e50c1cdec2e842e3120d6d5b5540 Mon Sep 17 00:00:00 2001 From: disinvite Date: Sun, 9 Mar 2025 15:12:38 -0400 Subject: [PATCH 2/2] Modular binfile download in CI. Tests now expect binfile directory parameter. --- .github/workflows/binfile-lego.yml | 29 +++++++++++ .github/workflows/binfile-ski.yml | 26 +++++++++ .github/workflows/binfiles.yml | 40 ++++++++++++++ .github/workflows/legobin.yml | 42 --------------- .github/workflows/unittest.yml | 20 +++---- tests/binfiles/.gitignore | 2 + tests/conftest.py | 84 ++++++++++++++++++------------ tests/test_project.py | 4 +- 8 files changed, 158 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/binfile-lego.yml create mode 100644 .github/workflows/binfile-ski.yml create mode 100644 .github/workflows/binfiles.yml delete mode 100644 .github/workflows/legobin.yml create mode 100644 tests/binfiles/.gitignore diff --git a/.github/workflows/binfile-lego.yml b/.github/workflows/binfile-lego.yml new file mode 100644 index 00000000..bdc7503d --- /dev/null +++ b/.github/workflows/binfile-lego.yml @@ -0,0 +1,29 @@ +name: Download LEGO binaries + +on: + workflow_call: + +jobs: + lego: + runs-on: ubuntu-latest + steps: + + - name: Restore original binaries + id: cache + uses: actions/cache@v3 + with: + path: binfiles + key: legobin + + - name: Download original island binaries + if: ${{ !steps.cache.outputs.cache-hit }} + run: | + wget https://legoisland.org/download/CONFIG.EXE --directory-prefix=binfiles + wget https://legoisland.org/download/ISLE.EXE --directory-prefix=binfiles + wget https://legoisland.org/download/LEGO1.DLL --directory-prefix=binfiles + + - name: Verify files + run: | + echo "864766d024d78330fed5e1f6efb2faf815f1b1c3405713a9718059dc9a54e52c binfiles/CONFIG.EXE" | sha256sum --check + echo "5cf57c284973fce9d14f5677a2e4435fd989c5e938970764d00c8932ed5128ca binfiles/ISLE.EXE" | sha256sum --check + echo "14645225bbe81212e9bc1919cd8a692b81b8622abb6561280d99b0fc4151ce17 binfiles/LEGO1.DLL" | sha256sum --check diff --git a/.github/workflows/binfile-ski.yml b/.github/workflows/binfile-ski.yml new file mode 100644 index 00000000..dfb7ebc6 --- /dev/null +++ b/.github/workflows/binfile-ski.yml @@ -0,0 +1,26 @@ +name: Download SKI binary + +on: + workflow_call: + +jobs: + ski: + runs-on: ubuntu-latest + steps: + + - name: Restore original binaries + id: cache + uses: actions/cache@v3 + with: + path: binfiles + key: skibin + + - name: Download original ski binaries + if: ${{ !steps.cache.outputs.cache-hit }} + run: | + wget https://archive.org/download/win3_SKIFREE/SKIFREE.ZIP + unzip -d binfiles SKIFREE.ZIP SKI.EXE + + - name: Verify files + run: | + echo "0b97b99fcf34af5f5d624080417c79c7d36ae11351a7870ce6e0a476f03515c2 binfiles/SKI.EXE" | sha256sum --check diff --git a/.github/workflows/binfiles.yml b/.github/workflows/binfiles.yml new file mode 100644 index 00000000..fb104436 --- /dev/null +++ b/.github/workflows/binfiles.yml @@ -0,0 +1,40 @@ +name: Prepare sample binaries + +on: + workflow_call: + +jobs: + lego: + name: Download LEGO + uses: ./.github/workflows/binfile-lego.yml + + ski: + name: Download SKI + uses: ./.github/workflows/binfile-ski.yml + + merge: + needs: [lego, ski] + runs-on: ubuntu-latest + steps: + + - name: Restore LEGO + uses: actions/cache/restore@v3 + with: + path: binfiles + key: legobin + fail-on-cache-miss: true + + - name: Restore SKI + uses: actions/cache/restore@v3 + with: + path: binfiles + key: skibin + fail-on-cache-miss: true + + - name: Cache binfiles + uses: actions/cache@v3 + with: + enableCrossOsArchive: true + path: binfiles + key: binfiles-${{ hashFiles('binfiles/*') }} + restore-keys: binfiles diff --git a/.github/workflows/legobin.yml b/.github/workflows/legobin.yml deleted file mode 100644 index 8c92cee3..00000000 --- a/.github/workflows/legobin.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Download legobin - -on: - workflow_call: - -jobs: - lego: - runs-on: ubuntu-latest - steps: - - - name: Cache original binaries - id: cache-lego - uses: actions/cache@v3 - with: - enableCrossOsArchive: true - path: legobin - key: legobin - - - name: Download original island binaries - if: ${{ !steps.cache-lego.outputs.cache-hit }} - run: | - wget https://legoisland.org/download/CONFIG.EXE --directory-prefix=legobin - wget https://legoisland.org/download/ISLE.EXE --directory-prefix=legobin - wget https://legoisland.org/download/LEGO1.DLL --directory-prefix=legobin - - skifree: - runs-on: ubuntu-latest - steps: - - - name: Cache original binaries - id: cache-skifree - uses: actions/cache@v3 - with: - enableCrossOsArchive: true - path: skifree - key: skifree - - - name: Download original ski binaries - if: ${{ !steps.cache-skifree.outputs.cache-hit }} - run: | - wget https://archive.org/download/win3_SKIFREE/SKIFREE.ZIP - unzip -d skifree SKIFREE.ZIP diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index a5614283..85baf7eb 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: fetch-deps: name: Prepare sample binary files - uses: ./.github/workflows/legobin.yml + uses: ./.github/workflows/binfiles.yml test: name: '${{ matrix.platform.name }} ${{ matrix.python-version }}' @@ -28,21 +28,13 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Restore lego files - id: cache-lego + - name: Restore sample binaries + id: cache uses: actions/cache/restore@v3 with: enableCrossOsArchive: true - path: legobin - key: legobin - - - name: Restore skifree files - id: cache-skifree - uses: actions/cache/restore@v3 - with: - enableCrossOsArchive: true - path: skifree - key: skifree + path: binfiles + key: binfiles - name: Install python libraries shell: bash @@ -52,4 +44,4 @@ jobs: - name: Run unit tests shell: bash run: | - pytest . --lego1=legobin/LEGO1.DLL --ski=skifree/SKI.EXE \ No newline at end of file + pytest . --binfiles=binfiles --require-binfiles \ No newline at end of file diff --git a/tests/binfiles/.gitignore b/tests/binfiles/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/tests/binfiles/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/conftest.py b/tests/conftest.py index ca6affd0..11af4d2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,58 +1,78 @@ import hashlib from pathlib import Path -from typing import Iterator +from typing import Callable, Iterator import pytest -from reccmp.isledecomp import NEImage, PEImage, detect_image +from reccmp.isledecomp import Image, NEImage, PEImage, detect_image def pytest_addoption(parser): """Allow the option to run tests against sample binaries.""" - parser.addoption("--lego1", action="store", help="Path to LEGO1.DLL") - parser.addoption("--ski", action="store", help="Path to SKI.EXE") + parser.addoption("--binfiles", action="store", help="Path to sample binary files.") + parser.addoption( + "--require-binfiles", + action="store_true", + help="Fail tests that depend on binary samples if we cannot load them.", + ) -# LEGO1.DLL: v1.1 English, September -LEGO1_SHA256 = "14645225bbe81212e9bc1919cd8a692b81b8622abb6561280d99b0fc4151ce17" +def check_hash(path: Path, hash_str: str) -> bool: + with path.open("rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + return digest == hash_str + # SkiFree 1.0 # https://ski.ihoc.net/ SKI_SHA256 = "0b97b99fcf34af5f5d624080417c79c7d36ae11351a7870ce6e0a476f03515c2" -@pytest.fixture(name="binfile", scope="session") -def fixture_binfile(pytestconfig) -> Iterator[PEImage]: - filename = pytestconfig.getoption("--lego1") +@pytest.fixture(name="bin_loader", scope="session") +def fixture_loader(pytestconfig) -> Iterator[Callable[[str, str], Image | None]]: + # Search path is ./tests/binfiles unless the user provided an alternate location. + binfiles_arg = pytestconfig.getoption("--binfiles") + if binfiles_arg is not None: + binfile_path = Path(binfiles_arg).resolve() + else: + binfile_path = Path(__file__).resolve().parent / "binfiles" - # Skip this if we have not provided the path to LEGO1.dll. - if filename is None: - pytest.skip(allow_module_level=True, reason="No path to LEGO1") + def loader(filename: str, hash_str: str) -> Image | None: + file = binfile_path / filename + if file.exists(): + if not check_hash(file, hash_str): + pytest.fail( + pytrace=False, reason="Did not match expected " + filename.upper() + ) - filename = Path(filename) - with filename.open("rb") as f: - digest = hashlib.sha256(f.read()).hexdigest() - if digest != LEGO1_SHA256: - pytest.fail(reason="Did not match expected LEGO1.DLL") + return detect_image(file) - image = detect_image(filename) - assert isinstance(image, PEImage) - yield image + not_found_reason = "No path to " + filename.upper() + if pytestconfig.getoption("--require-binfiles"): + pytest.fail(pytrace=False, reason=not_found_reason) + pytest.skip(allow_module_level=True, reason=not_found_reason) -@pytest.fixture(name="skifree", scope="session") -def fixture_skifree(pytestconfig) -> Iterator[NEImage]: - filename = pytestconfig.getoption("--ski") + return None - # Skip this if we have not provided the path to SKI.EXE. - if filename is None: - pytest.skip(allow_module_level=True, reason="No path to SKI") + yield loader - filename = Path(filename) - with filename.open("rb") as f: - digest = hashlib.sha256(f.read()).hexdigest() - if digest != SKI_SHA256: - pytest.fail(reason="Did not match expected SKI.EXE") - image = detect_image(filename) +@pytest.fixture(name="binfile", scope="session") +def fixture_binfile(bin_loader) -> Iterator[PEImage]: + """LEGO1.DLL: v1.1 English, September""" + image = bin_loader( + "LEGO1.DLL", "14645225bbe81212e9bc1919cd8a692b81b8622abb6561280d99b0fc4151ce17" + ) + assert isinstance(image, PEImage) + yield image + + +@pytest.fixture(name="skifree", scope="session") +def fixture_skifree(bin_loader) -> Iterator[NEImage]: + """SkiFree 1.0 + https://ski.ihoc.net/""" + image = bin_loader( + "SKI.EXE", "0b97b99fcf34af5f5d624080417c79c7d36ae11351a7870ce6e0a476f03515c2" + ) assert isinstance(image, NEImage) yield image diff --git a/tests/test_project.py b/tests/test_project.py index 82f6c4b0..5196e56d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -13,7 +13,9 @@ RecCmpProjectException, ) from reccmp.isledecomp.formats import PEImage -from .conftest import LEGO1_SHA256 + + +LEGO1_SHA256 = "14645225bbe81212e9bc1919cd8a692b81b8622abb6561280d99b0fc4151ce17" def test_project_loading(tmp_path_factory, binfile: PEImage):