diff --git a/.github/workflows/check-btcli-tests.yml b/.github/workflows/check-btcli-tests.yml new file mode 100644 index 0000000..18b93a2 --- /dev/null +++ b/.github/workflows/check-btcli-tests.yml @@ -0,0 +1,255 @@ +name: Bittensor BTCLI Test + +permissions: + pull-requests: write + contents: read + +concurrency: + group: e2e-cli-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: + - main + - staging + types: [opened, synchronize, reopened, labeled, unlabeled] + +env: + CARGO_TERM_COLOR: always + VERBOSE: ${{ github.event.inputs.verbose }} + +jobs: + apply-label-to-new-pr: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.draft == false }} + outputs: + should_continue_cli: ${{ steps.check.outputs.should_continue_cli }} + steps: + - name: Check + id: check + run: | + ACTION="${{ github.event.action }}" + if [[ "$ACTION" == "opened" || "$ACTION" == "reopened" ]]; then + echo "should_continue_cli=true" >> $GITHUB_OUTPUT + else + echo "should_continue_cli=false" >> $GITHUB_OUTPUT + fi + shell: bash + + - name: Add label + if: steps.check.outputs.should_continue_cli == 'true' + uses: actions-ecosystem/action-add-labels@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + labels: run-bittensor-cli-tests + + check-labels: + needs: apply-label-to-new-pr + runs-on: ubuntu-latest + if: always() + outputs: + run-cli-tests: ${{ steps.get-labels.outputs.run-cli-tests }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Get labels from PR + id: get-labels + run: | + LABELS=$(gh pr view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name') + echo "Current labels: $LABELS" + if echo "$LABELS" | grep -q "run-bittensor-cli-tests"; then + echo "run-cli-tests=true" >> $GITHUB_ENV + echo "::set-output name=run-cli-tests::true" + else + echo "run-cli-tests=false" >> $GITHUB_ENV + echo "::set-output name=run-cli-tests::false" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + find-e2e-tests: + needs: check-labels + if: always() && needs.check-labels.outputs.run-cli-tests == 'true' + runs-on: ubuntu-latest + outputs: + test-files: ${{ steps.get-tests.outputs.test-files }} + steps: + - name: Research preparation + working-directory: ${{ github.workspace }} + run: git clone https://github.com/opentensor/btcli.git + + - name: Checkout + working-directory: ${{ github.workspace }}/btcli + run: git checkout staging + + - name: Install dependencies + run: sudo apt-get install -y jq + + - name: Find e2e test files + id: get-tests + run: | + test_files=$(find ${{ github.workspace }}/btcli/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))') + echo "::set-output name=test-files::$test_files" + shell: bash + + pull-docker-image: + needs: check-labels + runs-on: ubuntu-latest + if: always() && needs.check-labels.outputs.run-cli-tests == 'true' + steps: + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + + - name: Pull Docker Image + run: docker pull ghcr.io/opentensor/subtensor-localnet:devnet-ready + + - name: Save Docker Image to Cache + run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:devnet-ready + + - name: Upload Docker Image as Artifact + uses: actions/upload-artifact@v4 + with: + name: subtensor-localnet + path: subtensor-localnet.tar + + run-e2e-tests: + needs: + - check-labels + - find-e2e-tests + - pull-docker-image + + if: always() && needs.check-labels.outputs.run-cli-tests == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 16 + matrix: + rust-branch: + - stable + rust-target: + - x86_64-unknown-linux-gnu + os: + - ubuntu-latest + test-file: ${{ fromJson(needs.find-e2e-tests.outputs.test-files) }} + + env: + RELEASE_NAME: development + RUSTV: ${{ matrix.rust-branch }} + RUST_BACKTRACE: full + RUST_BIN_DIR: target/${{ matrix.rust-target }} + TARGET: ${{ matrix.rust-target }} + + timeout-minutes: 60 + name: "e2e tests: ${{ matrix.test-file }}" + steps: + - name: Check-out repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update && + sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler + + - name: Create Python virtual environment + working-directory: ${{ github.workspace }} + run: python3 -m venv ${{ github.workspace }}/venv + + - name: Clone Bittensor CLI repo + working-directory: ${{ github.workspace }} + run: git clone https://github.com/opentensor/btcli.git + + - name: Setup Bittensor-cli from cloned repo + working-directory: ${{ github.workspace }}/btcli + run: | + source ${{ github.workspace }}/venv/bin/activate + git checkout staging + git fetch origin staging + python3 -m pip install --upgrade pip + python3 -m pip install '.[dev]' + python3 -m pip install pytest + + - name: Clone async-substrate-interface repo + run: git clone https://github.com/opentensor/async-substrate-interface.git + + - name: Checkout PR async-substrate-interface repo + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + git fetch origin ${{ github.event.pull_request.head.ref }} + git checkout ${{ github.event.pull_request.head.ref }} + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + - name: Install async-substrate-interface package + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + source ${{ github.workspace }}/venv/bin/activate + python3 -m pip uninstall async-substrate-interface -y + python3 -m pip install . + + - name: Download Cached Docker Image + uses: actions/download-artifact@v4 + with: + name: subtensor-localnet + + - name: Load Docker Image + run: docker load -i subtensor-localnet.tar + + - name: Run tests + run: | + source ${{ github.workspace }}/venv/bin/activate + pytest ${{ matrix.test-file }} -s + + + run-unit-test: + needs: + - check-labels + if: always() && needs.check-labels.outputs.run-cli-tests == 'true' + runs-on: ubuntu-latest + steps: + - name: Check-out repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update && + sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler + + - name: Create Python virtual environment + working-directory: ${{ github.workspace }} + run: python3 -m venv venv + + - name: Clone Bittensor CLI repo + working-directory: ${{ github.workspace }} + run: git clone https://github.com/opentensor/btcli.git + + - name: Setup Bittensor SDK from cloned repo + working-directory: ${{ github.workspace }}/btcli + run: | + source ${{ github.workspace }}/venv/bin/activate + git checkout staging + git fetch origin staging + python3 -m pip install --upgrade pip + python3 -m pip install '.[dev]' + + - name: Clone async-substrate-interface repo + run: git clone https://github.com/opentensor/async-substrate-interface.git + + - name: Checkout PR branch in async-substrate-interface repo + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + git fetch origin ${{ github.event.pull_request.head.ref }} + git checkout ${{ github.event.pull_request.head.ref }} + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + - name: Install /async-substrate-interface package + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + source ${{ github.workspace }}/venv/bin/activate + pip uninstall async-substrate-interface -y + pip install . + + - name: Run SDK unit tests + run: | + source ${{ github.workspace }}/venv/bin/activate + pytest ${{ github.workspace }}/btcli/tests/unit_tests \ No newline at end of file diff --git a/.github/workflows/check-sdk-tests.yml b/.github/workflows/check-sdk-tests.yml new file mode 100644 index 0000000..4bcd4ff --- /dev/null +++ b/.github/workflows/check-sdk-tests.yml @@ -0,0 +1,259 @@ +name: Bittensor SDK Test + +permissions: + pull-requests: write + contents: read + +concurrency: + group: e2e-sdk-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: + - main + - staging + types: [opened, synchronize, reopened, labeled, unlabeled] + +env: + CARGO_TERM_COLOR: always + VERBOSE: ${{ github.event.inputs.verbose }} + +jobs: + apply-label-to-new-pr: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.draft == false }} + outputs: + should_continue_sdk: ${{ steps.check.outputs.should_continue_sdk }} + steps: + - name: Check + id: check + run: | + ACTION="${{ github.event.action }}" + if [[ "$ACTION" == "opened" || "$ACTION" == "reopened" ]]; then + echo "should_continue_sdk=true" >> $GITHUB_OUTPUT + else + echo "should_continue_sdk=false" >> $GITHUB_OUTPUT + fi + shell: bash + + - name: Add label + if: steps.check.outputs.should_continue_sdk == 'true' + uses: actions-ecosystem/action-add-labels@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + labels: run-bittensor-sdk-tests + + check-labels: + needs: apply-label-to-new-pr + runs-on: ubuntu-latest + if: always() + outputs: + run-sdk-tests: ${{ steps.get-labels.outputs.run-sdk-tests }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Get labels from PR + id: get-labels + run: | + sleep 5 + LABELS=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels --jq '.[].name') + echo "Current labels: $LABELS" + if echo "$LABELS" | grep -q "run-bittensor-sdk-tests"; then + echo "run-sdk-tests=true" >> $GITHUB_OUTPUT + else + echo "run-sdk-tests=false" >> $GITHUB_OUTPUT + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + find-e2e-tests: + needs: check-labels + if: always() && needs.check-labels.outputs.run-sdk-tests == 'true' + runs-on: ubuntu-latest + outputs: + test-files: ${{ steps.get-tests.outputs.test-files }} + steps: + - name: Research preparation + working-directory: ${{ github.workspace }} + run: git clone https://github.com/opentensor/bittensor.git + + - name: Checkout + working-directory: ${{ github.workspace }}/bittensor + run: git checkout staging + + - name: Install dependencies + run: sudo apt-get install -y jq + + - name: Find e2e test files + id: get-tests + run: | + test_files=$(find ${{ github.workspace }}/bittensor/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))') + echo "::set-output name=test-files::$test_files" + shell: bash + + pull-docker-image: + needs: check-labels + runs-on: ubuntu-latest + if: always() && needs.check-labels.outputs.run-sdk-tests == 'true' + steps: + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + + - name: Pull Docker Image + run: docker pull ghcr.io/opentensor/subtensor-localnet:devnet-ready + + - name: Save Docker Image to Cache + run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:devnet-ready + + - name: Upload Docker Image as Artifact + uses: actions/upload-artifact@v4 + with: + name: subtensor-localnet + path: subtensor-localnet.tar + + run-e2e-tests: + needs: + - check-labels + - find-e2e-tests + - pull-docker-image + + if: always() && needs.check-labels.outputs.run-sdk-tests == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 16 + matrix: + rust-branch: + - stable + rust-target: + - x86_64-unknown-linux-gnu + os: + - ubuntu-latest + test-file: ${{ fromJson(needs.find-e2e-tests.outputs.test-files) }} + + env: + RELEASE_NAME: development + RUSTV: ${{ matrix.rust-branch }} + RUST_BACKTRACE: full + RUST_BIN_DIR: target/${{ matrix.rust-target }} + TARGET: ${{ matrix.rust-target }} + + timeout-minutes: 60 + name: "e2e tests: ${{ matrix.test-file }}" + steps: + - name: Check-out repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update && + sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler + + - name: Create Python virtual environment + working-directory: ${{ github.workspace }} + run: python3 -m venv ${{ github.workspace }}/venv + + - name: Clone Bittensor SDK repo + working-directory: ${{ github.workspace }} + run: git clone https://github.com/opentensor/bittensor.git + + - name: Setup Bittensor SDK from cloned repo + working-directory: ${{ github.workspace }}/bittensor + run: | + source ${{ github.workspace }}/venv/bin/activate + git checkout staging + git fetch origin staging + python3 -m pip install --upgrade pip + python3 -m pip install '.[dev]' + + - name: Clone Bittensor async-substrate-interface repo + run: git clone https://github.com/opentensor/async-substrate-interface.git + + - name: Checkout PR branch in async-substrate-interface repo + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + git fetch origin ${{ github.event.pull_request.head.ref }} + git checkout ${{ github.event.pull_request.head.ref }} + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + - name: Install async-substrate-interface package + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + source ${{ github.workspace }}/venv/bin/activate + python3 -m pip uninstall async-substrate-interface -y + python3 -m pip install . + + - name: Download Cached Docker Image + uses: actions/download-artifact@v4 + with: + name: subtensor-localnet + + - name: Load Docker Image + run: docker load -i subtensor-localnet.tar + + - name: Run tests + run: | + source ${{ github.workspace }}/venv/bin/activate + python3 -m pytest ${{ matrix.test-file }} -s + + + run-integration-and-unit-test: + needs: + - check-labels + if: always() && needs.check-labels.outputs.run-sdk-tests == 'true' + runs-on: ubuntu-latest + steps: + - name: Check-out repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update && + sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler + + - name: Create Python virtual environment + working-directory: ${{ github.workspace }} + run: python3 -m venv venv + + - name: Clone Bittensor SDK repo + working-directory: ${{ github.workspace }} + run: git clone https://github.com/opentensor/bittensor.git + + - name: Setup Bittensor SDK from cloned repo + working-directory: ${{ github.workspace }}/bittensor + run: | + source ${{ github.workspace }}/venv/bin/activate + git checkout staging + git fetch origin staging + python3 -m pip install --upgrade pip + python3 -m pip install '.[dev]' + python3 -m pip install -r requirements/torch.txt + + - name: Clone async-substrate-interface repo + run: git clone https://github.com/opentensor/async-substrate-interface.git + + - name: Checkout PR branch in async-substrate-interface repo + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + git fetch origin ${{ github.event.pull_request.head.ref }} + git checkout ${{ github.event.pull_request.head.ref }} + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + - name: Install /async-substrate-interface package + working-directory: ${{ github.workspace }}/async-substrate-interface + run: | + source ${{ github.workspace }}/venv/bin/activate + pip uninstall async-substrate-interface -y + pip install . + + - name: Run SDK integration tests + run: | + source ${{ github.workspace }}/venv/bin/activate + pytest ${{ github.workspace }}/bittensor/tests/integration_tests + + - name: Run bittensor-sdk unit tests + run: | + source ${{ github.workspace }}/venv/bin/activate + pytest ${{ github.workspace }}/bittensor/tests/unit_tests \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 735b296..6469d4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 1.0.9 /2025-03-26 + +## What's Changed +* Add workflows for run SDK and BTCLI tests if labels are applied by @roman-opentensor in https://github.com/opentensor/async-substrate-interface/pull/83 +* Update docker image name by @roman-opentensor in https://github.com/opentensor/async-substrate-interface/pull/85 +* Updates `_load_registry_type_map` by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/87 + +**Full Changelog**: https://github.com/opentensor/async-substrate-interface/compare/v1.0.8...v1.0.9 + ## 1.0.8 /2025-03-17 ## What's Changed diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py index 319302a..754b860 100644 --- a/async_substrate_interface/types.py +++ b/async_substrate_interface/types.py @@ -626,40 +626,61 @@ def _load_registry_type_map(self, registry): registry_type_map = {} type_id_to_name = {} types = json.loads(registry.registry)["types"] + type_by_id = {entry["id"]: entry for entry in types} + + # Pass 1: Gather simple types for type_entry in types: - type_type = type_entry["type"] type_id = type_entry["id"] - type_def = type_type["def"] - type_path = type_type.get("path") - if type_path and type_path[-1] == "Option": - self._handle_option_type( - type_entry, type_id, registry_type_map, type_id_to_name - ) + type_def = type_entry["type"]["def"] + type_path = type_entry["type"].get("path") + if type_entry.get("params") or "variant" in type_def: continue - if type_entry.get("params") or type_def.get("variant"): - continue # has generics or is Enum if type_path: type_name = type_path[-1] registry_type_map[type_name] = type_id type_id_to_name[type_id] = type_name else: - # probably primitive - if type_def.get("primitive"): - type_name = type_def["primitive"] - registry_type_map[type_name] = type_id - type_id_to_name[type_id] = type_name - for type_entry in types: - type_type = type_entry["type"] - type_id = type_entry["id"] - type_def = type_type["def"] - if type_def.get("sequence"): + # Possibly a primitive + if "primitive" in type_def: + prim_name = type_def["primitive"] + registry_type_map[prim_name] = type_id + type_id_to_name[type_id] = prim_name + + # Pass 2: Resolve remaining types + pending_ids = set(type_by_id.keys()) - set(type_id_to_name.keys()) + + def resolve_type_definition(type_id): + type_entry = type_by_id[type_id] + type_def = type_entry["type"]["def"] + type_path = type_entry["type"].get("path", []) + type_params = type_entry["type"].get("params", []) + + if type_id in type_id_to_name: + return type_id_to_name[type_id] + + # Resolve complex types with paths (including generics like Option etc) + if type_path: + type_name = type_path[-1] + if type_params: + inner_names = [] + for param in type_params: + dep_id = param["type"] + if dep_id not in type_id_to_name: + return None + inner_names.append(type_id_to_name[dep_id]) + return f"{type_name}<{', '.join(inner_names)}>" + if "variant" in type_def: + return None + return type_name + + elif "sequence" in type_def: sequence_type_id = type_def["sequence"]["type"] inner_type = type_id_to_name.get(sequence_type_id) if inner_type: type_name = f"Vec<{inner_type}>" - type_id_to_name[type_id] = type_name - registry_type_map[type_name] = type_id - elif type_def.get("array"): + return type_name + + elif "array" in type_def: array_type_id = type_def["array"]["type"] inner_type = type_id_to_name.get(array_type_id) maybe_len = type_def["array"].get("len") @@ -668,45 +689,44 @@ def _load_registry_type_map(self, registry): type_name = f"[{inner_type}; {maybe_len}]" else: type_name = f"[{inner_type}]" - type_id_to_name[type_id] = type_name - registry_type_map[type_name] = type_id - elif type_def.get("compact"): + return type_name + + elif "compact" in type_def: compact_type_id = type_def["compact"]["type"] inner_type = type_id_to_name.get(compact_type_id) if inner_type: type_name = f"Compact<{inner_type}>" - type_id_to_name[type_id] = type_name - registry_type_map[type_name] = type_id - elif type_def.get("tuple"): + return type_name + + elif "tuple" in type_def: tuple_type_ids = type_def["tuple"] type_names = [] for inner_type_id in tuple_type_ids: - inner_type = type_id_to_name.get(inner_type_id) - if inner_type: - type_names.append(inner_type) + if inner_type_id not in type_id_to_name: + return None + type_names.append(type_id_to_name[inner_type_id]) type_name = ", ".join(type_names) type_name = f"({type_name})" - type_id_to_name[type_id] = type_name - registry_type_map[type_name] = type_id - self.registry_type_map = registry_type_map - self.type_id_to_name = type_id_to_name + return type_name - def _handle_option_type( - self, type_entry, type_id, registry_type_map, type_id_to_name - ): - params = type_entry["type"].get("params", []) - if params: - inner_names = [] - for param in params: - inner_id = param["type"] - inner_name = type_id_to_name.get(inner_id, f"Type{inner_id}") - inner_names.append(inner_name) - type_name = f"Option<{', '.join(inner_names)}>" - else: - type_name = "Option" + elif "variant" in type_def: + return None - registry_type_map[type_name] = type_id - type_id_to_name[type_id] = type_name + return None + + resolved_type = True + while resolved_type and pending_ids: + resolved_type = False + for type_id in list(pending_ids): + name = resolve_type_definition(type_id) + if name is not None: + type_id_to_name[type_id] = name + registry_type_map[name] = type_id + pending_ids.remove(type_id) + resolved_type = True + + self.registry_type_map = registry_type_map + self.type_id_to_name = type_id_to_name def reload_type_registry( self, use_remote_preset: bool = True, auto_discover: bool = True @@ -836,11 +856,10 @@ def _encode_scale(self, type_string, value: Any) -> bytes: ) except KeyError: vec_acct_id = "scale_info::152" - try: optional_acct_u16 = f"scale_info::{self.registry_type_map['Option<(AccountId32, u16)>']}" except KeyError: - optional_acct_u16 = "scale_info::573" + optional_acct_u16 = "scale_info::579" if type_string == "scale_info::0": # Is an AccountId # encode string into AccountId diff --git a/pyproject.toml b/pyproject.toml index 3fff69a..f80eb46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "async-substrate-interface" -version = "1.0.8" +version = "1.0.9" description = "Asyncio library for interacting with substrate. Mostly API-compatible with py-substrate-interface" readme = "README.md" license = { file = "LICENSE" }