Experimental: multiprocessing for speeding up query map decodes.

[thewhaleking]

Works (mostly) with the following snippet:

import asyncio
from concurrent.futures import ProcessPoolExecutor
from async_substrate_interface.async_substrate import AsyncSubstrateInterface
from bittensor.core.chain_data import decode_account_id
from bittensor.core.settings import SS58_FORMAT
import time


async def fish():
    async def exhaust(qmr):
        r = []
        async for k, v in await qmr:
            r.append((k, v))
        return r

    start = time.time()
    executor = ProcessPoolExecutor(max_workers=None)
    async with AsyncSubstrateInterface("wss://entrypoint-finney.opentensor.ai:443", ss58_format=SS58_FORMAT) as substrate:
        block_hash = await substrate.get_chain_head()
        tasks = [substrate.query_map("SubtensorModule", "TaoDividendsPerSubnet", [netuid], block_hash=block_hash, executor=executor) for
                 netuid in range(1, 51)]
        tasks = [exhaust(task) for task in tasks]
        print(time.time() - start)
        results_dicts_list = []
        for future in asyncio.as_completed(tasks):
            result = await future
            results_dicts_list.extend([{decode_account_id(k): v.value} for k, v in result])

    elapsed = time.time() - start
    print(f"time elapsed: {elapsed}")

    print("Async Results", len(results_dicts_list))
    time.sleep(5)
    try:
        executor.shutdown(wait=False, cancel_futures=True)
    except Exception:
        pass


if __name__ == '__main__':
    asyncio.run(fish())

Note that during the executor.shutdown line, we get lots of EOF errors, and idk why:

output

3.92362904548645 time elapsed: 19.864284992218018 Async Results 14803 Exception in thread Exception in thread Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Exception in thread Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Exception in thread Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Exception in thread Exception in thread Thread-2 (_monitor)Exception in thread Thread-2 (_monitor)Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner : Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Exception in thread : Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Exception in thread Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner Exception in thread Thread-2 (_monitor): Traceback (most recent call last): File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1075, in _bootstrap_inner self.run() self.run() self.run()self.run() self.run() self.run() File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run

  File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
self.run()

self.run() File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run

File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
self.run()
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
self.run()self._target(*self._args, **self._kwargs)

self._target(*self._args, **self._kwargs)

File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)self._target(*self._args, **self._kwargs)self._target(*self._args, **self._kwargs)
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor

self._target(*self._args, **self._kwargs)  File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor

File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
self._target(*self._args, **self._kwargs)

File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
self._target(*self._args, **self._kwargs)
self._target(*self._args, **self._kwargs)
self._target(*self._args, **self._kwargs)
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1577, in _monitor
record = self.dequeue(True)
record = self.dequeue(True) record = self.dequeue(True)
record = self.dequeue(True)
record = self.dequeue(True)record = self.dequeue(True)

record = self.dequeue(True)  
  record = self.dequeue(True)     
   record = self.dequeue(True) record = self.dequeue(True) 

                            ^  ^          ^          ^    ^     ^  ^ ^^ ^        ^     ^^^ ^^^ ^ ^ ^ ^ ^  ^  ^^  ^^    ^^ ^^ ^^^^ ^^^^^ ^ ^^^^^^^^^^^^^^^ ^^^ ^^^^^^ ^^ ^^^^^^^^^^^^^^^^ ^^^^^^^^^^

^^
^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^^^^^^^^^^^^^^^^^^^^^^^^^
^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue

^^^

^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^^^^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^^^^^^^^^^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
^^^^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue

File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/handlers.py", line 1526, in dequeue
return self.queue.get(block)
^^^^^^^^^^^^^^^^^^^^^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
return self.queue.get(block)
return self.queue.get(block)
return self.queue.get(block)return self.queue.get(block)

   ^^^^^^^   ^ ^ ^^      return self.queue.get(block)

^^^ ^^^^^ ^ return self.queue.get(block)^ return self.queue.get(block)
return self.queue.get(block)
^
^^^ ^ ^ ^ ^ ^ ^ ^ ^^^ ^ ^
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^ ^^ ^^^^^ ^^ ^^^^^^^^^ ^^return self.queue.get(block)^
^ ^^^^ ^
^ ^ ^^^^^^^^^^ ^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^ ^^^^^^^^^
^ ^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^^^^ ^ ^^^^^^^^^^ ^^^ ^^^res = self._recv_bytes()
^^ ^res = self._recv_bytes()

^res = self._recv_bytes() File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^ ^ ^ ^ ^^^res = self._recv_bytes()^^^
^ ^
^^^^^^^^^ ^ ^^^ ^
res = self._recv_bytes() ^^ ^^^^^^^^^
^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^ ^ ^ ^ ^
^ ^ ^^ ^ ^res = self._recv_bytes()
^ ^
^^ ^^^
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^ ^^^^^ ^^^^^^^^^^^^ ^^^^^^^^^^^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^^^^^^ ^^^^^^^^^^
^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^ ^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/queues.py", line 103, in get
^^^ ^^

File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^res = self._recv_bytes()^ ^res = self._recv_bytes()^^^ ^
res = self._recv_bytes()^

^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^ ^ ^ ^ ^^^^^ ^^ ^^ ^^^^ ^^^ ^^res = self._recv_bytes()^
^ ^^ ^^^^^^^^^^ ^ ^^ ^
^^^^^ ^ ^ ^ ^^^^^^^ ^ ^^ ^^^^^ ^ ^^
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^
^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^ ^^^^buf = self._recv_bytes(maxlength) ^
^buf = self._recv_bytes(maxlength)^
^^^
^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^^buf = self._recv_bytes(maxlength)^^^
^^ buf = self._recv_bytes(maxlength)^
^^^^^ ^ ^^ ^ ^ buf = self._recv_bytes(maxlength)^^
^ ^ ^^ ^ ^
^^^^ ^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
buf = self._recv_bytes(maxlength)^

^ ^ ^ ^ ^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 216, in recv_bytes
^^^^^^^^^^ ^buf = self._recv_bytes(maxlength)^^
^^^^^^^ ^ ^ ^ ^ ^^^buf = self._recv_bytes(maxlength)
^^^ ^ ^ ^^ ^^buf = self._recv_bytes(maxlength)
^^^^^^^^^^^ ^ ^^ ^^ ^ ^^ ^^buf = self._recv_bytes(maxlength)
^^ ^ ^ ^ ^^^ ^^ ^^ ^^ ^^^^^ ^ ^^ ^ ^
^ ^^ ^ ^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^^^^ ^ ^^^ ^^ ^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^ ^^^^^^^^
^buf = self._recv(4)^^^^^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^^^^^
^^^^^^
^^^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^^^^^^^^^^ ^^^ ^^ ^^^ ^^^ ^^^
^ buf = self._recv(4)
^ ^^^^ ^^ ^^ ^ ^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^^^^ buf = self._recv(4)
^ ^^^^ ^ ^^ ^ ^^
^buf = self._recv(4)
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^^^^^^^^^^^^^^
^^ ^^
^ ^^^
^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^ ^ ^^buf = self._recv(4)^^^
^ ^buf = self._recv(4)^^

^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes
^^ ^ ^
^ ^^^^ ^^^ ^^ ^^ ^ ^^^^^^^

  File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv

raise EOFError File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv

 buf = self._recv(4) 
 buf = self._recv(4)

^^EOFError^^
^^^ ^ ^^^^ ^
buf = self._recv(4) File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv

  raise EOFError
     buf = self._recv(4)      ^^^^^    

raise EOFErrorEOFError^
^
^ ^^ ^EOFError
raise EOFError^^
^^ ^^^^^ ^ ^EOFError ^ ^ ^
^ ^^ ^^^^^^^^ ^ raise EOFError^�[0m
^^^^^
�[0m
^ File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
^
^^ EOFError^�[0m File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
^ ^^^^^^^^�[0m
^^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
raise EOFError
EOFError
^^^ raise EOFError
EOFError
^^^
File "/opt/homebrew/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/connection.py", line 399, in _recv
�[0m raise EOFError�[0m
raise EOFError
EOFError
EOFError
�[0m raise EOFError
EOFError
�[0m�[0m�[0m�[0m
Process finished with exit code 0