Merge pull request #130 from eclipsevortex/release/3.1.5

make metagraph more resilient (#129)

Author: eclipsevortex

VERSION

@@ -1 +1 @@
-3.1.4
+3.1.5

scripts/state/moving_scores.py

@@ -0,0 +1,71 @@
+import numpy as np
+import argparse
+import os
+import sys
+
+def main():
+    parser = argparse.ArgumentParser(description="Display neuron weights from .npz model.")
+
+    parser.add_argument(
+        "--path",
+        type=str,
+        default=None,
+        help="Full path to the model .npz file. If not provided, use --neuron instead.",
+    )
+
+    parser.add_argument(
+        "--neuron",
+        type=str,
+        help="Neuron name to build the default model path: /root/.bittensor/miners/default/default/netuid7/subvortex_<neuron>/model.npz",
+    )
+
+    parser.add_argument(
+        "--uid",
+        type=int,
+        help="Optional UID to display only its weight and rank (starting from 0)",
+    )
+
+    args = parser.parse_args()
+
+    # Determine path from neuron if needed
+    if args.path is None:
+        if not args.neuron:
+            print("❌ You must provide either --path or --neuron.")
+            sys.exit(1)
+        args.path = f"/root/.bittensor/miners/default/default/netuid7/subvortex_{args.neuron}/model.npz"
+
+    if not os.path.isfile(args.path):
+        print(f"❌ File not found: {args.path}")
+        sys.exit(1)
+
+    # Load model
+    data = np.load(args.path)
+    print("✅ Loaded model. Available keys:", data.files)
+
+    if "neuron_weights" not in data:
+        print("❌ 'neuron_weights' key not found in the .npz file.")
+        sys.exit(1)
+
+    weights = data["neuron_weights"]
+
+    # Sort by (-weight, uid): descending weight, ascending UID
+    uids = np.arange(len(weights))
+    sorted_indices = sorted(uids, key=lambda uid: (-weights[uid], uid))
+
+    if args.uid is not None:
+        if args.uid < 0 or args.uid >= len(weights):
+            print(f"❌ UID {args.uid} is out of bounds (0–{len(weights)-1})")
+            sys.exit(1)
+
+        rank = sorted_indices.index(args.uid)  # 0-based
+        print("\n🏆 UID Ranking by Weight (Descending) + UID (Ascending on ties):")
+        print(f"{rank:>3}. UID {args.uid:>3} → Weight: {weights[args.uid]}")
+        return
+
+    # Otherwise, display all rankings
+    print("\n🏆 UID Ranking by Weight (Descending) + UID (Ascending on ties):")
+    for rank, uid in enumerate(sorted_indices):
+        print(f"{rank:>3}. UID {uid:>3} → Weight: {weights[uid]}")
+
+if __name__ == "__main__":
+    main()

subvortex/core/core_bittensor/axon.py

@@ -14,105 +14,25 @@
 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
-import copy
-import uuid
-import uvicorn
-import bittensor.core.config as btcc
 import bittensor.core.axon as btca
-import bittensor.core.threadpool as btct
-import bittensor.utils.networking as btun
-import bittensor.utils.btlogging as btul
-import bittensor_wallet.wallet as btw
-from inspect import Signature
-from fastapi import FastAPI, APIRouter
-from typing import Optional, Callable, Tuple
+from typing import Callable, Tuple
 
 from subvortex.core.core_bittensor.synapse import Synapse
 
 
 class SubVortexAxon(btca.Axon):
     def __init__(
         self,
-        wallet: Optional["btw.Wallet"] = None,
-        config: Optional["btcc.Config"] = None,
-        port: Optional[int] = None,
-        ip: Optional[str] = None,
-        external_ip: Optional[str] = None,
-        external_port: Optional[int] = None,
-        max_workers: Optional[int] = None,
+        *args,
         blacklist_fn: Callable[[Synapse], Tuple[bool, str]] = None,
+        **kwargs,
     ):
-        """Creates a new bittensor.Axon object from passed arguments.
+        super().__init__(*args, **kwargs)
 
-        Args:
-            config (:obj:`Optional[bittensor.core.config.Config]`): bittensor.Axon.config()
-            wallet (:obj:`Optional[bittensor_wallet.Wallet]`): bittensor wallet with hotkey and coldkeypub.
-            port (:type:`Optional[int]`): Binding port.
-            ip (:type:`Optional[str]`): Binding ip.
-            external_ip (:type:`Optional[str]`): The external ip of the server to broadcast to the network.
-            external_port (:type:`Optional[int]`): The external port of the server to broadcast to the network.
-            max_workers (:type:`Optional[int]`): Used to create the threadpool if not passed, specifies the number of active threads servicing requests.
-        """
-        # Build and check config.
-        if config is None:
-            config = btca.Axon.config()
-        config = copy.deepcopy(config)
-        config.axon.ip = ip or config.axon.ip
-        config.axon.port = port or config.axon.port
-        config.axon.external_ip = external_ip or config.axon.external_ip
-        config.axon.external_port = external_port or config.axon.external_port
-        config.axon.max_workers = max_workers or config.axon.max_workers
-        btca.Axon.check_config(config)
-        self.config = config  # type: ignore
+        # Remove the initial ping endpoint
+        self.router.delete("/ping")
 
-        # Get wallet or use default.
-        self.wallet = wallet or btw.Wallet(config=self.config)
-
-        # Build axon objects.
-        self.uuid = str(uuid.uuid1())
-        self.ip = self.config.axon.ip  # type: ignore
-        self.port = self.config.axon.port  # type: ignore
-        self.external_ip = (
-            self.config.axon.external_ip  # type: ignore
-            if self.config.axon.external_ip is not None  # type: ignore
-            else btun.get_external_ip()
-        )
-        self.external_port = (
-            self.config.axon.external_port  # type: ignore
-            if self.config.axon.external_port is not None  # type: ignore
-            else self.config.axon.port  # type: ignore
-        )
-        self.full_address = str(self.config.axon.ip) + ":" + str(self.config.axon.port)  # type: ignore
-        self.started = False
-
-        # Build middleware
-        self.thread_pool = btct.PriorityThreadPoolExecutor(
-            max_workers=self.config.axon.max_workers  # type: ignore
-        )
-        self.nonces: dict[str, int] = {}
-
-        # Request default functions.
-        self.forward_class_types: dict[str, list[Signature]] = {}
-        self.blacklist_fns: dict[str, Optional[Callable]] = {}
-        self.priority_fns: dict[str, Optional[Callable]] = {}
-        self.forward_fns: dict[str, Optional[Callable]] = {}
-        self.verify_fns: dict[str, Optional[Callable]] = {}
-
-        # Instantiate FastAPI
-        self.app = FastAPI()
-        log_level = "trace" if btul.logging.__trace_on__ else "critical"
-        self.fast_config = uvicorn.Config(
-            self.app, host="0.0.0.0", port=self.config.axon.port, log_level=log_level
-        )
-        self.fast_server = btca.FastAPIThreadedServer(config=self.fast_config)
-        self.router = APIRouter()
-        self.app.include_router(self.router)
-
-        # Build ourselves as the middleware.
-        self.middleware_cls = btca.AxonMiddleware
-        self.app.add_middleware(self.middleware_cls, axon=self)
-
-        # Attach default forward.
+        # Attach new ping endpoint
         def ping(r: Synapse) -> Synapse:
             return r
 

subvortex/core/core_bittensor/dendrite.py

@@ -15,9 +15,14 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 import time
+import aiohttp
+import asyncio
+import traceback
+
 import bittensor.core.axon as btca
 import bittensor.core.dendrite as btcd
 import bittensor.core.settings as btcs
+import bittensor.utils.btlogging as btul
 
 from subvortex.core.core_bittensor.synapse import TerminalInfo, Synapse
 
@@ -83,3 +88,67 @@ def preprocess_synapse_for_request(
         synapse.dendrite.signature = f"0x{self.keypair.sign(message).hex()}"
 
         return synapse
+
+
+async def close_dendrite(dendrite: btcd.Dendrite):
+    try:
+        session: aiohttp.ClientSession = await dendrite.session
+        connector = getattr(session, "_connector", None)
+
+        if session.closed:
+            btul.logging.debug("Session is already closed.")
+            return
+
+        # Step 1: Attempt to release all connection handlers (aggressive cleanup)
+        if connector and hasattr(connector, "_conns"):
+            try:
+                btul.logging.debug(
+                    "Releasing all idle connections from connector._conns..."
+                )
+                for key, handlers in connector._conns.items():
+                    for handler in handlers:
+                        try:
+                            transport = getattr(handler, "transport", None)
+                            if transport:
+                                transport.close()
+                        except Exception as h_ex:
+                            btul.logging.debug(
+                                f"Failed to close handler transport: {h_ex}"
+                            )
+            except Exception as ex:
+                btul.logging.warning(f"Error while releasing idle connections: {ex}")
+
+        # Step 2: Cancel any tasks using this session (helps unblock .close())
+        cancelled = 0
+        for task in asyncio.all_tasks():
+            coro = getattr(task, "_coro", None)
+            if coro and session in getattr(coro, "cr_frame", {}).f_locals.values():
+                task.cancel()
+                cancelled += 1
+        btul.logging.debug(f"Cancelled {cancelled} tasks using session.")
+
+        # Step 3: Forcibly close the connector FIRST (prevent future hangs)
+        if connector:
+            try:
+                btul.logging.debug(
+                    "Forcibly closing connector before session.close()..."
+                )
+                connector.close()
+                btul.logging.debug("Connector forcibly closed.")
+            except Exception as conn_ex:
+                btul.logging.error(f"Failed to forcibly close connector: {conn_ex}")
+
+        # Step 4: Close session with timeout — will never hang
+        try:
+            btul.logging.debug("Attempting session.close() with timeout...")
+            await session.close()
+            btul.logging.debug("Session.close() finished successfully.")
+        except asyncio.TimeoutError:
+            btul.logging.error("session.close() timed out after 3 seconds.")
+        except Exception as close_ex:
+            btul.logging.error(f"Exception during session.close(): {close_ex}")
+            btul.logging.debug(traceback.format_exc())
+
+    except Exception as ex:
+        btul.logging.error(f"Unexpected error in close_dendrite(): {ex}")
+        btul.logging.debug(traceback.format_exc())

subvortex/core/core_bittensor/subtensor/__init__.py

@@ -5,9 +5,11 @@
     wait_for_block,
     get_block_seed,
     process_weights_for_netuid,
-    get_number_of_neurons,
+    get_number_of_uids,
     get_next_block,
     get_hyperparameter_value,
+    get_number_of_uids,
+    RetryAsyncSubstrate,
 )
 
 from .subtensor_settings import Settings
@@ -20,7 +22,9 @@
     "wait_for_block",
     "get_block_seed",
     "process_weights_for_netuid",
-    "get_number_of_neurons",
+    "get_number_of_uids",
     "get_next_block",
     "get_hyperparameter_value",
+    "get_number_of_uids",
+    "RetryAsyncSubstrate",
 ]