Merge pull request #151 from MetaCell/feature/x_hierarchy

X hierarchy feature

Author: ddelpiano

applications/sckanner/backend/sckanner/management/commands/update_knowledge_statements.py

@@ -55,13 +55,24 @@ class JsonData(BaseModel):
 class Command(BaseCommand):
     help = "Fetch and update knowledge statements from an external server"
 
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--a_b_via_c_json_url',
+            type=str,
+            help='URL to the A-B-via-C JSON file',
+        )
+
     def handle(self, *args, **kwargs):
         self.stdout.write("Starting the ingestion process...")
-
-        # Step 1: Fetch raw JSON from external source
-        raw_data_url = "https://raw.githubusercontent.com/smtifahim/SCKAN-Apps/master/sckan-explorer/json/a-b-via-c-2.json"
-        self.stdout.write(f"Fetching data from {raw_data_url}...")
-        response = requests.get(raw_data_url)
+        a_b_via_c_json_url = kwargs.get('a_b_via_c_json_url')
+
+        # Step 1: Fetch raw JSON from URL
+        if not a_b_via_c_json_url:
+            # Fallback to hardcoded URL (for backward compatibility)
+            a_b_via_c_json_url = "https://raw.githubusercontent.com/smtifahim/SCKAN-Apps/refs/heads/master/sckan-explorer/json/sckanner-data/hierarchy/sckanner-hierarchy.json"
+        
+        self.stdout.write(f"Fetching data from {a_b_via_c_json_url}...")
+        response = requests.get(a_b_via_c_json_url)
         response.raise_for_status()
         raw_data = JsonData(**response.json())
 
@@ -125,4 +136,4 @@ def update_database(self, detailed_data):
         # Insert new data
         KnowledgeStatement.objects.bulk_create(
             [KnowledgeStatement(data=entry) for entry in detailed_data]
-        )
+        )

applications/sckanner/backend/sckanner/migrations/data/composer_ingestion_script.py

@@ -123,15 +123,25 @@ def fetch_paginated_data(population_ids: list[str], stdout=None):
 	return detailed_data
 
 
-def get_statements(version="", stdout=None):
+def get_statements(version="", stdout=None, a_b_via_c_json_file_path=None):
     try:
-        # Step 1: Fetch raw JSON from external source
-        raw_data_url = "https://raw.githubusercontent.com/smtifahim/SCKAN-Apps/master/sckan-explorer/json/a-b-via-c-2.json"
-        if stdout:
-            stdout.write(f"Fetching data from {raw_data_url}...\n")
-        response = requests.get(raw_data_url)
-        response.raise_for_status()
-        raw_data = JsonData(**response.json())
+        # Step 1: Load raw JSON from file or external source
+        if a_b_via_c_json_file_path:
+            # Load from provided file path
+            if stdout:
+                stdout.write(f"Loading data from file: {a_b_via_c_json_file_path}...\n")
+            import json
+            with open(a_b_via_c_json_file_path, 'r') as f:
+                raw_json = json.load(f)
+            raw_data = JsonData(**raw_json)
+        else:
+            # Fallback to hardcoded URL (for backward compatibility)
+            raw_data_url = "https://raw.githubusercontent.com/smtifahim/SCKAN-Apps/refs/heads/master/sckan-explorer/json/sckanner-data/hierarchy/sckanner-hierarchy.json"
+            if stdout:
+                stdout.write(f"Fetching data from {raw_data_url}...\n")
+            response = requests.get(raw_data_url)
+            response.raise_for_status()
+            raw_data = JsonData(**response.json())
 
         # Step 2: Extract population IDs
         if stdout:
@@ -149,7 +159,7 @@ def get_statements(version="", stdout=None):
             # --- NOTE: ONLY FOR TESTING LOCALLY ---
 
         for population_id in batched(population_ids, KNOWLEDGE_STATEMENTS_BATCH_SIZE):
-            detailed_data.extend(fetch_paginated_data(list(population_id)))
+            detailed_data.extend(fetch_paginated_data(list(population_id), stdout))
 
         if stdout:
             stdout.write(f"Ingestion process completed successfully! Total statements: {len(detailed_data)}\n")

applications/sckanner/backend/sckanner/services/ingestion/connectivity_statement_adapter.py

@@ -40,7 +40,15 @@ def _parse_and_validate_statements(
 
         # Get statements from the uploaded module
         if hasattr(module, "get_statements") and callable(module.get_statements):
-            statements = module.get_statements(self.snapshot.version)
+            # Pass the a_b_via_c_json_file path if available
+            a_b_via_c_json_file_path = None
+            if self.snapshot.a_b_via_c_json_file:
+                a_b_via_c_json_file_path = self.snapshot.a_b_via_c_json_file.path
+            
+            statements = module.get_statements(
+                self.snapshot.version, 
+                a_b_via_c_json_file_path=a_b_via_c_json_file_path
+            )
             try:
                 # Validate the statements against the schema
                 current_path = os.path.dirname(os.path.abspath(__file__))

applications/sckanner/frontend/src/App.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useState, useRef } from 'react';
+import React, { useEffect, useState, useRef, useCallback } from 'react';
 import { useDispatch, useStore } from 'react-redux';
 import { Box } from '@mui/material';
 import { getLayoutManagerInstance } from '@metacell/geppetto-meta-client/common/layout/LayoutManager';
@@ -28,6 +28,7 @@ import {
   fetchKnowledgeStatements,
   fetchMajorNerves,
   fetchOrderJson,
+  fetchEndorgansOrder,
 } from './services/fetchService.ts';
 import { getUniqueMajorNerves } from './services/filterValuesService.ts';
 import {
@@ -37,10 +38,10 @@ import {
 } from './models/explorer.ts';
 import {
   getHierarchicalNodes,
-  getOrgans,
-} from './services/hierarchyService.ts';
+  getOrgansAndTargetSystems,
+} from './services/hierarchyService';
 import ReactGA from 'react-ga4';
-import { Datasnapshot, OrderJson } from './models/json.ts';
+import { Datasnapshot, OrderJson, NerveResponse } from './models/json.ts';
 import { useDataContext } from './context/DataContext.ts';
 import LoadingOverlay from './components/common/LoadingOverlay.tsx';
 import ErrorModal from './components/common/ErrorModal.tsx';
@@ -99,6 +100,12 @@ const AppContent = () => {
     Record<string, HierarchicalNode>
   >({});
   const [organs, setOrgans] = useState<Record<string, Organ>>({});
+  const [targetSystems, setTargetSystems] = useState<Record<string, Organ[]>>(
+    {},
+  );
+  const [targetSystemNames, setTargetSystemNames] = useState<
+    Record<string, string>
+  >({});
   const [majorNerves, setMajorNerves] = useState<Set<string>>();
   const [knowledgeStatements, setKnowledgeStatements] = useState<
     Record<string, KnowledgeStatement>
@@ -108,6 +115,7 @@ const AppContent = () => {
     useState<string>('');
   const [hasValidatedInitialURL, setHasValidatedInitialURL] = useState(false);
   const [isDataLoading, setIsDataLoading] = useState(false);
+  const [hasCriticalError, setHasCriticalError] = useState(false);
   const [fetchError, setFetchError] = useState<{
     show: boolean;
     message: string;
@@ -120,6 +128,9 @@ const AppContent = () => {
   });
   const previousDatasnaphshot = useRef<string>('');
   const [orderData, setOrderData] = useState<OrderJson>({});
+  const [endorgansOrder, setEndorgansOrder] = useState<
+    Record<string, string[]>
+  >({});
   const [searchParams] = useSearchParams();
   const [urlState, setUrlState] = useState<URLState>({
     datasnapshot: null,
@@ -146,40 +157,106 @@ const AppContent = () => {
     dispatch(addWidget(connectionsWidget()));
   }, [LayoutComponent, dispatch]);
 
-  const fetchJSONAndSetHierarchicalNodes = (
-    datasnapshot: Datasnapshot,
-    orderData: OrderJson,
-  ) => {
-    fetchJSON(datasnapshot.a_b_via_c_json_file).then((jsonData) => {
-      setHierarchicalNodes(getHierarchicalNodes(jsonData, orderData));
-      setOrgans(getOrgans(jsonData));
-    });
-  };
+  const fetchJSONAndSetHierarchicalNodes = useCallback(
+    (datasnapshot: Datasnapshot, orderData: OrderJson) => {
+      fetchJSON(datasnapshot.a_b_via_c_json_file).then((jsonData) => {
+        setHierarchicalNodes(getHierarchicalNodes(jsonData, orderData));
+        const { organs, targetSystems, targetSystemNames } =
+          getOrgansAndTargetSystems(jsonData, endorgansOrder);
+        setOrgans(organs);
+        setTargetSystems(targetSystems);
+        setTargetSystemNames(targetSystemNames);
+      });
+    },
+    [endorgansOrder],
+  );
 
   useEffect(() => {
     const fetchInitialData = async () => {
+      const failedResources: string[] = [];
+
       try {
-        const [orderDataFetched, majorNervesData, datasnapshots] =
-          await Promise.all([
-            fetchOrderJson(),
-            fetchMajorNerves(),
-            fetchDatasnapshots(),
-          ]);
+        const results = await Promise.allSettled([
+          fetchOrderJson().catch((err) => {
+            failedResources.push('Organ hierarchy order (GitHub)');
+            throw err;
+          }),
+          fetchMajorNerves().catch((err) => {
+            failedResources.push('Major nerves data (GitHub)');
+            throw err;
+          }),
+          fetchDatasnapshots().catch((err) => {
+            failedResources.push('Data snapshots (API)');
+            throw err;
+          }),
+          fetchEndorgansOrder().catch((err) => {
+            failedResources.push('End organs hierarchy (GitHub)');
+            throw err;
+          }),
+        ]);
+
+        // Check if any fetch failed
+        const failures = results.filter(
+          (result) => result.status === 'rejected',
+        );
+
+        if (failures.length > 0) {
+          const errorDetails = failures
+            .map((failure, index) => {
+              if (failure.status === 'rejected') {
+                const resourceNames = [
+                  'Organ hierarchy order (GitHub)',
+                  'Major nerves data (GitHub)',
+                  'Data snapshots (API)',
+                  'End organs hierarchy (GitHub)',
+                ];
+                return `• ${resourceNames[index]}: ${failure.reason instanceof Error ? failure.reason.message : String(failure.reason)}`;
+              }
+              return '';
+            })
+            .filter(Boolean)
+            .join('\n');
+
+          setHasCriticalError(true);
+          setFetchError({
+            show: true,
+            title: 'Resources Unavailable',
+            message:
+              'The resources needed to start the application are not available at the moment. Please try again later or contact support if the problem persists.',
+            details: `Failed to fetch:\n${errorDetails}`,
+          });
+          return; // Don't proceed with partial data
+        }
+
+        // All fetches succeeded, extract the data
+        const successfulResults = results.map((result) =>
+          result.status === 'fulfilled' ? result.value : null,
+        );
+
+        const orderDataFetched = successfulResults[0] as OrderJson;
+        const majorNervesData = successfulResults[1] as NerveResponse;
+        const datasnaphotsData = successfulResults[2] as Datasnapshot[];
+        const endorgansOrderFetched = successfulResults[3] as Record<
+          string,
+          string[]
+        >;
 
         setOrderData(orderDataFetched);
         setMajorNerves(getUniqueMajorNerves(majorNervesData));
-        setdatasnapshots(datasnapshots);
+        setdatasnapshots(datasnaphotsData);
+        setEndorgansOrder(endorgansOrderFetched);
       } catch (error) {
-        console.error('Failed to fetch initial data:', error);
+        // This catch is a fallback for unexpected errors
+        console.error('Unexpected error during initial data fetch:', error);
+        setHasCriticalError(true);
         setFetchError({
           show: true,
-          title: 'Data Loading Error',
+          title: 'Resources Unavailable',
           message:
-            'Failed to load initial application data. Please refresh the page.',
+            'The resources needed to start the application are not available at the moment. Please try again later.',
           details:
             error instanceof Error ? error.message : 'Unknown error occurred',
         });
-        setMajorNerves(undefined);
       }
     };
 
@@ -241,10 +318,20 @@ const AppContent = () => {
     const selectedSnapshotObj = datasnapshots.find(
       (ds: Datasnapshot) => ds.id === parseInt(selectedDatasnaphshot),
     );
-    if (selectedSnapshotObj && Object.keys(orderData).length > 0) {
+    if (
+      selectedSnapshotObj &&
+      Object.keys(orderData).length > 0 &&
+      Object.keys(endorgansOrder).length > 0
+    ) {
       fetchJSONAndSetHierarchicalNodes(selectedSnapshotObj, orderData);
     }
-  }, [selectedDatasnaphshot, orderData, datasnapshots]);
+  }, [
+    selectedDatasnaphshot,
+    orderData,
+    datasnapshots,
+    endorgansOrder,
+    fetchJSONAndSetHierarchicalNodes,
+  ]);
 
   useEffect(() => {
     if (Object.keys(hierarchicalNodes).length > 0 && selectedDatasnaphshot) {
@@ -324,7 +411,7 @@ const AppContent = () => {
     Object.keys(knowledgeStatements).length == 0,
   ];
 
-  const isLoading = loadingConditions.some(Boolean);
+  const isLoading = !hasCriticalError && loadingConditions.some(Boolean);
   const loadingProgress =
     (loadingConditions.filter((c) => !c).length / loadingConditions.length) *
     100;
@@ -363,7 +450,10 @@ const AppContent = () => {
         majorNerves={majorNerves ? majorNerves : new Set<string>()}
         hierarchicalNodes={hierarchicalNodes}
         organs={organs}
+        targetSystems={targetSystems}
+        targetSystemNames={targetSystemNames}
         knowledgeStatements={knowledgeStatements}
+        endorgansOrder={endorgansOrder}
       >
         <Box>
           <Header

applications/sckanner/frontend/src/components/Connections.tsx

@@ -629,7 +629,13 @@ function Connections() {
             <HeatmapGrid
               yAxis={filteredYAxis}
               setYAxis={setYAxis}
-              xAxis={reorderedAxis}
+              xAxis={reorderedAxis.map((label) => ({
+                id: label,
+                label: label,
+                children: [],
+                expanded: false,
+              }))}
+              setXAxis={() => {}} // No-op for secondary heatmap
               onCellClick={handleCellClick}
               selectedCell={selectedCell}
               setSelectedCell={setSelectedCell}