Add overall graph projection statistics to anomaly detection reports

Author: JohT

domains/anomaly-detection/anomalyDetectionCsv.sh

@@ -108,6 +108,8 @@ anomaly_detection_queries() {
     mkdir -p "${detail_report_directory}"
 
     echo "anomalyDetectionCsv: $(date +'%Y-%m-%dT%H:%M:%S%z') Executing Queries for ${nodeLabel} nodes..."
+    execute_cypher "${ANOMALY_DETECTION_QUERY_CYPHER_DIR}/AnomalyDetectionProjectionStatistics.cypher" "${@}" > "${detail_report_directory}/AnomalyDetection_GraphProjectionStatistics.csv"
+
     execute_cypher "${ANOMALY_DETECTION_QUERY_CYPHER_DIR}/AnomalyDetectionPotentialImbalancedRoles.cypher" "${@}" > "${detail_report_directory}/AnomalyDetection_PotentialImbalancedRoles.csv"
     execute_cypher "${ANOMALY_DETECTION_QUERY_CYPHER_DIR}/AnomalyDetectionPotentialOverEngineerOrIsolated.cypher" "${@}" > "${detail_report_directory}/AnomalyDetection_PotentialOverEngineerOrIsolated.csv"
 

domains/anomaly-detection/queries/AnomalyDetectionProjectionStatistics.cypher

@@ -0,0 +1,17 @@
+// Reads projection statistics
+
+ CALL gds.graph.list($projection_name + '-cleaned')
+YIELD nodeCount, relationshipCount, density, sizeInBytes, degreeDistribution
+RETURN nodeCount
+      ,relationshipCount
+      ,density
+      ,sizeInBytes
+      ,degreeDistribution.min
+      ,degreeDistribution.mean
+      ,degreeDistribution.max
+      ,degreeDistribution.p50
+      ,degreeDistribution.p75
+      ,degreeDistribution.p90
+      ,degreeDistribution.p95
+      ,degreeDistribution.p99
+      ,degreeDistribution.p999

domains/anomaly-detection/summary/AnomaliesDeepDiveOverview.cypher

@@ -1,20 +1,34 @@
 // Anomaly Detection DeepDive: Overview of analyzed code units and the number of anomalies detected. Requires all other labels/*.cypher queries to run first. Variables: projection_language, projection_node_label
 
-MATCH (codeUnit)
-WHERE $projection_node_label IN labels(codeUnit)
-  AND (codeUnit.incomingDependencies IS NOT NULL 
-   OR  codeUnit.outgoingDependencies IS NOT NULL)
+ MATCH (codeUnit)
+ WHERE $projection_node_label IN labels(codeUnit)
+   AND (codeUnit.incomingDependencies > 0 
+    OR  codeUnit.outgoingDependencies > 0)
+   AND coalesce(codeUnit.testMarkerInteger, 0) = 0
+OPTIONAL MATCH (codeUnit)-[dependency:DEPENDS_ON]-(target)
+ WHERE $projection_node_label IN labels(target)
+   AND (target.incomingDependencies > 0 
+    OR  target.outgoingDependencies > 0)
+   AND coalesce(target.testMarkerInteger, 0) = 0
+  WITH count(dependency)          AS relationshipCount
+      ,collect(DISTINCT codeUnit) AS codeUnits
+UNWIND codeUnits                  AS codeUnit
   WITH sum(codeUnit.anomalyLabel)                AS anomalyCount
       ,sum(sign(codeUnit.anomalyAuthorityRank))  AS authorityCount
       ,sum(sign(codeUnit.anomalyBottleneckRank)) AS bottleNeckCount
       ,sum(sign(codeUnit.anomalyBridgeRank))     AS bridgeCount
       ,sum(sign(codeUnit.anomalyHubRank))        AS hubCount
       ,sum(sign(codeUnit.anomalyOutlierRank))    AS outlierCount
+      ,count(codeUnit)                           AS nodeCount
+      ,relationshipCount
       //,collect(codeUnit.name)[0..4]              AS exampleNames
- RETURN anomalyCount     AS `Anomalies`
-       ,authorityCount   AS `Authorities`
-       ,bottleNeckCount  AS `Bottlenecks`
-       ,bridgeCount      AS `Bridges`
-       ,hubCount         AS `Hubs`
-       ,outlierCount     AS `Outliers`
-      //,exampleNames
+RETURN anomalyCount      AS `Anomalies`
+      ,authorityCount    AS `Authorities`
+      ,bottleNeckCount   AS `Bottlenecks`
+      ,bridgeCount       AS `Bridges`
+      ,hubCount          AS `Hubs`
+      ,outlierCount      AS `Outliers`
+      ,nodeCount         AS `CodeUnits`
+      ,relationshipCount AS `Dependencies`
+      ,round(toFloat(relationshipCount) / (nodeCount * (nodeCount - 1)), 6) AS `GraphDensity`
+     //,exampleNames