Refine anomaly detection features

.gitignore

@@ -102,4 +102,8 @@ __pycache__/
 *.pyc
 
 # Optuna (and other) Database data
-*.db
+*.db
+
+# Documentation generation
+domains/**/documentation/package.json
+domains/**/documentation/package-lock.json

cypher/Metrics/Calculate_and_set_Abstractness_for_Java.cypher

@@ -9,16 +9,20 @@ MATCH (artifact:Artifact)-[:CONTAINS]->(package)
      ,count{(package)-[:CONTAINS]->(:Annotation)}              AS numberAnnotations
      ,count{(package)-[:CONTAINS]->(:Interface)}               AS numberInterfaces
  WITH *
-     ,numberInterfaces + numberAnnotations + numberAbstractClasses AS numberAbstractTypes
+     ,numberInterfaces + numberAnnotations +  numberAbstractClasses        AS numberAbstractTypes
+     ,numberInterfaces + numberAnnotations + (numberAbstractClasses * 0.7) AS weightedAbstractTypes
  WITH *
-     ,toFloat(numberAbstractTypes) / (numberTypes + 1E-38) AS abstractness
-  SET package.abstractness          = abstractness
-     ,package.numberOfAbstractTypes = numberAbstractTypes
-     ,package.numberOfTypes         = numberTypes
+     ,toFloat(weightedAbstractTypes) / (numberTypes + 1E-38) AS abstractness
+  SET package.abstractness            = abstractness
+     ,package.numberOfAbstractTypes   = numberAbstractTypes
+     ,package.numberOfAbstractClasses = numberAbstractClasses
+     ,package.numberOfTypes           = numberTypes
 RETURN artifactName
       ,package.fqn  AS fullQualifiedPackageName
       ,package.name AS packageName
       ,abstractness
       ,numberAbstractTypes
       ,numberTypes
+      ,numberAbstractClasses
+      ,weightedAbstractTypes
 ORDER BY abstractness ASC, numberTypes DESC

cypher/Metrics/Calculate_and_set_Abstractness_for_Java_including_Subpackages.cypher

@@ -13,10 +13,14 @@ MATCH (artifact:Artifact)-[:CONTAINS]->(package)
      ,package
      ,sum(subpackage.numberOfTypes)            AS numberTypes
      ,sum(subpackage.numberOfAbstractTypes)    AS numberAbstractTypes
+     ,sum(subpackage.numberOfAbstractClasses)  AS numberAbstractClasses
      ,count(path) - 1                          AS numberOfIncludedSubPackages
      ,max(length(path))                        AS maxSubpackageDepth
  WITH *
-     ,toFloat(numberAbstractTypes) / (numberTypes + 1E-38) AS abstractness
+      // Calculate abstract classes out of abstract types and then add 70% of them back in (weighted)
+     ,numberAbstractTypes - (numberAbstractClasses * 0.3) AS weightedAbstractTypes
+ WITH *
+     ,toFloat(weightedAbstractTypes) / (numberTypes + 1E-38) AS abstractness
   SET package.abstractnessIncludingSubpackages          = abstractness
      ,package.numberOfAbstractTypesIncludingSubpackages = numberAbstractTypes
      ,package.numberOfTypesIncludingSubpackages         = numberTypes
@@ -26,6 +30,8 @@ RETURN artifactName
       ,abstractness
       ,numberAbstractTypes
       ,numberTypes
+      ,numberAbstractClasses
+      ,weightedAbstractTypes
       ,numberOfIncludedSubPackages
       ,maxSubpackageDepth
 ORDER BY abstractness ASC, maxSubpackageDepth DESC, numberTypes DESC

cypher/Metrics/Calculate_and_set_Abstractness_for_Typescript.cypher

@@ -10,16 +10,20 @@ OPTIONAL MATCH (projectdir:Directory)<-[:HAS_ROOT]-(project:TS:Project)-[:CONTAI
       ,count{(module)-[:EXPORTS]->(:TypeAlias)}             AS numberTypeAliases
       ,count{(module)-[:EXPORTS]->(:Interface)}             AS numberInterfaces
   WITH *
-      ,numberInterfaces + numberTypeAliases + numberAbstractClasses AS numberAbstractTypes
+      ,numberInterfaces + numberTypeAliases +  numberAbstractClasses        AS numberAbstractTypes
+      ,numberInterfaces + numberTypeAliases + (numberAbstractClasses * 0.7) AS weightedAbstractTypes
   WITH *
-      ,toFloat(numberAbstractTypes) / (numberTypes + 1E-38) AS abstractness
-   SET module.abstractness          = abstractness
-      ,module.numberOfAbstractTypes = numberAbstractTypes
-      ,module.numberOfTypes         = numberTypes
+      ,toFloat(weightedAbstractTypes) / (numberTypes + 1E-38) AS abstractness
+   SET module.abstractness            = abstractness
+      ,module.numberOfAbstractTypes   = numberAbstractTypes
+      ,module.numberOfAbstractClasses = numberAbstractClasses
+      ,module.numberOfTypes           = numberTypes
 RETURN projectName
       ,module.globalFqn  AS fullQualifiedModuleName
       ,module.name       AS moduleName
       ,abstractness
       ,numberAbstractTypes
       ,numberTypes
+      ,numberAbstractClasses
+      ,weightedAbstractTypes
 ORDER BY abstractness ASC, numberTypes DESC

cypher/Metrics/Clear_all_metrics.cypher

@@ -28,6 +28,7 @@ REMOVE package.incomingDependencies
       ,package.abstractness
       ,package.numberOfTypes
       ,package.numberOfAbstractTypes
+      ,package.numberOfAbstractClasses
       ,package.abstractnessIncludingSubpackages
       ,package.numberOfAbstractTypesIncludingSubpackages
       ,package.numberOfTypesIncludingSubpackages

domains/anomaly-detection/anomalyDetectionCsv.sh

@@ -64,6 +64,27 @@ anomaly_detection_features() {
     # Determine the normalized difference between Page Rank and Article Rank if not already done
     execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-PageToArticleRank-Exists.cypher" \
                                          "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-PageToArticleRank-Write.cypher" "${@}"
+    # Determine the "abstractness" (interfaces = 100%, abstract classes = 70%, classes & functions = 0%)
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-Abstractness-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature_Abstractness_Java.cypher" "${@}"
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-Abstractness-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature_Abstractness_JavaType.cypher" "${@}"
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-Abstractness-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature_Abstractness_TypeScriptModules.cypher" "${@}"
+    # Determines strongly connected components if not already done
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-Write.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-CreateNode.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-CreateDependency.cypher" "${@}"
+    # Determines weakly connected components if not already done
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-WeaklyConnectedComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-WeaklyConnectedComponents-Write.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-WeaklyConnectedComponents-CreateNode.cypher" "${@}"
+    # Determines topological sort max distance from source for strongly connected components if not already done
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Projection.cypher" "${@}"
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Write.cypher" "${@}"
 }
 
 # Run queries to find anomalies in the graph.

domains/anomaly-detection/anomalyDetectionPython.sh

@@ -111,6 +111,27 @@ anomaly_detection_features() {
     # Determine the normalized difference between Page Rank and Article Rank if not already done
     execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-PageToArticleRank-Exists.cypher" \
                                          "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-PageToArticleRank-Write.cypher" "${@}"
+    # Determine the "abstractness" (interfaces = 100%, abstract classes = 70%, classes & functions = 0%)
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-Abstractness-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature_Abstractness_Java.cypher" "${@}"
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-Abstractness-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature_Abstractness_JavaType.cypher" "${@}"
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-Abstractness-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature_Abstractness_TypeScriptModules.cypher" "${@}"
+    # Determines strongly connected components if not already done
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-Write.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-CreateNode.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-StronglyConnectedComponents-CreateDependency.cypher" "${@}"
+    # Determines weakly connected components if not already done
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-WeaklyConnectedComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-WeaklyConnectedComponents-Write.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-WeaklyConnectedComponents-CreateNode.cypher" "${@}"
+    # Determines topological sort max distance from source for strongly connected components if not already done
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Projection.cypher" "${@}"
+    execute_cypher_queries_until_results "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Exists.cypher" \
+                                         "${ANOMALY_DETECTION_FEATURE_CYPHER_DIR}/AnomalyDetectionFeature-TopologicalSortComponents-Write.cypher" "${@}"
 }
 
 # Execute the Python scripts for anomaly detection.

domains/anomaly-detection/documentation/Architecture.gv

@@ -114,8 +114,15 @@ digraph AnomalyDetectionPipeline {
         BetweennessCentrality      [label="Betweenness\nCentrality"];
         LocalClusteringCoefficient [label="Local Clustering\nCoefficient"];
         Degree                     [label="Degree\n(in, out, sum)"];
+        Abstractness               [label="Abstractness\n(Robert C. Martin)"];
+        StronglyConnectedComponents[label="StronglyConnectedComponents\n(member count)"]
+        WeaklyConnectedComponents  [label="WeaklyConnectedComponents\n(median members size)"]
+        TopologicalSort            [label="TopologicalSort\n(max component distance from source)"]
     }
 
+    // Inter graph algorithm feature connections
+    StronglyConnectedComponents -> TopologicalSort
+
     // Anomaly detection model area
     subgraph cluster_anomaly {
         label="Anomaly Detection Model";
@@ -152,6 +159,10 @@ digraph AnomalyDetectionPipeline {
     BetweennessCentrality -> AnomalyStandardizer;
     LocalClusteringCoefficient -> AnomalyStandardizer;
     Degree -> AnomalyStandardizer;
+    Abstractness -> AnomalyStandardizer;
+    StronglyConnectedComponents -> AnomalyStandardizer;
+    WeaklyConnectedComponents -> AnomalyStandardizer;
+    TopologicalSort -> AnomalyStandardizer;
 
     // Proxy RandomForest used as a backing/tuning model for the Isolation Forest
     TuningAnomaly -> IsolationMinCluster;