Remove constant features from anomaly detection

Author: JohT

domains/anomaly-detection/explore/AnomalyDetectionIsolationForestExploration.ipynb

@@ -407,14 +407,35 @@
     "    return scaler.fit_transform(features_to_scale)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "04d510a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def remove_constant_features(features: pd.DataFrame, feature_names: list[str]) -> list[str]:\n",
+    "    \"\"\"\n",
+    "    Removes constant features from the feature list.\n",
+    "    \"\"\"\n",
+    "    non_constant_features = []\n",
+    "    for feature in feature_names:\n",
+    "        if features[feature].nunique() > 1:\n",
+    "            non_constant_features.append(feature)\n",
+    "        else:\n",
+    "            print(f\"Removed constant feature: {feature}\")\n",
+    "    return non_constant_features"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "2de5ade1",
    "metadata": {},
    "outputs": [],
    "source": [
-    "java_package_anomaly_detection_features_standardized = standardize_features(java_package_anomaly_detection_features, java_package_features_to_standardize)"
+    "java_package_anomaly_detection_feature_names_to_standardize = remove_constant_features(java_package_anomaly_detection_features, java_package_features_to_standardize)\n",
+    "java_package_anomaly_detection_features_standardized = standardize_features(java_package_anomaly_detection_features, java_package_anomaly_detection_feature_names_to_standardize)"
    ]
   },
   {
@@ -491,7 +512,7 @@
    "outputs": [],
    "source": [
     "java_package_anomaly_detection_features_prepared = np.hstack([java_package_anomaly_detection_features_standardized, java_package_anomaly_detection_node_embeddings_reduced])\n",
-    "java_package_anomaly_detection_feature_names = list(java_package_features_to_standardize) + [f'nodeEmbeddingPCA_{i}' for i in range(java_package_anomaly_detection_node_embeddings_reduced.shape[1])]"
+    "java_package_anomaly_detection_feature_names = list(java_package_anomaly_detection_feature_names_to_standardize) + [f'nodeEmbeddingPCA_{i}' for i in range(java_package_anomaly_detection_node_embeddings_reduced.shape[1])]"
    ]
   },
   {
@@ -1986,13 +2007,14 @@
    "outputs": [],
    "source": [
     "validate_data(java_type_anomaly_detection_features)\n",
-    "java_type_anomaly_detection_features_standardized = standardize_features(java_type_anomaly_detection_features, java_type_features_to_standardize)\n",
+    "java_type_anomaly_detection_feature_names_to_standardize = remove_constant_features(java_type_anomaly_detection_features, java_type_features_to_standardize )\n",
+    "java_type_anomaly_detection_features_standardized = standardize_features(java_type_anomaly_detection_features, java_type_anomaly_detection_feature_names_to_standardize)\n",
     "java_type_anomaly_detection_node_embeddings_reduced = reduce_dimensionality_of_node_embeddings(java_type_anomaly_detection_features, max_dimensions=35)\n",
     "\n",
     "java_type_anomaly_detection_features_prepared = np.hstack([java_type_anomaly_detection_features_standardized, java_type_anomaly_detection_node_embeddings_reduced])\n",
-    "java_type_anomaly_detection_feature_names = list(java_type_features_to_standardize) + [f'nodeEmbeddingPCA_{i}' for i in range(java_type_anomaly_detection_node_embeddings_reduced.shape[1])]\n",
+    "java_type_anomaly_detection_feature_names = list(java_type_anomaly_detection_feature_names_to_standardize) + [f'nodeEmbeddingPCA_{i}' for i in range(java_type_anomaly_detection_node_embeddings_reduced.shape[1])]\n",
     "\n",
-    "plot_feature_correlation_matrix(java_type_anomaly_detection_features[java_type_features_to_standardize])"
+    "plot_feature_correlation_matrix(java_type_anomaly_detection_features[java_type_anomaly_detection_feature_names_to_standardize])"
    ]
   },
   {

domains/anomaly-detection/tunedAnomalyDetectionExplained.py

@@ -296,6 +296,20 @@ def standardize_features(features: pd.DataFrame, feature_list: list[str]) -> num
     return scaler.fit_transform(features_to_scale)
 
 
+def remove_constant_features(features: pd.DataFrame, feature_names: list[str], is_verbose: bool = False) -> list[str]:
+    """
+    Removes constant features from the feature list.
+    """
+    non_constant_features = []
+    for feature in feature_names:
+        if features[feature].nunique() > 1:
+            non_constant_features.append(feature)
+        else:
+            if is_verbose:
+                print("tunedAnomalyDetectionExplained: Removing constant feature {feature}")
+    return non_constant_features
+
+
 def reduce_dimensionality_of_node_embeddings(
         features: pd.DataFrame,
         min_dimensions: int = 20,
@@ -1162,6 +1176,7 @@ def output_top_shap_explained_global_features_as_markdown_table(
     sys.exit(0)
 
 features_to_standardize = features.columns.drop(features_for_visualization_to_exclude_from_training + ['embedding']).to_list()
+features_to_standardize = remove_constant_features(features, features_to_standardize, is_verbose=parameters.is_verbose())
 features_standardized = standardize_features(features, features_to_standardize)
 node_embeddings_reduced = reduce_dimensionality_of_node_embeddings(features)
 features_prepared = np.hstack([features_standardized, node_embeddings_reduced])