switch to gitleaks (#1518)

Switch to https://github.com/gitleaks/gitleaks, which seems to be faster
and less error-prone than the detect-secrets hook we previously were
using

---------

Signed-off-by: Peter St. John <pstjohn@nvidia.com>

Author: pstjohn

.gitleaks.toml

@@ -0,0 +1,12 @@
+# Gitleaks configuration
+# https://github.com/gitleaks/gitleaks
+
+title = "BioNeMo Framework Gitleaks Config"
+
+[extend]
+useDefault = true
+
+[allowlist]
+paths = [
+    '''3rdparty/''',
+]

.gitleaksignore

@@ -0,0 +1,5 @@
+# Gitleaks ignore file - fingerprints of known false positives
+# https://github.com/gitleaks/gitleaks#gitleaksignore
+
+# False positive: training hyperparameter string `lr=3e-4` matches generic-api-key rule
+bionemo-recipes/interpretability/sparse_autoencoders/recipes/esm2/README.md:generic-api-key:116

.pre-commit-config.yaml

@@ -40,13 +40,7 @@ repos:
         - mdformat-black
         - mdformat-frontmatter
         - black==25.12.0
-  - repo: https://github.com/Yelp/detect-secrets
-    rev: v1.5.0
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.24.2
     hooks:
-      - id: detect-secrets
-        name: detect-secrets (everything but notebooks)
-        args: ['--baseline', '.secrets.baseline', '--exclude-files', '(.*\.ipynb|.*\.baseline)$', ]
-        exclude: package.lock.json
-      - id: detect-secrets
-        name: detect-secrets (notebooks only)
-        args: ['--baseline', '.secrets-nb.baseline', '--exclude-files', '^.(?!.*\.ipynb)', '--exclude-lines', '"(hash|id|image/\w+)":.*|<.*at 0x[0-9a-f]+>|object at 0x[0-9a-f]+', ]
+      - id: gitleaks

.secrets-nb.baseline

@@ -82,11 +82,11 @@ override-dependencies = [
 # Internal dependencies
 bionemo-recipeutils = { git = "https://github.com/NVIDIA/bionemo-framework.git", branch = "main", subdirectory = "sub-packages/bionemo-recipeutils" }
 bionemo-core = { git = "https://github.com/NVIDIA/bionemo-framework.git", branch = "main", subdirectory = "sub-packages/bionemo-core" }
-nvidia-resiliency-ext = { git = "https://github.com/NVIDIA/nvidia-resiliency-ext.git", rev = "54f85fe422d296cf04ea524130014bd3a2c3add1" }  # pragma: allowlist secret
+nvidia-resiliency-ext = { git = "https://github.com/NVIDIA/nvidia-resiliency-ext.git", rev = "54f85fe422d296cf04ea524130014bd3a2c3add1" }
 
 # Megatron Bundle. This points to a version that still supports the deprecated no_weight_decay_cond field until the API for an alternative has been finalized.
-megatron-bridge = { git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git", rev = "549e3cb970c170b1d7a86d021261efe05e8a5d9f" }  # pragma: allowlist secret
-megatron-core = { git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git", rev = "549e3cb970c170b1d7a86d021261efe05e8a5d9f", subdirectory = "3rdparty/Megatron-LM" }  # pragma: allowlist secret
+megatron-bridge = { git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git", rev = "549e3cb970c170b1d7a86d021261efe05e8a5d9f" }
+megatron-core = { git = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git", rev = "549e3cb970c170b1d7a86d021261efe05e8a5d9f", subdirectory = "3rdparty/Megatron-LM" }
 
 [tool.uv.extra-build-dependencies]
 warp-lang = ["wheel_stub"]

.secrets.baseline

@@ -31,11 +31,11 @@
 DATASET_FILES = {
     "dataset_train55k_80%.txt": {
         "output": "porter6_train_dataset_55k.parquet",
-        "sha256": "4b1c011d8cea0b892743053eb4234db80344b8d9c90243f19b4637781ce8922b",  # pragma: allowlist secret
+        "sha256": "4b1c011d8cea0b892743053eb4234db80344b8d9c90243f19b4637781ce8922b",
     },
     "2024Testset_692.adataset": {
         "output": "porter6_val_dataset_2024_692.parquet",
-        "sha256": "b4a1b69f2003a66a62eb106aded784f9938fc734e876458223459fd9a10f1ca2",  # pragma: allowlist secret
+        "sha256": "b4a1b69f2003a66a62eb106aded784f9938fc734e876458223459fd9a10f1ca2",
     },
 }
 

bionemo-recipes/recipes/eden_megatron/pyproject.toml

@@ -3,7 +3,7 @@ defaults:
 - _self_
 
 model_tag: "nvidia/esm2_t6_8M_UR50D"
-base_model_config_dir: "<checkpoint.ckpt_dir>/train_ddp/final_model" # pragma: allowlist secret
+base_model_config_dir: "<checkpoint.ckpt_dir>/train_ddp/final_model"
 
 output_file: preds.csv
 

bionemo-recipes/recipes/esm2_peft_te/data/prepare_porter6_dataset.py

@@ -42,7 +42,7 @@ lr_scheduler_kwargs:
   num_training_steps: 1_000
 
 checkpoint:
-  ckpt_dir: "checkpoints/facebook_esm2_t48_15B_UR50D" # pragma: allowlist secret
+  ckpt_dir: "checkpoints/facebook_esm2_t48_15B_UR50D"
   save_final_model: false
 
 logger: