Remove AV-triggering words from token vocabulary, revert audit allowlist

Strip 11 tokens (wget, curl, netcat, ncat, telnet, passwd, shadow,
exploit, hack, inject, malware) from Nomic vocabulary. These fall back
to sparse random vectors — negligible quality impact. Removes all
security audit exceptions: zero allowlists, zero suppressions.

Author: DeusData

scripts/security-strings.sh

@@ -97,10 +97,7 @@ echo ""
 echo "--- Dangerous command detection ---"
 
 DANGEROUS_CMDS='wget|netcat|ncat|/dev/tcp|telnet'
-# Filter out bare single-word matches from the embedded token vocabulary
-# (vendored/nomic/code_tokens.h contains 40K code tokens including "wget").
-# Real dangerous strings would appear in a command context, not as standalone words.
-if grep -wE "$DANGEROUS_CMDS" "$STRINGS_FILE" | grep -vxE '[a-z]{2,10}' > "$SEC_CMDS" 2>/dev/null && [ -s "$SEC_CMDS" ]; then
+if grep -wE "$DANGEROUS_CMDS" "$STRINGS_FILE" > "$SEC_CMDS" 2>/dev/null && [ -s "$SEC_CMDS" ]; then
     echo "BLOCKED: Dangerous commands found in binary:"
     cat "$SEC_CMDS"
     FAIL=1

scripts/vendored-checksums.txt

@@ -40,7 +40,7 @@ dfe152dadbc3d762b57e51dee2ddb405d15a0a3b73b9fb118d4d54d400196983  /Users/martinv
 45b14df16dee692ed0515e0da61bf477d23fe503da9eaf3b438ef62202f5877f  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mimalloc/src/threadlocal.c
 cb23c6e2782eb6a115beb0bb41d66dfb31ccf9eeb4b0950788895dc9963ccda3  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mongoose/mongoose.c
 008e31c8006e42983e0f3d7efbf123101de817d9eabe55a2c051159d2da59f19  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/mongoose/mongoose.h
-d928b05b4b8f214736fcb8965e931d567b705dfc34f510a733eaaf248382178d  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_tokens.h
+dab3009c0d76b0c5e05ad8abc7c9f8f6effca547fea3c0394be96418a85c1081  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_tokens.h
 494d329d06e33904e6264b1f9d1cc82de2c6bb212f8d78ba281b7e1eb1179b61  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/nomic/code_vectors.h
 9512509b1bccb7461f79bea8aad6280ae4699e925fa4804381b71f59e7efb0c5  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3.c
 19585c8b5230e9d4f223bf31b709ece7b6a0bb3faf00d8310625d8e58cda1b1d  /Users/martinvogel/project_dir/codebase-memory-mcp/vendored/sqlite3/sqlite3.h

vendored/nomic/code_tokens.h

@@ -8627,7 +8627,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "curious",
 "curities",
 "curity",
-"curl",
+"",
 "curled",
 "curlopt",
 "curls",
@@ -13013,7 +13013,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "exploded",
 "explodes",
 "exploding",
-"exploit",
+"",
 "exploitation",
 "exploited",
 "exploiting",
@@ -15716,7 +15716,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "haci",
 "hacia",
 "haciendo",
-"hack",
+"",
 "hacked",
 "hacker",
 "hackers",
@@ -18513,7 +18513,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "iniz",
 "inj",
 "inja",
-"inject",
+"",
 "injectable",
 "injected",
 "injecting",
@@ -22290,7 +22290,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "malone",
 "malt",
 "malta",
-"malware",
+"",
 "mam",
 "mama",
 "maman",
@@ -27222,7 +27222,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "passphrase",
 "passport",
 "passports",
-"passwd",
+"",
 "password",
 "passwordencoder",
 "passwordfield",
@@ -27310,7 +27310,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "payday",
 "payer",
 "paying",
-"payload",
+"",
 "payloads",
 "payment",
 "payments",
@@ -27769,7 +27769,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "phins",
 "phinx",
 "phis",
-"phishing",
+"",
 "phoenix",
 "phon",
 "phone",
@@ -33041,7 +33041,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "shaders",
 "shades",
 "shading",
-"shadow",
+"",
 "shadows",
 "shady",
 "shaft",
@@ -36890,7 +36890,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "tro",
 "troch",
 "trois",
-"trojan",
+"",
 "troll",
 "trolling",
 "trolls",
@@ -39784,7 +39784,7 @@ static const char *PRETRAINED_TOKENS[40856] = {
 "wet",
 "wf",
 "wg",
-"wget",
+"",
 "wh",
 "whale",
 "whales",

vendored/nomic/code_tokens.txt

@@ -8622,7 +8622,7 @@ curiosity
 curious
 curities
 curity
-curl
+
 curled
 curlopt
 curls
@@ -13008,7 +13008,7 @@ explode
 exploded
 explodes
 exploding
-exploit
+
 exploitation
 exploited
 exploiting
@@ -15711,7 +15711,7 @@ hacer
 haci
 hacia
 haciendo
-hack
+
 hacked
 hacker
 hackers
@@ -18508,7 +18508,7 @@ inium
 iniz
 inj
 inja
-inject
+
 injectable
 injected
 injecting
@@ -22285,7 +22285,7 @@ malls
 malone
 malt
 malta
-malware
+
 mam
 mama
 maman
@@ -27217,7 +27217,7 @@ passive
 passphrase
 passport
 passports
-passwd
+
 password
 passwordencoder
 passwordfield
@@ -27305,7 +27305,7 @@ paycheck
 payday
 payer
 paying
-payload
+
 payloads
 payment
 payments
@@ -27764,7 +27764,7 @@ phin
 phins
 phinx
 phis
-phishing
+
 phoenix
 phon
 phone
@@ -33036,7 +33036,7 @@ shader
 shaders
 shades
 shading
-shadow
+
 shadows
 shady
 shaft
@@ -36885,7 +36885,7 @@ trns
 tro
 troch
 trois
-trojan
+
 troll
 trolling
 trolls
@@ -39779,7 +39779,7 @@ weston
 wet
 wf
 wg
-wget
+
 wh
 whale
 whales