improve ai extraction from policy

Author: sw33tLie

cmd/poll.go

@@ -564,28 +564,20 @@ func mergePolicyItems(processedItems []storage.TargetItem, policyItems []storage
 		return processedItems
 	}
 
-	// Build a set of all existing variant values (normalized) per category across all items,
-	// so we can detect duplicates.
-	type variantKey struct {
-		normalized string
-		category   string
-	}
-	existingVariants := make(map[variantKey]bool)
+	// Build a set of all existing target values (normalized, scheme-stripped).
+	// We deduplicate by normalized target ONLY (ignoring category), because
+	// the same domain should not be added as a variant if it already exists
+	// as a raw target or AI variant under any category.
+	existingTargets := make(map[string]bool)
 	for _, item := range processedItems {
-		// The raw target itself is also "known" — normalize it for comparison
 		rawNorm := normForDedup(item.URI)
-		cat := strings.ToLower(strings.TrimSpace(scope.NormalizeCategory(item.Category)))
 		if rawNorm != "" {
-			existingVariants[variantKey{normalized: rawNorm, category: cat}] = true
+			existingTargets[rawNorm] = true
 		}
 		for _, v := range item.Variants {
 			vNorm := normForDedup(v.Value)
-			vCat := cat
-			if v.HasCategory {
-				vCat = strings.ToLower(strings.TrimSpace(v.Category))
-			}
 			if vNorm != "" {
-				existingVariants[variantKey{normalized: vNorm, category: vCat}] = true
+				existingTargets[vNorm] = true
 			}
 		}
 	}
@@ -607,8 +599,8 @@ func mergePolicyItems(processedItems []storage.TargetItem, policyItems []storage
 			continue
 		}
 
-		// Skip if this target+category already exists
-		if existingVariants[variantKey{normalized: piNorm, category: piCat}] {
+		// Skip if this target already exists under any category
+		if existingTargets[piNorm] {
 			continue
 		}
 
@@ -639,7 +631,7 @@ func mergePolicyItems(processedItems []storage.TargetItem, policyItems []storage
 			HasCategory: true,
 			Category:    piCat,
 		})
-		existingVariants[variantKey{normalized: piNorm, category: piCat}] = true
+		existingTargets[piNorm] = true
 	}
 
 	return processedItems

website/pkg/core/poller.go

@@ -444,25 +444,17 @@ func webMergePolicyItems(processedItems []storage.TargetItem, policyItems []stor
 		return processedItems
 	}
 
-	type variantKey struct {
-		normalized string
-		category   string
-	}
-	existingVariants := make(map[variantKey]bool)
+	// Deduplicate by normalized target ONLY (ignoring category).
+	existingTargets := make(map[string]bool)
 	for _, item := range processedItems {
 		rawNorm := webNormForDedup(item.URI)
-		cat := strings.ToLower(strings.TrimSpace(scope.NormalizeCategory(item.Category)))
 		if rawNorm != "" {
-			existingVariants[variantKey{normalized: rawNorm, category: cat}] = true
+			existingTargets[rawNorm] = true
 		}
 		for _, v := range item.Variants {
 			vNorm := webNormForDedup(v.Value)
-			vCat := cat
-			if v.HasCategory {
-				vCat = strings.ToLower(strings.TrimSpace(v.Category))
-			}
 			if vNorm != "" {
-				existingVariants[variantKey{normalized: vNorm, category: vCat}] = true
+				existingTargets[vNorm] = true
 			}
 		}
 	}
@@ -482,7 +474,7 @@ func webMergePolicyItems(processedItems []storage.TargetItem, policyItems []stor
 			continue
 		}
 
-		if existingVariants[variantKey{normalized: piNorm, category: piCat}] {
+		if existingTargets[piNorm] {
 			continue
 		}
 
@@ -511,7 +503,7 @@ func webMergePolicyItems(processedItems []storage.TargetItem, policyItems []stor
 			HasCategory: true,
 			Category:    piCat,
 		})
-		existingVariants[variantKey{normalized: piNorm, category: piCat}] = true
+		existingTargets[piNorm] = true
 	}
 
 	return processedItems