feat: Implement anonymization operators and validation logic

- Added RecognizerResult class for handling analyzer results with score and index management.
- Introduced RedactOperator and ReplaceOperator for text anonymization.
- Created TextReplaceBuilder for managing text replacements while preserving original indices.
- Developed Validators class for input validation mirroring Python validators.
- Implemented unit tests for AnonymizerEngine, ConflictResolutionStrategy, and various operators.
- Added support for custom operators and conflict resolution strategies in the anonymization process.
- Enhanced error handling and validation for operator parameters.

Author: KSemenenko

src/ManagedCode.Presidio.Analyzer/ItFiscalCodeRecognizer.cs

@@ -121,5 +121,3 @@ public sealed class ItFiscalCodeRecognizer(
         return expected == control ? true : null;
     }
 }
-using System.Collections.Generic;
-using System.Linq;

src/ManagedCode.Presidio.Analyzer/ItVatCodeRecognizer.cs

@@ -72,4 +72,3 @@ public sealed class ItVatCodeRecognizer(
         return c == sanitized[10] - '0';
     }
 }
-using System.Linq;

src/ManagedCode.Presidio.Analyzer/RemoteRecognizer.cs

@@ -1,3 +1,5 @@
+using ManagedCode.Presidio.Core;
+
 namespace ManagedCode.Presidio.Analyzer;
 
 /// <summary>
@@ -20,7 +22,7 @@ protected abstract IReadOnlyCollection<RecognizerResult> AnalyzeRemote(
         IReadOnlyCollection<string> entities,
         NlpArtifacts artifacts);
 
-    public override IReadOnlyCollection<string> EnhanceUsingContext(
+    public override IReadOnlyCollection<RecognizerResult> EnhanceUsingContext(
         string text,
         IReadOnlyCollection<RecognizerResult> ownResults,
         IReadOnlyCollection<RecognizerResult> otherResults,
@@ -40,4 +42,3 @@ protected override IReadOnlyCollection<RecognizerResult> AnalyzeCore(
         return AnalyzeRemote(text, entities, artifacts);
     }
 }
-

src/ManagedCode.Presidio.Anonymizer/AesCipher.cs

@@ -0,0 +1,88 @@
+using System.Security.Cryptography;
+using System.Text;
+
+namespace ManagedCode.Presidio.Anonymizer;
+
+internal static class AesCipher
+{
+    public static string Encrypt(byte[] key, string text)
+    {
+        using var aes = CreateAes(key);
+        aes.GenerateIV();
+
+        using var encryptor = aes.CreateEncryptor();
+        var plainBytes = Encoding.UTF8.GetBytes(text ?? string.Empty);
+        var encrypted = encryptor.TransformFinalBlock(plainBytes, 0, plainBytes.Length);
+
+        var payload = new byte[aes.IV.Length + encrypted.Length];
+        Buffer.BlockCopy(aes.IV, 0, payload, 0, aes.IV.Length);
+        Buffer.BlockCopy(encrypted, 0, payload, aes.IV.Length, encrypted.Length);
+
+        return ToBase64Url(payload);
+    }
+
+    public static string Decrypt(byte[] key, string text)
+    {
+        using var aes = CreateAes(key);
+        var payload = FromBase64Url(text ?? string.Empty);
+        if (payload.Length < aes.BlockSize / 8)
+        {
+            throw new InvalidParamException("Ciphertext payload is too small");
+        }
+
+        var ivLength = aes.BlockSize / 8;
+        var iv = new byte[ivLength];
+        Buffer.BlockCopy(payload, 0, iv, 0, ivLength);
+        aes.IV = iv;
+
+        var cipherText = new byte[payload.Length - ivLength];
+        Buffer.BlockCopy(payload, ivLength, cipherText, 0, cipherText.Length);
+
+        using var decryptor = aes.CreateDecryptor();
+        var decrypted = decryptor.TransformFinalBlock(cipherText, 0, cipherText.Length);
+        return Encoding.UTF8.GetString(decrypted);
+    }
+
+    public static bool IsValidKeySize(byte[] key)
+    {
+        if (key is null)
+        {
+            return false;
+        }
+
+        var bitLength = key.Length * 8;
+        return bitLength == 128 || bitLength == 192 || bitLength == 256;
+    }
+
+    private static Aes CreateAes(byte[] key)
+    {
+        if (!IsValidKeySize(key))
+        {
+            throw new InvalidParamException("Invalid input, key must be of length 128, 192 or 256 bits");
+        }
+
+        var aes = Aes.Create();
+        aes.Key = key;
+        aes.Mode = CipherMode.CBC;
+        aes.Padding = PaddingMode.PKCS7;
+        return aes;
+    }
+
+    private static string ToBase64Url(byte[] payload)
+    {
+        var base64 = Convert.ToBase64String(payload);
+        return base64.Replace('+', '-').Replace('/', '_').TrimEnd('=');
+    }
+
+    private static byte[] FromBase64Url(string text)
+    {
+        var normalized = text.Replace('-', '+').Replace('_', '/');
+        var padding = 4 - normalized.Length % 4;
+        if (padding is > 0 and < 4)
+        {
+            normalized = normalized.PadRight(normalized.Length + padding, '=');
+        }
+
+        return Convert.FromBase64String(normalized);
+    }
+}

src/ManagedCode.Presidio.Anonymizer/AnonymizerEngine.cs

@@ -0,0 +1,234 @@
+using System.Text.RegularExpressions;
+using CoreRecognizerResult = ManagedCode.Presidio.Core.RecognizerResult;
+
+namespace ManagedCode.Presidio.Anonymizer;
+
+/// <summary>
+/// Entry point for Presidio anonymization pipeline.
+/// </summary>
+public sealed class AnonymizerEngine : EngineBase
+{
+    private static readonly Regex WhitespaceRegex = new("^( )+$", RegexOptions.Compiled);
+
+    public EngineResult Anonymize(
+        string text,
+        IReadOnlyCollection<CoreRecognizerResult>? analyzerResults,
+        IDictionary<string, OperatorConfig>? operators = null,
+        ConflictResolutionStrategy conflictResolution = ConflictResolutionStrategy.MergeSimilarOrContained)
+    {
+        ArgumentNullException.ThrowIfNull(text);
+        var safeResults = analyzerResults ?? Array.Empty<CoreRecognizerResult>();
+        ValidateIndices(text, safeResults);
+        var mutableResults = CopyRecognizerResults(safeResults);
+        if (mutableResults.Count == 0)
+        {
+            var defaultOperators = EnsureDefaultOperator(operators);
+            return Operate(text, Array.Empty<PiiEntity>(), defaultOperators, OperatorType.Anonymize);
+        }
+
+        mutableResults.Sort();
+        mutableResults = RemoveConflictsAndGetTextManipulationData(mutableResults, conflictResolution);
+        mutableResults = MergeEntitiesWithWhitespaceBetween(text, mutableResults);
+        var operatorMap = EnsureDefaultOperator(operators);
+
+        return Operate(
+            text,
+            mutableResults.Cast<PiiEntity>().ToArray(),
+            operatorMap,
+            OperatorType.Anonymize);
+    }
+
+    public IReadOnlyCollection<string> GetAnonymizers() => OperatorsFactory.GetAnonymizers().Keys.ToArray();
+
+    public void AddAnonymizer<T>() where T : Operator, new() => OperatorsFactory.AddAnonymizeOperator(typeof(T));
+
+    public void AddAnonymizer(Type operatorType) => OperatorsFactory.AddAnonymizeOperator(operatorType);
+
+    public void RemoveAnonymizer(Type operatorType) => OperatorsFactory.RemoveAnonymizeOperator(operatorType);
+
+    private static void ValidateIndices(string text, IEnumerable<CoreRecognizerResult> analyzerResults)
+    {
+        foreach (var result in analyzerResults)
+        {
+            if (result.Start < 0 || result.Start > text.Length || result.End < 0 || result.End > text.Length)
+            {
+                throw new InvalidParamException(
+                    $"Invalid analyzer result, start: {result.Start} and end: {result.End}, while text length is only {text.Length}.");
+            }
+        }
+    }
+
+    private static List<RecognizerResult> CopyRecognizerResults(IReadOnlyCollection<CoreRecognizerResult> analyzerResults)
+    {
+        var results = new List<RecognizerResult>(analyzerResults.Count);
+        foreach (var result in analyzerResults)
+        {
+            results.Add(RecognizerResult.FromCore(result));
+        }
+
+        return results;
+    }
+
+    private static List<RecognizerResult> RemoveConflictsAndGetTextManipulationData(
+        List<RecognizerResult> analyzerResults,
+        ConflictResolutionStrategy conflictResolution)
+    {
+        var tmpAnalyzerResults = new List<RecognizerResult>();
+        var otherElements = new List<RecognizerResult>(analyzerResults);
+
+        foreach (var result in analyzerResults)
+        {
+            otherElements.Remove(result);
+
+            var isMergeSameEntityType = false;
+            foreach (var other in otherElements)
+            {
+                if (!string.Equals(result.EntityType, other.EntityType, StringComparison.Ordinal))
+                {
+                    continue;
+                }
+
+                if (result.Intersects(other) == 0)
+                {
+                    continue;
+                }
+
+                other.UpdateSpan(Math.Min(result.Start, other.Start), Math.Max(result.End, other.End));
+                other.UpdateScore(Math.Max(result.Score, other.Score));
+                isMergeSameEntityType = true;
+                break;
+            }
+
+            if (!isMergeSameEntityType)
+            {
+                otherElements.Add(result);
+                tmpAnalyzerResults.Add(result);
+            }
+        }
+
+        var uniqueElements = new List<RecognizerResult>();
+        otherElements = new List<RecognizerResult>(tmpAnalyzerResults);
+        foreach (var result in tmpAnalyzerResults)
+        {
+            otherElements.Remove(result);
+            if (!IsResultConflicted(otherElements, result))
+            {
+                otherElements.Add(result);
+                uniqueElements.Add(result);
+            }
+        }
+
+        if (conflictResolution == ConflictResolutionStrategy.RemoveIntersections)
+        {
+            uniqueElements.Sort((a, b) => a.Start.CompareTo(b.Start));
+            var index = 0;
+            while (index < uniqueElements.Count - 1)
+            {
+                var current = uniqueElements[index];
+                var next = uniqueElements[index + 1];
+
+                if (current.End <= next.Start)
+                {
+                    index += 1;
+                    continue;
+                }
+
+                if (current.Score >= next.Score)
+                {
+                    if (current.End >= next.End)
+                    {
+                        uniqueElements.RemoveAt(index + 1);
+                        continue;
+                    }
+
+                    next.UpdateSpan(current.End, next.End);
+                }
+                else
+                {
+                    if (next.Start <= current.Start)
+                    {
+                        uniqueElements.RemoveAt(index);
+                        continue;
+                    }
+
+                    current.UpdateSpan(current.Start, next.Start);
+                }
+
+                uniqueElements.Sort((a, b) => a.Start.CompareTo(b.Start));
+                index = 0;
+            }
+
+            uniqueElements = uniqueElements
+                .Where(element => element.Start <= element.End)
+                .ToList();
+        }
+
+        return uniqueElements;
+    }
+
+    private static List<RecognizerResult> MergeEntitiesWithWhitespaceBetween(
+        string text,
+        List<RecognizerResult> analyzerResults)
+    {
+        var mergedResults = new List<RecognizerResult>();
+        RecognizerResult? previous = null;
+        foreach (var result in analyzerResults)
+        {
+            if (previous is not null &&
+                string.Equals(previous.EntityType, result.EntityType, StringComparison.Ordinal) &&
+                IsWhitespace(text, previous.End, result.Start))
+            {
+                mergedResults.Remove(previous);
+                result.UpdateSpan(previous.Start, result.End);
+            }
+
+            mergedResults.Add(result);
+            previous = result;
+        }
+
+        return mergedResults;
+    }
+
+    private static bool IsWhitespace(string text, int start, int end)
+    {
+        if (start < 0 || end > text.Length || end <= start)
+        {
+            return false;
+        }
+
+        var span = text[start..end];
+        return WhitespaceRegex.IsMatch(span);
+    }
+
+    private static bool IsResultConflicted(IEnumerable<RecognizerResult> others, RecognizerResult candidate)
+    {
+        foreach (var other in others)
+        {
+            if (candidate.HasConflict(other))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private static Dictionary<string, OperatorConfig> EnsureDefaultOperator(IDictionary<string, OperatorConfig>? operators)
+    {
+        var map = new Dictionary<string, OperatorConfig>(StringComparer.Ordinal);
+        if (operators is not null)
+        {
+            foreach (var pair in operators)
+            {
+                map[pair.Key] = pair.Value;
+            }
+        }
+
+        if (!map.ContainsKey("DEFAULT"))
+        {
+            map["DEFAULT"] = new OperatorConfig("replace");
+        }
+
+        return map;
+    }
+}

src/ManagedCode.Presidio.Anonymizer/AssemblyAttributes.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("ManagedCode.Presidio.Anonymizer.Tests")]

src/ManagedCode.Presidio.Anonymizer/ConflictResolutionStrategy.cs

@@ -0,0 +1,7 @@
+namespace ManagedCode.Presidio.Anonymizer;
+
+public enum ConflictResolutionStrategy
+{
+    MergeSimilarOrContained,
+    RemoveIntersections,
+}